test(app): route prompt e2e through mock llm

packages/app/e2e/prompt/prompt.spec.ts

@@ -1,44 +1,66 @@
-import fs from "node:fs/promises"
-import path from "node:path"
 import { test, expect } from "../fixtures"
 import { promptSelector } from "../selectors"
 import { sessionIDFromUrl } from "../actions"
-import { createSdk } from "../utils"
 
-async function config(dir: string, url: string) {
-  await fs.writeFile(
-    path.join(dir, "opencode.json"),
-    JSON.stringify({
-      $schema: "https://opencode.ai/config.json",
-      enabled_providers: ["e2e-llm"],
-      provider: {
-        "e2e-llm": {
-          name: "E2E LLM",
-          npm: "@ai-sdk/openai-compatible",
-          env: [],
-          models: {
-            "test-model": {
-              name: "Test Model",
-              tool_call: true,
-              limit: { context: 128000, output: 32000 },
-            },
-          },
-          options: {
-            apiKey: "test-key",
-            baseURL: url,
-          },
-        },
-      },
-      agent: {
-        build: {
-          model: "e2e-llm/test-model",
-        },
-      },
-    }),
-  )
+const mdl = { providerID: "openai", modelID: "gpt-5.3-chat-latest" }
+
+async function pickModel(page: Parameters<typeof test>[0]["page"], value: { providerID: string; modelID: string }) {
+  await expect
+    .poll(
+      () =>
+        page.evaluate(() => {
+          const win = window as Window & {
+            __opencode_e2e?: {
+              model?: {
+                controls?: {
+                  setModel?: (value: { providerID: string; modelID: string } | undefined) => void
+                }
+              }
+            }
+          }
+          return !!win.__opencode_e2e?.model?.controls?.setModel
+        }),
+      { timeout: 30_000 },
+    )
+    .toBe(true)
+
+  await page.evaluate((value) => {
+    const win = window as Window & {
+      __opencode_e2e?: {
+        model?: {
+          controls?: {
+            setModel?: (value: { providerID: string; modelID: string } | undefined) => void
+          }
+        }
+      }
+    }
+    const fn = win.__opencode_e2e?.model?.controls?.setModel
+    if (!fn) throw new Error("Model e2e model control is not enabled")
+    fn(value)
+  }, value)
+
+  await expect
+    .poll(
+      () =>
+        page.evaluate(() => {
+          const win = window as Window & {
+            __opencode_e2e?: {
+              model?: {
+                current?: {
+                  model?: { providerID: string; modelID: string }
+                }
+              }
+            }
+          }
+          const model = win.__opencode_e2e?.model?.current?.model
+          return model ? `${model.providerID}/${model.modelID}` : null
+        }),
+      { timeout: 30_000 },
+    )
+    .toBe(`${value.providerID}/${value.modelID}`)
 }
 
-test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => {
+test("can send a prompt and receive a reply", async ({ page, llm, sdk, gotoSession }) => {
   test.setTimeout(120_000)
 
   const pageErrors: string[] = []
@@ -47,50 +69,65 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject })
   }
   page.on("pageerror", onPageError)
 
+  const prev = await sdk.global.config.get().then((res) => res.data ?? {})
+
   try {
-    await withProject(
-      async (project) => {
-        const sdk = createSdk(project.directory)
-        const token = `E2E_OK_${Date.now()}`
-
-        await llm.text(token)
-        await project.gotoSession()
-
-        const prompt = page.locator(promptSelector)
-        await prompt.click()
-        await page.keyboard.type(`Reply with exactly: ${token}`)
-        await page.keyboard.press("Enter")
-
-        await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
-
-        const sessionID = (() => {
-          const id = sessionIDFromUrl(page.url())
-          if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
-          return id
-        })()
-        project.trackSession(sessionID)
-
-        await expect
-          .poll(
-            async () => {
-              const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
-              return messages
-                .filter((m) => m.info.role === "assistant")
-                .flatMap((m) => m.parts)
-                .filter((p) => p.type === "text")
-                .map((p) => p.text)
-                .join("\n")
+    await sdk.global.config.update({
+      config: {
+        ...prev,
+        model: `${mdl.providerID}/${mdl.modelID}`,
+        enabled_providers: ["openai"],
+        provider: {
+          ...prev.provider,
+          openai: {
+            ...prev.provider?.openai,
+            options: {
+              ...prev.provider?.openai?.options,
+              apiKey: "test-key",
+              baseURL: llm.url,
             },
-            { timeout: 30_000 },
-          )
-          .toContain(token)
+          },
+        },
       },
-      {
-        model: { providerID: "e2e-llm", modelID: "test-model" },
-        setup: (dir) => config(dir, llm.url),
-      },
-    )
+    })
+
+    const token = `E2E_OK_${Date.now()}`
+    await llm.text("E2E Title")
+    await llm.text(token)
+    await gotoSession()
+    await pickModel(page, mdl)
+
+    const prompt = page.locator(promptSelector)
+    await prompt.click()
+    await page.keyboard.type(`Reply with exactly: ${token}`)
+    await page.keyboard.press("Enter")
+
+    await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
+
+    const sessionID = (() => {
+      const id = sessionIDFromUrl(page.url())
+      if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
+      return id
+    })()
+
+    await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(2)
+
+    await expect
+      .poll(
+        async () => {
+          const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
+          return messages
+            .filter((m) => m.info.role === "assistant")
+            .flatMap((m) => m.parts)
+            .filter((p) => p.type === "text")
+            .map((p) => p.text)
+            .join("\n")
+        },
+        { timeout: 30_000 },
+      )
+      .toContain(token)
   } finally {
+    await sdk.global.config.update({ config: prev })
     page.off("pageerror", onPageError)
   }
 

packages/opencode/test/lib/llm-server.ts

@@ -119,6 +119,225 @@ function bytes(input: Iterable<unknown>) {
   return Stream.fromIterable([...input].map(line)).pipe(Stream.encodeText)
 }
 
+function created(model: string) {
+  return {
+    type: "response.created",
+    sequence_number: 1,
+    response: {
+      id: "resp_test",
+      created_at: Math.floor(Date.now() / 1000),
+      model,
+      service_tier: null,
+    },
+  }
+}
+
+function completed(input: { seq: number; usage?: Usage }) {
+  return {
+    type: "response.completed",
+    sequence_number: input.seq,
+    response: {
+      incomplete_details: null,
+      service_tier: null,
+      usage: {
+        input_tokens: input.usage?.input ?? 0,
+        input_tokens_details: { cached_tokens: null },
+        output_tokens: input.usage?.output ?? 0,
+        output_tokens_details: { reasoning_tokens: null },
+      },
+    },
+  }
+}
+
+function responses(item: Sse, model: string) {
+  let seq = 1
+  let msg: string | undefined
+  let reason: string | undefined
+  let call:
+    | {
+        id: string
+        item: string
+        name: string
+        args: string
+      }
+    | undefined
+  let usage: Usage | undefined
+  const lines: unknown[] = [created(model)]
+
+  const all = [...item.head, ...item.tail]
+  for (const part of all) {
+    if (!part || typeof part !== "object") continue
+    if (!("choices" in part) || !Array.isArray(part.choices)) continue
+    const choice = part.choices[0]
+    if (!choice || typeof choice !== "object") continue
+    const delta = "delta" in choice && choice.delta && typeof choice.delta === "object" ? choice.delta : undefined
+
+    if (delta && "content" in delta && typeof delta.content === "string") {
+      msg ||= "msg_1"
+      if (
+        !lines.some(
+          (item) =>
+            typeof item === "object" &&
+            item &&
+            "type" in item &&
+            item.type === "response.output_item.added" &&
+            "item" in item &&
+            item.item &&
+            typeof item.item === "object" &&
+            "id" in item.item &&
+            item.item.id === msg,
+        )
+      ) {
+        seq += 1
+        lines.push({
+          type: "response.output_item.added",
+          sequence_number: seq,
+          output_index: 0,
+          item: { type: "message", id: msg },
+        })
+      }
+      seq += 1
+      lines.push({
+        type: "response.output_text.delta",
+        sequence_number: seq,
+        item_id: msg,
+        delta: delta.content,
+        logprobs: null,
+      })
+    }
+
+    if (delta && "reasoning_content" in delta && typeof delta.reasoning_content === "string") {
+      reason ||= "rs_1"
+      if (
+        !lines.some(
+          (item) =>
+            typeof item === "object" &&
+            item &&
+            "type" in item &&
+            item.type === "response.output_item.added" &&
+            "item" in item &&
+            item.item &&
+            typeof item.item === "object" &&
+            "id" in item.item &&
+            item.item.id === reason,
+        )
+      ) {
+        seq += 1
+        lines.push({
+          type: "response.output_item.added",
+          sequence_number: seq,
+          output_index: 0,
+          item: { type: "reasoning", id: reason, encrypted_content: null },
+        })
+        seq += 1
+        lines.push({
+          type: "response.reasoning_summary_part.added",
+          sequence_number: seq,
+          item_id: reason,
+          summary_index: 0,
+        })
+      }
+      seq += 1
+      lines.push({
+        type: "response.reasoning_summary_text.delta",
+        sequence_number: seq,
+        item_id: reason,
+        summary_index: 0,
+        delta: delta.reasoning_content,
+      })
+    }
+
+    if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) {
+      for (const tool of delta.tool_calls) {
+        if (!tool || typeof tool !== "object") continue
+        const fn = "function" in tool && tool.function && typeof tool.function === "object" ? tool.function : undefined
+        const id = "id" in tool && typeof tool.id === "string" ? tool.id : call?.id
+        const name = fn && "name" in fn && typeof fn.name === "string" ? fn.name : call?.name
+        const args = fn && "arguments" in fn && typeof fn.arguments === "string" ? fn.arguments : ""
+        if (!id || !name) continue
+        if (!call) {
+          call = { id, item: "fc_1", name, args: "" }
+          seq += 1
+          lines.push({
+            type: "response.output_item.added",
+            sequence_number: seq,
+            output_index: 0,
+            item: {
+              type: "function_call",
+              id: call.item,
+              call_id: id,
+              name,
+              arguments: "",
+              status: "in_progress",
+            },
+          })
+        }
+        call.args += args
+        if (args) {
+          seq += 1
+          lines.push({
+            type: "response.function_call_arguments.delta",
+            sequence_number: seq,
+            output_index: 0,
+            item_id: call.item,
+            delta: args,
+          })
+        }
+      }
+    }
+
+    if ("usage" in part && part.usage && typeof part.usage === "object") {
+      const raw = part.usage as Record<string, unknown>
+      if (typeof raw.prompt_tokens === "number" && typeof raw.completion_tokens === "number") {
+        usage = { input: raw.prompt_tokens, output: raw.completion_tokens }
+      }
+    }
+  }
+
+  if (msg) {
+    seq += 1
+    lines.push({
+      type: "response.output_item.done",
+      sequence_number: seq,
+      output_index: 0,
+      item: { type: "message", id: msg },
+    })
+  }
+  if (reason) {
+    seq += 1
+    lines.push({
+      type: "response.output_item.done",
+      sequence_number: seq,
+      output_index: 0,
+      item: { type: "reasoning", id: reason, encrypted_content: null },
+    })
+  }
+  if (call && !item.hang && !item.error) {
+    seq += 1
+    lines.push({
+      type: "response.output_item.done",
+      sequence_number: seq,
+      output_index: 0,
+      item: {
+        type: "function_call",
+        id: call.item,
+        call_id: call.id,
+        name: call.name,
+        arguments: call.args,
+        status: "completed",
+      },
+    })
+  }
+  if (!item.hang && !item.error) lines.push(completed({ seq: seq + 1, usage }))
+  return { ...item, head: lines, tail: [] } satisfies Sse
+}
+
+function modelFrom(body: unknown) {
+  if (!body || typeof body !== "object") return "test-model"
+  if (!("model" in body) || typeof body.model !== "string") return "test-model"
+  return body.model
+}
+
 function send(item: Sse) {
   const head = bytes(item.head)
   const tail = bytes([...item.tail, ...(item.hang || item.error ? [] : [done])])
@@ -358,6 +577,9 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
             },
           ]
           yield* notify()
+          if (req.originalUrl.endsWith("/v1/responses") && next.type === "sse") {
+            return send(responses(next, modelFrom(body)))
+          }
           if (next.type === "sse" && next.reset) {
             yield* reset(next)
             return HttpServerResponse.empty()
@@ -367,6 +589,27 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
         }),
       )
 
+      yield* router.add(
+        "POST",
+        "/v1/responses",
+        Effect.gen(function* () {
+          const req = yield* HttpServerRequest.HttpServerRequest
+          const next = pull()
+          if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })
+          const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))
+          hits = [
+            ...hits,
+            {
+              url: new URL(req.originalUrl, "http://localhost"),
+              body: body && typeof body === "object" ? (body as Record<string, unknown>) : {},
+            },
+          ]
+          yield* notify()
+          if (next.type === "sse") return send(responses(next, modelFrom(body)))
+          return fail(next)
+        }),
+      )
+
       yield* server.serve(router.asHttpEffect())
 
       return TestLLMServer.of({