import { ModelV2 } from "@opencode-ai/core/model" import { ProviderV2 } from "@opencode-ai/core/provider" import type { JSONSchema7 } from "@ai-sdk/provider" import { Effect } from "effect" import { jsonSchema, streamText, tool, type LanguageModel } from "ai" import { AppRuntime } from "@/effect/app-runtime" import { InstanceRef } from "@/effect/instance-ref" import { Provider } from "@/provider/provider" import { InstanceStore } from "@/project/instance-store" import { MFJS } from "@/provider/mfjs" // Usage: // bun run script/tool-schema-compatibility-matrix.ts --models=model-a,model-b // [--provider=opencode-go] [--projection=none|mfjs] // [--cases=tuple items,...] [--concurrency=9] [--timeout=30000] type JsonRecord = Record type Case = { name: string schema: JsonRecord } type Result = { case: string model: string status: "accepted" | "rejected" | "rate-limited" | "error" code?: number error?: string } const projection = option("projection") ?? "none" if (projection !== "none" && projection !== "mfjs") throw new Error(`Unsupported projection: ${projection}`) const concurrency = Number(option("concurrency") ?? 9) const timeout = Number(option("timeout") ?? 30_000) const providerID = option("provider") ?? "opencode-go" const models = option("models")?.split(",").filter(Boolean) ?? [] if (models.length === 0) throw new Error("--models must contain at least one model ID") const selectedCases = new Set(option("cases")?.split(",").filter(Boolean) ?? []) const matrix: Case[] = [ property("enum/type mismatch", { type: "object", enum: ["move", "copy"] }), property("untyped enum", { enum: ["move", "copy"] }), property("mixed untyped enum", { enum: ["move", 1, null, true] }), property("const", { const: "move" }), property("tuple items", { type: "array", items: [{ type: "string" }, { type: "number" }] }), property("prefix items", { type: "array", prefixItems: [{ type: "string" }, { type: "number" }] }), property("typed anyOf", { type: "string", enum: ["move"], anyOf: [{ type: "string" }, { type: "null" }], }), property("anyOf count limit", { anyOf: Array.from({ length: 501 }, (_, index) => ({ const: `value_${index}` })), }), property("oneOf", { oneOf: [{ type: "string" }, { type: "integer" }] }), property("allOf", { allOf: [ { type: "object", properties: { left: { type: "string" } } }, { type: "object", properties: { right: { type: "number" } } }, ], }), property("not", { type: "string", not: { enum: ["blocked"] } }), property("if/then/else", { type: "object", properties: { mode: { type: "string" }, count: { type: "integer" } }, if: { properties: { mode: { enum: ["many"] } } }, then: { required: ["count"] }, else: { properties: { count: { maximum: 1 } } }, }), property("contains", { type: "array", contains: { type: "string" } }), objectCase("patternProperties", { type: "object", patternProperties: { "^extra_": { type: "number" } }, additionalProperties: false, }), objectCase("dependentSchemas", { type: "object", properties: { key: { type: "string" }, value: { type: "string" } }, dependentSchemas: { key: { required: ["value"] } }, }), objectCase("propertyNames", { type: "object", propertyNames: { pattern: "^[a-z]+$" }, }), property("uniqueItems", { type: "array", items: { type: "string" }, uniqueItems: true }), property("boolean true schema", true), property("boolean false schema", false), objectCase("empty property name", { type: "object", properties: { "": { type: "string" } }, required: [""], }), objectCase("dangling required", { type: "object", properties: {}, required: ["missing"], }), objectCase("external ref", { type: "object", properties: { value: { $ref: "https://example.com/schema.json" } }, }), objectCase("chained ref", { type: "object", properties: { value: { $ref: "#/$defs/A" } }, $defs: { A: { $ref: "#/$defs/B" }, B: { type: "string" } }, }), objectCase("recursive ref", { type: "object", properties: { node: { $ref: "#/$defs/Node" } }, $defs: { Node: { type: "object", properties: { value: { type: "string" }, next: { anyOf: [{ $ref: "#/$defs/Node" }, { type: "null" }] } }, required: ["value"], }, }, }), objectCase("reference depth limit", { type: "object", properties: { value: { $ref: "#/$defs/Value" } }, $defs: { Value: nested(30) }, }), objectCase("schema size limit", { type: "object", description: "x".repeat(120_001), properties: {}, }), objectCase("schema depth limit", nested(35)), objectCase("property count limit", { type: "object", properties: Object.fromEntries( Array.from({ length: 3001 }, (_, index) => [`property_${index}`, { type: "string" }]), ), }), property("enum count limit", { type: "string", enum: Array.from({ length: 1001 }, (_, index) => `value_${index}`), }), ] const cases = selectedCases.size === 0 ? matrix : matrix.filter((item) => selectedCases.has(item.name)) const { store, ctx } = await AppRuntime.runPromise( InstanceStore.Service.use((store) => store.load({ directory: process.cwd() }).pipe(Effect.map((ctx) => ({ store, ctx }))), ), ) try { const languages = await AppRuntime.runPromise( Effect.gen(function* () { const provider = yield* Provider.Service return yield* Effect.forEach( models, Effect.fnUntraced(function* (model) { const info = yield* provider.getModel(ProviderV2.ID.make(providerID), ModelV2.ID.make(model)) return [model, yield* provider.getLanguage(info)] as const }), { concurrency: "unbounded" }, ) }).pipe(Effect.provideService(InstanceRef, ctx)), ) const jobs = languages.flatMap(([model, language]) => cases.map((item) => () => run(language, model, item))) const results = await parallel(jobs, concurrency) print(results) if (projection !== "none" && results.some((result) => result.status !== "accepted")) process.exitCode = 1 } finally { await AppRuntime.runPromise(store.dispose(ctx)) } process.exit(process.exitCode ?? 0) async function run(language: LanguageModel, model: string, item: Case): Promise { const schema = projection === "mfjs" ? MFJS.sanitize(item.schema) : item.schema let providerError: unknown try { const response = streamText({ model: language, prompt: "Reply with exactly OK without calling tools.", maxOutputTokens: 16, abortSignal: AbortSignal.timeout(timeout), onError(event) { providerError = event.error }, tools: { probe: tool({ description: `Tool schema probe: ${item.name}`, inputSchema: jsonSchema(schema as JSONSchema7), execute: async () => "ok", }), }, }) await response.text if (providerError) throw providerError console.error(`accepted: ${model} / ${item.name}`) return { case: item.name, model, status: "accepted" } } catch (error) { const failure = providerError ?? error const code = statusCode(failure) const message = failure instanceof Error ? failure.message : String(failure) const status = code === 429 || /rate limit/i.test(message) ? "rate-limited" : code === 400 ? "rejected" : "error" console.error(`${status}: ${model} / ${item.name}`) return { case: item.name, model, status, code, error: message, } } } async function parallel(jobs: Array<() => Promise>, limit: number) { const output = new Array(jobs.length) let next = 0 await Promise.all( Array.from({ length: Math.min(limit, jobs.length) }, async () => { while (true) { const index = next++ const job = jobs[index] if (!job) return output[index] = await job() } }), ) return output } function print(results: Result[]) { const byCase = new Map>() results.forEach((result) => { const row = byCase.get(result.case) ?? new Map() row.set(result.model, result) byCase.set(result.case, row) }) console.log(`Projection: ${projection}`) console.log(`Provider: ${providerID}`) console.log(`| Case | ${models.join(" | ")} |`) console.log(`| --- | ${models.map(() => "---").join(" | ")} |`) cases.forEach((item) => { const row = byCase.get(item.name) const values = models.map((model) => { const result = row?.get(model) if (!result) return "missing" return result.status === "accepted" ? "accepted" : `${result.status}${result.code ? ` (${result.code})` : ""}` }) console.log(`| ${item.name} | ${values.join(" | ")} |`) }) const rejected = results.filter((result) => result.status !== "accepted") if (rejected.length > 0) { console.log("\nRejected details:") rejected.forEach((result) => console.log(`- ${result.model} / ${result.case}: ${result.error}`)) } } function property(name: string, schema: unknown): Case { return objectCase(name, { type: "object", properties: { value: schema }, required: ["value"] }) } function objectCase(name: string, schema: JsonRecord): Case { return { name, schema } } function nested(depth: number): JsonRecord { return Array.from({ length: depth }).reduce( (schema) => ({ type: "object", properties: { next: schema }, required: ["next"] }), { type: "string" }, ) } function option(name: string) { const prefix = `--${name}=` return process.argv.find((arg) => arg.startsWith(prefix))?.slice(prefix.length) } function statusCode(error: unknown, seen = new Set()): number | undefined { if (!isRecord(error) || seen.has(error)) return seen.add(error) if (typeof error.statusCode === "number") return error.statusCode for (const value of Object.values(error)) { const nested = statusCode(value, seen) if (nested !== undefined) return nested } } function isRecord(value: unknown): value is JsonRecord { return typeof value === "object" && value !== null && !Array.isArray(value) }