diff --git a/bun.lock b/bun.lock index 02ed7a56e4..bfc1aa2e0a 100644 --- a/bun.lock +++ b/bun.lock @@ -393,7 +393,6 @@ "mime-types": "3.0.2", "tree-sitter-bash": "0.25.0", "tree-sitter-powershell": "0.25.10", - "turndown": "7.2.0", "venice-ai-sdk-provider": "2.1.1", "web-tree-sitter": "0.25.10", "which": "6.0.1", @@ -414,7 +413,6 @@ "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@types/node": "catalog:", - "@types/turndown": "5.0.5", "@types/which": "3.0.4", "drizzle-kit": "catalog:", }, @@ -1966,8 +1964,6 @@ "@mermaid-js/parser": ["@mermaid-js/parser@1.2.0", "", { "dependencies": { "@chevrotain/types": "~11.1.2" } }, "sha512-oYPyv8A4As1yH5Bx+04iQEQxXuIQDe0GKCNSRgao6z8AM9jixXIfP0vsppRLvGf+nKIOb9/LdpWA4YuJiVvESA=="], - "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="], - "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="], "@motionone/animation": ["@motionone/animation@10.18.0", "", { "dependencies": { "@motionone/easing": "^10.18.0", "@motionone/types": "^10.17.1", "@motionone/utils": "^10.18.0", "tslib": "^2.3.1" } }, "sha512-9z2p5GFGCm0gBsZbi8rVMOAJCtw1WqBTIPw3ozk06gDvZInBPIsQcHgYogEJ4yuHJ+akuW8g1SEIOpTOvYs8hw=="], @@ -3258,8 +3254,6 @@ "@types/tunnel": ["@types/tunnel@0.0.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA=="], - "@types/turndown": ["@types/turndown@5.0.5", "", {}, "sha512-TL2IgGgc7B5j78rIccBtlYAnkuv8nUQqhQc+DSYV5j9Be9XOcm/SKOVRuA47xAVI3680Tk9B1d8flK2GWT2+4w=="], - "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], "@types/whatwg-mimetype": ["@types/whatwg-mimetype@3.0.2", "", {}, "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA=="], @@ -5774,8 +5768,6 @@ "turbo": ["turbo@2.10.2", "", { "optionalDependencies": { "@turbo/darwin-64": "2.10.2", "@turbo/darwin-arm64": "2.10.2", "@turbo/linux-64": "2.10.2", "@turbo/linux-arm64": "2.10.2", "@turbo/windows-64": "2.10.2", "@turbo/windows-arm64": "2.10.2" }, "bin": { "turbo": "bin/turbo" } }, "sha512-wTExrNrRjB8qzIcg+ZLm0A3GFNLDsWNwdS/RBXB0FPrBDyzk3i96Yx+TxWZC7a0k1SIreFB8ciUbxjmEqTH8IQ=="], - "turndown": ["turndown@7.2.0", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A=="], - "tw-animate-css": ["tw-animate-css@1.4.0", "", {}, "sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ=="], "tw-to-css": ["tw-to-css@0.0.12", "", { "dependencies": { "postcss": "8.4.31", "postcss-css-variables": "0.18.0", "tailwindcss": "3.3.2" } }, "sha512-rQAsQvOtV1lBkyCw+iypMygNHrShYAItES5r8fMsrhhaj5qrV2LkZyXc8ccEH+u5bFjHjQ9iuxe90I7Kykf6pw=="], diff --git a/packages/core/package.json b/packages/core/package.json index 719f449bd8..0d3fc015e8 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -79,7 +79,6 @@ "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@types/node": "catalog:", - "@types/turndown": "5.0.5", "@types/which": "3.0.4", "@parcel/watcher-darwin-arm64": "2.5.1", "@parcel/watcher-darwin-x64": "2.5.1", @@ -137,7 +136,6 @@ "ignore": "7.0.5", "jsonc-parser": "3.3.1", "mime-types": "3.0.2", - "turndown": "7.2.0", "tree-sitter-bash": "0.25.0", "tree-sitter-powershell": "0.25.10", "venice-ai-sdk-provider": "2.1.1", diff --git a/packages/core/src/tool/html-markdown.ts b/packages/core/src/tool/html-markdown.ts new file mode 100644 index 0000000000..241c048ab4 --- /dev/null +++ b/packages/core/src/tool/html-markdown.ts @@ -0,0 +1,662 @@ +import { Parser } from "htmlparser2" + +const omitted = new Set(["script", "style", "noscript", "iframe", "object", "embed", "meta", "link", "template"]) +const blocks = new Set([ + "address", + "article", + "aside", + "details", + "dialog", + "div", + "dl", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "header", + "main", + "nav", + "p", + "section", + "summary", +]) + +type Frame = { + suppressed: boolean + link?: { href: string; title?: string } + suspendedLink?: Frame["link"] + marker?: { index: number; block: number; leadingSpace?: boolean; previous?: Frame["marker"] } + code?: { inline: boolean; text: string; language?: string } + linkCode?: NonNullable + resumedCode?: NonNullable + list?: { ordered: boolean; next: number; previous?: Frame["list"] } + item?: { indent: string; previous?: Frame["item"] } + table?: { + start: number + rows: string[][] + row?: string[] + caption?: string + fallback: boolean + previous?: Frame["table"] + } + cell?: { start: number } + caption?: { start: number } + details?: { open: boolean; summary: boolean; previous?: Frame["details"] } +} + +type Chunk = string | { raw: string } + +export const MAX_MARKDOWN_BYTES = 5 * 1024 * 1024 +const CONTENT_BYTES = MAX_MARKDOWN_BYTES - 64 * 1024 + +export function convertHTMLToMarkdown(html: string) { + const output: Chunk[] = [] + const stack: Frame[] = [] + const encoder = new TextEncoder() + let pendingSpace = false + let pendingIndent = "" + let last = "" + let quoteDepth = 0 + let needsQuotePrefix = false + let blockCount = 0 + let depth = 0 + let stopped = false + let outputBytes = 0 + let activeCode: NonNullable | undefined + let activeLink: Frame["link"] | undefined + let linkOpen = false + let activeMarker: Frame["marker"] | undefined + let activeList: Frame["list"] | undefined + let activeItem: Frame["item"] | undefined + let activeTable: NonNullable | undefined + let activeCell: Frame["cell"] | undefined + let tableDepth = 0 + let fallbackSuppressedDepth = 0 + let fallbackOmittedDepth = 0 + let activeDetails: Frame["details"] | undefined + + const sliceBytes = (value: string, bytes: number) => { + if (encoder.encode(value).byteLength <= bytes) return value + const characters = Array.from(value) + let low = 0 + let high = characters.length + while (low < high) { + const middle = Math.ceil((low + high) / 2) + if (encoder.encode(characters.slice(0, middle).join("")).byteLength <= bytes) low = middle + else high = middle - 1 + } + return characters.slice(0, low).join("") + } + const append = (value: string, content = false) => { + const limit = content ? CONTENT_BYTES : MAX_MARKDOWN_BYTES + if (!value || outputBytes >= limit) return + const bytes = encoder.encode(value) + const remaining = limit - outputBytes + const next = bytes.byteLength <= remaining ? value : sliceBytes(value, remaining) + output.push(next) + outputBytes += encoder.encode(next).byteLength + last = next.at(-1) ?? last + } + const appendRaw = (value: string) => { + const before = output.length + append(value) + if (output.length > before) output[output.length - 1] = { raw: output[output.length - 1] as string } + } + const take = (start: number) => { + const chunks = output.splice(start) + const value = chunks.map((chunk) => (typeof chunk === "string" ? chunk : chunk.raw)).join("") + outputBytes -= encoder.encode(value).byteLength + return value + } + const prefixQuote = () => { + if (!needsQuotePrefix || quoteDepth === 0 || activeCell) return + append(`${"> ".repeat(Math.min(8, quoteDepth))}`) + needsQuotePrefix = false + } + const flushSpace = () => { + if (!pendingSpace) return + const marker = activeMarker + if (marker && output.length === marker.index + 1 && last !== " " && last !== "\n") { + const value = output[marker.index] + if (typeof value === "string") output[marker.index] = ` ${value}` + outputBytes++ + marker.leadingSpace = true + pendingSpace = false + return + } + if (last && last !== "\n" && last !== " ") append(" ") + pendingSpace = false + } + const inline = (value: string, open = false) => { + if (open) flushSpace() + prefixQuote() + if (pendingIndent) { + append(pendingIndent) + pendingIndent = "" + } + if (activeLink && !linkOpen) { + append("[") + linkOpen = true + } + append(value) + } + const block = () => { + if (activeLink && linkOpen) { + append(`](${destination(activeLink.href)}${title(activeLink.title)})`) + linkOpen = false + } + pendingSpace = false + append("\n\n") + blockCount++ + needsQuotePrefix = quoteDepth > 0 + pendingIndent = activeItem?.indent ?? "" + } + const suspendLink = (frame: Frame) => { + if (!activeLink) return + frame.suspendedLink = activeLink + block() + activeLink = undefined + linkOpen = false + } + const text = (value: string) => { + if (activeCode) { + activeCode.text += value + return + } + for (const part of value.split(/([\t\n\f\r ]+)/)) { + if (!part) continue + if (/^[\t\n\f\r ]+$/.test(part)) { + pendingSpace = true + continue + } + flushSpace() + prefixQuote() + if (pendingIndent) { + append(pendingIndent) + pendingIndent = "" + } + if (activeLink && !linkOpen) { + append("[") + linkOpen = true + } + const escaped = part + .replace(/([\\`*_[\]<>|])/g, "\\$1") + .replace(/~/g, "\\~") + .replace(/^([#+-])/, "\\$1") + .replace(/^(\d+)\./, "$1\\.") + append(escaped, true) + } + } + const destination = (value: string) => value.replace(/([\\()])/g, "\\$1").replace(/[\t\n\r ]+/g, "%20") + const title = (value: string | undefined) => + value + ? ` "${value + .replace(/[\t\n\r ]+/g, " ") + .trim() + .replace(/([\\"])/g, "\\$1")}"` + : "" + const finishCode = (code: NonNullable) => { + if (code.inline && !code.text) return + let backticks = 0 + let tildes = 0 + let currentBackticks = 0 + let currentTildes = 0 + for (const character of code.text) { + currentBackticks = character === "`" ? currentBackticks + 1 : 0 + currentTildes = character === "~" ? currentTildes + 1 : 0 + backticks = Math.max(backticks, currentBackticks) + tildes = Math.max(tildes, currentTildes) + } + if (code.inline) { + const fence = "`".repeat(Math.max(1, backticks + 1)) + const padding = /^ | $/.test(code.text) && !/^ +$/.test(code.text) ? " " : "" + flushSpace() + prefixQuote() + const wrapper = encoder.encode(`${fence}${padding}${padding}${fence}`).byteLength + appendRaw( + `${fence}${padding}${sliceBytes(code.text, Math.max(0, CONTENT_BYTES - outputBytes - wrapper))}${padding}${fence}`, + ) + return + } + if (activeCell) { + text(code.text) + return + } + const marker = backticks <= tildes ? "`" : "~" + const length = Math.max(3, (marker === "`" ? backticks : tildes) + 1) + const fence = marker.repeat(length) + block() + const prefix = `${fence}${code.language ?? ""}\n` + const quote = quoteDepth > 0 ? `${"> ".repeat(Math.min(8, quoteDepth))}` : "" + const closing = `${code.text.endsWith("\n") ? "" : "\n"}${fence}` + let payload = code.text + for (;;) { + const candidate = `${prefix}${payload}${payload.endsWith("\n") ? "" : "\n"}${fence}` + const value = quote ? candidate.replace(/^/gm, quote) : candidate + const valueBytes = encoder.encode(value).byteLength + if (outputBytes + valueBytes <= CONTENT_BYTES) { + appendRaw(value) + block() + return + } + const excess = valueBytes - Math.max(0, CONTENT_BYTES - outputBytes) + payload = sliceBytes(payload, Math.max(0, encoder.encode(payload).byteLength - Math.ceil(excess))) + } + } + + const parser = new Parser({ + onopentag(name, attributes) { + depth++ + if (depth > 10_000) { + if (stack.at(-1)?.suppressed) { + const visibleParent = stack.findLastIndex((frame) => !frame.suppressed) + fallbackSuppressedDepth = visibleParent + 2 + } + activeCode = undefined + stopped = true + } + if (stopped) { + if (omitted.has(name)) fallbackOmittedDepth++ + else pendingSpace = true + return + } + const suppressed = (stack.at(-1)?.suppressed ?? false) || omitted.has(name) + const frame: Frame = { suppressed } + const hidden = "hidden" in attributes || attributes["aria-hidden"]?.toLowerCase() === "true" || name === "head" + const details = activeDetails + if (hidden || (details && !details.open && !details.summary && name !== "summary")) frame.suppressed = true + stack.push(frame) + if (frame.suppressed) return + + if (activeCode && !activeCode.inline) { + if (name === "br") activeCode.text += "\n" + if (name === "code" && attributes.class) + activeCode.language = attributes.class.match(/(?:language-|lang-)([^\s]+)/)?.[1] + return + } + if (name === "details") { + frame.details = { open: "open" in attributes, summary: false, previous: activeDetails } + activeDetails = frame.details + block() + return + } + if (name === "summary") { + if (details) details.summary = true + block() + return + } + if (name === "pre") { + suspendLink(frame) + frame.code = { inline: false, text: "" } + activeCode = frame.code + return + } + if (name === "code") { + if (activeCode?.inline) return + frame.code = { inline: true, text: "" } + activeCode = frame.code + return + } + if (/^h[1-6]$/.test(name)) { + block() + inline(`${"#".repeat(Number(name[1]))} `) + return + } + if (blocks.has(name)) { + suspendLink(frame) + if (name === "p" && last === " ") return + block() + return + } + if (name === "br") { + pendingSpace = false + inline(" \n") + needsQuotePrefix = quoteDepth > 0 + return + } + if (name === "hr") { + block() + inline("---") + block() + return + } + if (name === "strong" || name === "b") { + inline("**", true) + frame.marker = { index: output.length - 1, block: blockCount, previous: activeMarker } + activeMarker = frame.marker + return + } + if (name === "em" || name === "i") { + inline("*", true) + frame.marker = { index: output.length - 1, block: blockCount, previous: activeMarker } + activeMarker = frame.marker + return + } + if (name === "s" || name === "strike" || name === "del") { + inline("~~", true) + frame.marker = { index: output.length - 1, block: blockCount, previous: activeMarker } + activeMarker = frame.marker + return + } + if (name === "a") { + if (activeLink && activeCode?.inline) { + const parent = stack.findLast( + (candidate) => candidate.link === activeLink && candidate.linkCode === activeCode, + ) + if (parent) { + finishCode(activeCode) + if (linkOpen) append(`](${destination(activeLink.href)}${title(activeLink.title)})`) + activeCode = parent.resumedCode + parent.link = undefined + parent.linkCode = undefined + activeLink = undefined + linkOpen = false + } + } + if (activeCode?.inline) { + frame.resumedCode = activeCode + if (activeCode.text) finishCode(activeCode) + activeCode.text = "" + activeCode = undefined + frame.link = { href: attributes.href ?? "", title: attributes.title } + activeLink = frame.link + linkOpen = true + inline("[", true) + frame.linkCode = { inline: true, text: "" } + activeCode = frame.linkCode + return + } + if (activeLink) { + if (linkOpen) append(`](${destination(activeLink.href)}${title(activeLink.title)})`) + const parent = stack.findLast((candidate) => candidate.link === activeLink) + if (parent) parent.link = undefined + activeLink = undefined + linkOpen = false + } + frame.link = { href: attributes.href ?? "", title: attributes.title } + activeLink = frame.link + linkOpen = true + return inline("[", true) + } + if (name === "img") { + const alt = (attributes.alt ?? "").replace(/([\\\]])/g, "\\$1") + const close = `](${destination(attributes.src ?? "")}${title(attributes.title)})` + const open = "![" + const available = CONTENT_BYTES - outputBytes - encoder.encode(open + close).byteLength + inline(`${open}${sliceBytes(alt, Math.max(0, available))}${close}`, true) + return + } + if (name === "blockquote") { + suspendLink(frame) + block() + quoteDepth++ + needsQuotePrefix = true + return + } + if (name === "ul" || name === "ol") { + suspendLink(frame) + const start = Number.parseInt(attributes.start ?? "1") + frame.list = { ordered: name === "ol", next: Number.isNaN(start) ? 1 : start, previous: activeList } + activeList = frame.list + block() + return + } + if (name === "li") { + block() + const value = Number.parseInt(attributes.value ?? "") + if (activeList?.ordered && !Number.isNaN(value)) activeList.next = value + const marker = activeList?.ordered ? `${activeList.next++}.` : "-" + const prefix = `${(activeItem?.indent ?? "").slice(0, 24)}${marker} ` + frame.item = { indent: " ".repeat(prefix.length), previous: activeItem } + activeItem = frame.item + pendingIndent = "" + inline(prefix) + return + } + if (name === "table") { + suspendLink(frame) + tableDepth++ + if (tableDepth === 1) { + block() + frame.table = { start: output.length, rows: [], fallback: false, previous: activeTable } + activeTable = frame.table + } else pendingSpace = true + return + } + if (name === "tr") { + if (tableDepth !== 1) { + pendingSpace = true + return + } + if (activeTable) activeTable.row = [] + return + } + if (name === "th" || name === "td") { + if (tableDepth !== 1) { + pendingSpace = true + return + } + if (attributes.colspan || attributes.rowspan) activeTable!.fallback = true + frame.cell = { start: output.length } + activeCell = frame.cell + return + } + if (name === "caption") { + frame.caption = { start: output.length } + return + } + if (name === "dt") { + block() + inline("**") + return + } + if (name === "dd") { + inline("\n: ") + return + } + }, + ontext(value) { + if (stopped) { + if (fallbackSuppressedDepth === 0 && fallbackOmittedDepth === 0) text(value) + return + } + if (stack.at(-1)?.suppressed) return + text(value) + }, + onclosetag(name) { + depth-- + if (stopped) { + if (fallbackOmittedDepth > 0 && omitted.has(name)) fallbackOmittedDepth-- + if (fallbackSuppressedDepth > 0 && depth < fallbackSuppressedDepth) fallbackSuppressedDepth = 0 + return + } + const frame = stack.pop() + if (!frame || frame.suppressed) return + if (frame.linkCode) { + if (activeLink !== frame.link || !linkOpen) { + frame.resumedCode!.text += frame.linkCode.text + activeCode = frame.resumedCode + if (activeLink === frame.link) activeLink = undefined + linkOpen = false + return + } + activeCode = undefined + finishCode(frame.linkCode) + if (frame.link && linkOpen) append(`](${destination(frame.link.href)}${title(frame.link.title)})`) + activeLink = undefined + linkOpen = false + activeCode = frame.resumedCode + return + } + if (activeCode && !activeCode.inline && !frame.code) return + if (frame.code) { + activeCode = undefined + finishCode(frame.code) + if (frame.suspendedLink) activeLink = frame.suspendedLink + return + } + if (name === "summary") { + if (activeDetails) activeDetails.summary = false + return block() + } + if (name === "details") { + activeDetails = frame.details?.previous + return block() + } + if (name === "dt") { + inline("**") + return + } + if (name === "dd") return block() + if ( + name === "strong" || + name === "b" || + name === "em" || + name === "i" || + name === "s" || + name === "strike" || + name === "del" + ) { + const value = name === "strong" || name === "b" ? "**" : name === "em" || name === "i" ? "*" : "~~" + const trailingSpace = pendingSpace + pendingSpace = false + if (frame.marker) activeMarker = frame.marker.previous + if (frame.marker && (frame.marker.block !== blockCount || output.length === frame.marker.index + 1)) { + output[frame.marker.index] = "" + pendingSpace = trailingSpace || frame.marker.leadingSpace === true + return + } + inline(value) + pendingSpace = trailingSpace || frame.marker?.leadingSpace === true + return + } + if (name === "a") { + if (frame.link && (activeLink === frame.link || !activeLink)) { + activeLink = frame.link + if (linkOpen) append(`](${destination(frame.link?.href ?? "")}${title(frame.link?.title)})`) + else if (last && last !== "\n") append(`](${destination(frame.link?.href ?? "")}${title(frame.link?.title)})`) + linkOpen = false + activeLink = undefined + } + return + } + if (/^h[1-6]$/.test(name) || blocks.has(name)) { + block() + if (frame.suspendedLink) activeLink = frame.suspendedLink + return + } + if (name === "blockquote") { + quoteDepth-- + block() + if (frame.suspendedLink) activeLink = frame.suspendedLink + return + } + if (name === "li") { + activeItem = frame.item?.previous + return block() + } + if (name === "ul" || name === "ol") { + activeList = frame.list?.previous + block() + if (frame.suspendedLink) activeLink = frame.suspendedLink + return + } + if ((name === "th" || name === "td") && tableDepth === 1) { + activeCell = undefined + if (frame.cell) { + const value = take(frame.cell.start) + .replace(/[\t\r\n ]+/g, " ") + .trim() + .replace(/(? 0 && table.rows.every((row) => row.length === width) + if (loose) { + append(loose) + block() + } + if (table.caption) { + append(table.caption) + block() + } + if (!table.fallback && rectangular) { + const prefix = `${quoteDepth > 0 ? `${"> ".repeat(Math.min(8, quoteDepth))}` : ""}${pendingIndent}` + pendingIndent = "" + append(`${prefix}| ${table.rows[0].join(" | ")} |\n${prefix}|${" --- |".repeat(width)}`) + for (const row of table.rows.slice(1)) append(`\n${prefix}| ${row.join(" | ")} |`) + } else { + for (const [index, row] of table.rows.entries()) { + if (index > 0) block() + append(row.join(" | ")) + } + } + } + block() + if (frame.suspendedLink) activeLink = frame.suspendedLink + return + } + pendingSpace = true + } + }, + }) + for (let index = 0; index < html.length; index += 64 * 1024) parser.write(html.slice(index, index + 64 * 1024)) + parser.end() + + const normalized: string[] = [] + let pendingText = "" + const flushText = () => { + if (!pendingText) return + const lines = pendingText + .replace(/[ \t]+\n/g, (space) => (space.startsWith(" ") ? " \n" : "\n")) + .replace(/\n{3,}/g, "\n\n") + .split("\n") + normalized.push( + lines + .map((line, index) => { + if (line) return line + const before = lines[index - 1]?.match(/^(?:> )+/)?.[0] + const after = lines[index + 1]?.match(/^(?:> )+/)?.[0] + if (!before || !after || before.length === after.length) return line + return "> ".repeat(Math.min(before.length, after.length) / 2).trimEnd() + }) + .join("\n"), + ) + pendingText = "" + } + for (const chunk of output) { + if (typeof chunk !== "string") { + flushText() + normalized.push(chunk.raw) + continue + } + pendingText += chunk + } + flushText() + return sliceBytes(normalized.join("").trim(), MAX_MARKDOWN_BYTES) +} diff --git a/packages/core/src/tool/plugin/webfetch.ts b/packages/core/src/tool/plugin/webfetch.ts index 9c1b2a892d..cbbe6a224c 100644 --- a/packages/core/src/tool/plugin/webfetch.ts +++ b/packages/core/src/tool/plugin/webfetch.ts @@ -5,12 +5,12 @@ import { ToolFailure } from "@opencode-ai/ai" import { Duration, Effect, Schema } from "effect" import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" import { Parser } from "htmlparser2" -import TurndownService from "turndown" import { Permission } from "../../permission.js" +import { convertHTMLToMarkdown, MAX_MARKDOWN_BYTES } from "../html-markdown.js" import { collectBoundedResponseBody } from "../http-body.js" export const name = "webfetch" -export const MAX_RESPONSE_BYTES = 5 * 1024 * 1024 +export const MAX_RESPONSE_BYTES = MAX_MARKDOWN_BYTES export const DEFAULT_TIMEOUT_SECONDS = 30 export const MAX_TIMEOUT_SECONDS = 120 @@ -196,14 +196,4 @@ export function extractTextFromHTML(html: string) { return text.trim() } -export function convertHTMLToMarkdown(html: string) { - const turndown = new TurndownService({ - headingStyle: "atx", - hr: "---", - bulletListMarker: "-", - codeBlockStyle: "fenced", - emDelimiter: "*", - }) - turndown.remove(["script", "style", "meta", "link"]) - return turndown.turndown(html) -} +export { convertHTMLToMarkdown } diff --git a/packages/core/test/tool-webfetch.test.ts b/packages/core/test/tool-webfetch.test.ts index bac467da1f..a99a427bfd 100644 --- a/packages/core/test/tool-webfetch.test.ts +++ b/packages/core/test/tool-webfetch.test.ts @@ -68,9 +68,316 @@ describe("WebFetchTool helpers", () => { }) test("ports HTML text and markdown conversions without active content", () => { - const html = "

Hello

world wide

" - expect(WebFetchTool.extractTextFromHTML(html)).toBe("Helloworld wide") - expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide**") + const html = + "

Hello

world wide today

" + expect(WebFetchTool.extractTextFromHTML(html)).toBe("Helloworld wide today") + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide** today") + }) + + test("renders headings, inline semantics, links, images, breaks, and thematic breaks", () => { + const html = `

Read this

docs
a ] b

old

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `## Read *this*\n\n[docs](https://example.com/a%20\\(b\\) "Example") \n![a \\] b](diagram.png)\n\n---\n\n~~old~~`, + ) + }) + + test("preserves inline and preformatted code verbatim with safe fences", () => { + const html = `

Use say(\`hello\`) now.

const fence = \`\`\`\n& stays decoded
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `Use \`\`say(\`hello\`)\`\` now.\n\n~~~ts\nconst fence = \`\`\`\n& stays decoded\n~~~`, + ) + }) + + test("keeps nested ordered and unordered lists structurally readable", () => { + const html = `
  1. alpha
    • nested item

  2. beta first

    beta second

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `3. alpha\n\n - nested **item**\n\n4. beta first\n\n beta second`, + ) + }) + + test("renders blockquotes and tables as readable Markdown", () => { + const html = `

quoted text

  • point
NameValue
one1
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `> quoted *text*\n\n> - point\n\n| Name | Value |\n| --- | --- |\n| one | \`1\` |`, + ) + }) + + test("decodes entities and normalizes prose whitespace without joining words", () => { + const html = `

alpha\n & beta café gamma 😀

delta

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`alpha & beta café gamma 😀\n\ndelta`) + }) + + test("omits active and fallback content while retaining surrounding prose", () => { + const html = `

before bad after

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("before after") + }) + + test("is deterministic and bounded for malformed maximum-size input", () => { + const html = `

${"visible & text ".repeat(250_000)}

` + const first = WebFetchTool.convertHTMLToMarkdown(html) + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(first) + expect(first.startsWith("visible & text visible & text")).toBe(true) + expect(first.length).toBeLessThanOrEqual(html.length) + }) + + test("bounds deeply nested list output and fragmented code fences", () => { + const lists = `${"
  • item".repeat(2_000)}${"
".repeat(2_000)}` + const quotes = `${"

item".repeat(2_000)}${"

".repeat(2_000)}` + const code = `
${"` x ".repeat(250_000)}
` + expect(WebFetchTool.convertHTMLToMarkdown(lists).length).toBeLessThan(lists.length * 4) + expect(WebFetchTool.convertHTMLToMarkdown(quotes).length).toBeLessThan(quotes.length * 4) + expect(() => WebFetchTool.convertHTMLToMarkdown(code)).not.toThrow() + expect( + WebFetchTool.convertHTMLToMarkdown( + "
".repeat(20_000) + "safe

tail &

", + ), + ).toBe("safe tail &") + }) + + test("escapes prose that would otherwise become Markdown structure", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`

# heading

1. item

---

a | b

`)).toBe( + `\\# heading\n\n1\\. item\n\n\\---\n\na \\| b`, + ) + }) + + test("preserves code whitespace and quotes every line of multiline blocks", () => { + const html = `
line  \n\n\nnext
a|bc
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `> \`\`\`\n> line \n> \n> \n> next\n> \`\`\`\n\n> | a\\|b | c |\n> | --- | --- |`, + ) + }) + + test("keeps nested blockquotes inside their outer quote", () => { + const html = `

outer

inner

end

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`> outer\n>\n> > inner\n>\n> end`) + }) + + test("keeps visible whitespace around inline emphasis", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`

a b c a b c

`)).toBe(`a **b** c a *b* c`) + expect(WebFetchTool.convertHTMLToMarkdown(`a b a b`)).toBe(`a b a b`) + }) + + test("captures formatting elements inside preformatted content as code only", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`
xyz
`)).toBe(`\`\`\`\nxyz\n\`\`\``) + }) + + test("normalizes multiline table cells without changing their columns", () => { + const html = `
x
y		a|b

first

second

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| x y | \`a\\|b\` | first second |\n| --- | --- | --- |`) + }) + + test("flattens nested tables without corrupting the outer table", () => { + const html = `
ParentSibling
Before
KeyValue
A1
After		Tail
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `| Parent | Sibling |\n| --- | --- |\n| Before Key Value A 1 After | Tail |`, + ) + }) + + test("preserves loose text around malformed table rows", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`beforeafter
cell
`)).toBe( + `before after\n\n| cell |\n| --- |`, + ) + expect(WebFetchTool.convertHTMLToMarkdown(`alpha
`)).toBe(`alpha`) + }) + + test("escapes tilde fences and removes empty emphasis markers", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`

~~~

content

~~~

`)).toBe( + `\\~\\~\\~\n\ncontent\n\n\\~\\~\\~`, + ) + }) + + test("parses malformed tag prefixes in linear time without a regex prepass", () => { + const small = " { + const prose = `

${"*".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}

` + const code = `
${"`".repeat(WebFetchTool.MAX_RESPONSE_BYTES - 11)}
` + const proseOutput = WebFetchTool.convertHTMLToMarkdown(prose) + const codeOutput = WebFetchTool.convertHTMLToMarkdown(code) + expect(Buffer.byteLength(proseOutput)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) + expect(Buffer.byteLength(codeOutput)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) + expect(codeOutput.startsWith("~~~\n")).toBe(true) + }) + + test("does not confuse source NUL text with buffered code", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`

before \u00000\u0000 after

code
`)).toBe( + `before \u00000\u0000 after\n\n\`\`\`\ncode\n\`\`\``, + ) + }) + + test("preserves multiline inline code verbatim", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`

first\n\n\nsecond

`)).toBe( + "` first\n\n\nsecond `", + ) + }) + + test("prefixes inline code at the start of a blockquote line", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`
x y
`)).toBe(`> \`x\` y`) + }) + + test("keeps links nested in inline code associated with their text", () => { + const html = `
socket = new WebSocket(url)
Creates one.
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `**\` socket = new \`[\`WebSocket\`](#constructor)\`(url)\`**\n: Creates one.`, + ) + expect(WebFetchTool.convertHTMLToMarkdown(`x after`)).toBe(`[\`x\`](#x) after`) + expect( + WebFetchTool.convertHTMLToMarkdown( + `
socket = new WebSocket(url)
Creates one.
`, + ), + ).toBe(`**\` socket = new \`[\`WebSocket\`](#constructor)\`(url)\`**\n: Creates one.`) + expect(WebFetchTool.convertHTMLToMarkdown(`abcde`)).toBe( + `\`a\`[\`b\`](\/x)[\`c\`](\/y)\`de\``, + ) + expect(WebFetchTool.convertHTMLToMarkdown(`abc`)).toBe(`\`a\`[\`b\`](\/x)c`) + expect(WebFetchTool.convertHTMLToMarkdown(`a
b
cd`)).toBe( + `\`a\`[](\/x)\n\n\`bcd\``, + ) + }) + + test("indents nested list continuations and preserves ordered numbering", () => { + const html = `

  1. first

    continued

    • nested

      continued nested

  2. next
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `4. first\n\n continued\n\n - nested\n\n continued nested\n\n5. next`, + ) + }) + + test("renders block content outside link syntax", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`before
block
after`)).toBe( + `[before](/docs)\n\nblock\n\n[after](/docs)`, + ) + }) + + test("recovers nested anchors without unmatched Markdown syntax", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`xyz`)).toBe(`[x](/a)[y](/b)z`) + }) + + test("keeps emphasis whitespace through neutral wrappers", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`

a boldc

`)).toBe(`a **bold** c`) + }) + + test("flattens preformatted content inside table cells", () => { + const html = `
a|b\nnext
x|y
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| a\\|b next | \`x\\|y\` |\n| --- | --- |`) + }) + + test("keeps each near-boundary inline construct closed and UTF-8-safe", () => { + const payload = "😀".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 4) + const cases = [ + [`${payload}`, /^\*\*[\s\S]*\*\*$/], + [`${payload}`, /^\[[\s\S]*\]\(\/docs\)$/], + [`${payload}`, /^!\[[\s\S]*\]\(image\.png\)$/], + [`${payload}`, /^`[\s\S]*`$/], + ] as const + for (const [html, pattern] of cases) { + const output = WebFetchTool.convertHTMLToMarkdown(html) + expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) + expect(output).not.toContain("�") + expect(output).toMatch(pattern) + } + }) + + test("keeps near-boundary block constructs syntactically complete", () => { + const payload = "x".repeat(WebFetchTool.MAX_RESPONSE_BYTES) + const table = WebFetchTool.convertHTMLToMarkdown( + `
Name
${payload}
`, + ) + const list = WebFetchTool.convertHTMLToMarkdown(`
  • ${payload}
  • nested
`) + const code = WebFetchTool.convertHTMLToMarkdown(`
${payload}
`) + for (const output of [table, list, code]) { + expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) + expect(output).not.toContain("�") + } + expect(table).toMatch(/^\| Name \|\n\| --- \|\n\| [\s\S]* \|$/) + expect(list).toMatch(/^- [\s\S]*$/) + expect(list.includes("nested")).toBe(false) + expect(code.match(/^(`{3,}|~{3,})$/gm)).toHaveLength(2) + }) + + test("keeps quoted code within budget with a safe closed fence", () => { + const html = `
${"`".repeat(32)}${"~".repeat(32)}${"x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}
` + const output = WebFetchTool.convertHTMLToMarkdown(html) + expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) + const lines = output.split("\n") + expect(lines[0]).toMatch(/^> (`{33}|~{33})$/) + expect(lines.at(-1)).toBe(lines[0]) + }) + + test("separates reconstructed tables from adjacent inline and quoted content", () => { + const html = `intro
x
outro
quote
cell
  • item
    cell
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `intro\n\n| x |\n| --- |\n\noutro\n\n> quote\n\n> | cell |\n> | --- |\n\n- item\n\n| cell |\n| --- |`, + ) + }) + + test("keeps multiline quoted code closed at the content budget", () => { + const html = `
${"x\n".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 2)}

tail

` + const output = WebFetchTool.convertHTMLToMarkdown(html) + expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) + expect((output.match(/(`{3}|~{3})/g) ?? []).length).toBe(2) + expect(output.includes("\uFFFD")).toBe(false) + expect(output.endsWith("tail")).toBe(true) + }) + + test("keeps active content suppressed when depth fallback begins", () => { + const html = `${"
".repeat(10_001)}LEAK${"
".repeat(10_001)}

visible

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("visible") + }) + + test("keeps visible text after depth fallback begins inside preformatted content", () => { + const html = `
${"".repeat(10_001)}visible${"".repeat(10_001)}

after

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("visible after") + }) + + test("resumes links around every block structure", () => { + const html = `before

quote

  • item
code
cell
after` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `[before](/x)\n\n> quote\n\n- item\n\n\`\`\`\ncode\n\`\`\`\n\n| cell |\n| --- |\n\n[after](/x)`, + ) + }) + + test("indents child lists from the actual parent marker width", () => { + expect(WebFetchTool.convertHTMLToMarkdown(`
  1. outer
    • inner
`)).toBe( + `100. outer\n\n - inner`, + ) + }) + + test("renders captions and definition lists with readable boundaries", () => { + const html = `
Cache modes
NameMeaning
ALocal
Cache
A local store
Origin
The remote source
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `Cache modes\n\n| Name | Meaning |\n| --- | --- |\n| A | Local |\n\n**Cache**\n: A local store\n\n**Origin**\n: The remote source`, + ) + }) + + test("falls back to row-oriented text for table spans", () => { + const html = `
Group
AShared
B
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`Group\n\nA | Shared\n\nB`) + }) + + test("suppresses head and hidden subtrees while retaining visible body content", () => { + const html = `noise

visible

shown
` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`visible\n\nshown`) + }) + + test("preserves pre breaks and normalizes multiline link titles", () => { + const html = `
first
second

link

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe( + `\`\`\`\nfirst\nsecond\n\`\`\`\n\n[link](/x "line one line two")`, + ) + }) + + test("renders closed and open details according to visibility", () => { + const html = `
Closed

secret

Open

visible

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`Closed\n\nOpen\n\nvisible`) }) }) @@ -178,7 +485,7 @@ describe("WebFetchTool registration", () => { }), ) - it.effect("returns an error result when HTML-to-Markdown conversion throws", () => + it.effect("converts deeply nested HTML without overflowing", () => Effect.gen(function* () { reset() respond = () => @@ -191,8 +498,8 @@ describe("WebFetchTool registration", () => { const url = "https://1.1.1.1/deep-html" expect(yield* executeTool(registry, call({ url, format: "markdown" }))).toMatchObject({ - status: "error", - error: { type: "unknown" }, + status: "completed", + content: [{ type: "text", text: "content" }], }) }), )