refactor(cli): simplify isBinaryFile check to UTF-16 BOM only
Per review: CJK and legacy single-byte encodings already passed the old control-char heuristic (their bytes are all >= 0x80, never NUL), so the full Encoding.detect pass in isBinaryFile was solving a non-problem for them. The only realistic regression in the old heuristic is UTF-16 with BOM, where the second byte of every ASCII character is 0x00 and fires the NUL-byte early-return. Replace the detect call with a 2-byte BOM check via a new Encoding.hasUtf16Bom helper. Also reword the move/add inline markers in patch/index.ts to note that Encoding.write handles mkdir.
This commit is contained in:
@@ -38,6 +38,12 @@ export namespace Encoding {
|
||||
return bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
|
||||
}
|
||||
|
||||
/** True if `bytes[0..limit]` starts with a UTF-16 LE or BE byte-order mark. */
|
||||
export function hasUtf16Bom(bytes: Buffer, limit = bytes.length): boolean {
|
||||
if (limit < 2) return false
|
||||
return (bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff)
|
||||
}
|
||||
|
||||
/** Remap jschardet labels to iconv-lite compatible names. */
|
||||
function normalize(name: string): string {
|
||||
const lower = name.toLowerCase().replace(/[^a-z0-9]/g, "")
|
||||
|
||||
@@ -549,7 +549,7 @@ export namespace Patch {
|
||||
|
||||
if (hunk.move_path) {
|
||||
// Handle file move
|
||||
await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) // kilocode_change
|
||||
await Encoding.write(hunk.move_path, fileUpdate.content, fileUpdate.encoding) // kilocode_change - encoding-aware write (mkdirs)
|
||||
await fs.unlink(hunk.path)
|
||||
modified.push(hunk.move_path)
|
||||
log.info(`Moved file: ${hunk.path} -> ${hunk.move_path}`)
|
||||
|
||||
@@ -330,10 +330,8 @@ export async function isBinaryFile(filepath: string, fileSize: number): Promise<
|
||||
const result = await fh.read(bytes, 0, sampleSize, 0)
|
||||
if (result.bytesRead === 0) return false
|
||||
|
||||
// kilocode_change start - treat detected non-UTF-8 text (CJK, UTF-16 with BOM) as text, not binary
|
||||
const sample = bytes.subarray(0, result.bytesRead)
|
||||
const enc = Encoding.detect(sample)
|
||||
if (enc !== "utf-8") return false
|
||||
// kilocode_change start - UTF-16 BOM: NUL bytes are legitimate, skip the NUL/control-char heuristic
|
||||
if (Encoding.hasUtf16Bom(bytes, result.bytesRead)) return false
|
||||
// kilocode_change end
|
||||
|
||||
let nonPrintableCount = 0
|
||||
|
||||
Reference in New Issue
Block a user