From 7b71e2150fd10a4900992ad383b37cd4533f3205 Mon Sep 17 00:00:00 2001 From: productdevbook Date: Thu, 13 Aug 2026 19:13:28 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20the=20write=20side=20of=20#475=20?= =?UTF-8?q?=E2=80=94=20text-format=20options,=20and=20a=20CLI=20--bom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #475 answered the read side: `parseCsv` takes bytes, honours the byte-order mark, and `--encoding` names what no mark can. The write side was left with a documented remedy nobody could reach. ## Two ways to not get a BOM `hucre convert veriler.xlsx out.csv` emits UTF-8 with no mark, and had no flag to ask for one. Excel on a Turkish, Polish or Greek Windows reads a UTF-8 CSV as the system code page unless the file opens with `EF BB BF`, so the CLI's own output came back as mojibake — the #475 complaint, mirrored on the way out. `write({ sheets, format: "csv" })` had the same problem for a different reason: **it called every text writer with no options at all.** `writeCsv`, `writeTsv`, `writeJson`, `writeNdjson`, `writeXml`, `toHtml` and `toMarkdown` each take an options bag, and `write` passed none to any of them. So the entry #469 added precisely so one call could reach all nine formats was the only way to reach seven of them that could not configure a single thing about them — no delimiter, no `pretty`, no `rootTag`, no `caption`, and no `bom`. ## What changed `write` takes one bag per text format, passed to the writer it dispatches to: await write({ sheets, format: "csv", csv: { delimiter: ";", bom: true } }) await write({ sheets, format: "json", json: { pretty: true } }) and the CLI takes `--bom`: hucre convert veriler.xlsx out.csv --bom `renderWorkbook`'s separate CSV branch is gone with it: it existed to pass a delimiter that `write` could not take, and the comment above it already said everything else goes through `write`, "which is the function that is supposed to know how to do this". Now CSV does too. ## Not changed, and worth saying Output is UTF-8 and cannot be anything else — `TextEncoder` encodes only UTF-8 by specification, and `src/` is Web-APIs-only. A caller who needs windows-1254 bytes has to encode them, and the BOM is what makes UTF-8 work everywhere instead. ## Checked 10 new tests. The two CLI ones assert `EF BB BF` is present with the flag and absent without it, on both `.csv` and `.tsv`, and that the rows survive the prefix. The eight library ones each pass an option through `write` and assert it reached the writer. `pnpm test` green — 10,605 tests, 235 files. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 20 +++++- src/_inline-cells.ts | 6 +- src/cli/commands.ts | 36 ++++++---- src/defter.ts | 45 +++++++++--- test/cli.test.ts | 27 ++++++++ test/unified-text-formats.test.ts | 111 ++++++++++++++++++++++++++++++ 6 files changed, 218 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index ec2fe99..3daedd6 100644 --- a/README.md +++ b/README.md @@ -1854,8 +1854,24 @@ without being asked: hucre convert veriler.csv out.xlsx --encoding windows-1254 ``` -On the write side there is nothing to configure: `writeCsv` emits UTF-8, -and `bom: true` is what makes Excel open it correctly on every locale. +On the write side the output is always UTF-8 — `TextEncoder` encodes +nothing else, by specification, so writing windows-1254 is not something a +Web-API-only library can do. What matters instead is the mark, because +Excel on a non-UTF-8 locale reads a UTF-8 CSV as its system code page +without one: + +```ts +writeCsv(rows, { bom: true }) +await write({ sheets, format: "csv", csv: { delimiter: ";", bom: true } }) +``` + +```bash +hucre convert veriler.xlsx out.csv --bom +``` + +Each text format takes its own bag through `write` — `csv`, `tsv`, `json`, +`ndjson`, `xml`, `html`, `markdown` — so the format-agnostic entry can +configure the writer it dispatches to. ### Schema Validation diff --git a/src/_inline-cells.ts b/src/_inline-cells.ts index 241848d..1d66556 100644 --- a/src/_inline-cells.ts +++ b/src/_inline-cells.ts @@ -108,7 +108,7 @@ export function splitInlineCells(sheet: WriteSheet): WriteSheet { for (let r = 0; r < rows.length; r++) { const row = rows[r]! - const plain: CellValue[] = new Array(row.length) + const plain: CellValue[] = [] for (let c = 0; c < row.length; c++) { const v = row[c] if (isInlineCell(v)) { @@ -116,9 +116,9 @@ export function splitInlineCells(sheet: WriteSheet): WriteSheet { // The value stays in the grid too, so everything that reads only // `rows` — auto-width, a pivot's source range, a table's extent — // sees the cell rather than a hole. - plain[c] = v.value ?? null + plain.push(v.value ?? null) } else { - plain[c] = v as CellValue + plain.push(v as CellValue) } } plainRows[r] = plain diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 35271e4..655d152 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -17,7 +17,6 @@ import { readXlsb } from "../xlsx/xlsb/reader" import { readXls } from "../xls/reader" import { readOds } from "../ods/reader" import { parseCsv } from "../csv/reader" -import { writeCsv } from "../csv/writer" import { validateWithSchema } from "../_schema" import { read, write } from "../defter" import type { Workbook, CellValue, WriteOptions, SchemaDefinition } from "../_types" @@ -238,6 +237,13 @@ export const convertCommand = defineCommand({ "Output format when writing to stdout (`-`), which has no " + "extension to read. E.g. `--to csv`.", }, + bom: { + type: "boolean", + description: + "Start CSV/TSV output with a UTF-8 byte-order mark. Excel needs " + + "it to read a UTF-8 CSV as UTF-8 on a non-UTF-8 locale; without " + + "it the accented characters arrive as mojibake.", + }, }, async run({ args }) { const inputPath = args.input as string @@ -262,7 +268,7 @@ export const convertCommand = defineCommand({ say(consola.start, `Writing ${outputPath}...`) - const output = await renderWorkbook(workbook, outputFormat, outputPath) + const output = await renderWorkbook(workbook, outputFormat, outputPath, args.bom === true) if (toStdout) writeFileSync(1, output) else writeFileSync(outputPath, output) @@ -303,25 +309,27 @@ async function renderWorkbook( workbook: Workbook, format: WritableFormat, outputPath: string, + bom: boolean, ): Promise { - const encoder = new TextEncoder() - - if (format === "csv") { - const sheet = workbook.sheets[0] - if (!sheet) throw new CliError("No sheets found in input file") - // Every row goes through writeCsv, including the first. It used to - // be pulled out as `headers` and stringified separately, so a Date - // in row 0 came out ISO while the same Date in row 1 came out in - // writeCsv's format — one column, two formats, decided by which - // row the value happened to land in. - return encoder.encode(writeCsv(sheet.rows, { delimiter: delimiterForExtension(outputPath) })) + if (format === "csv" && !workbook.sheets[0]) { + throw new CliError("No sheets found in input file") } const writeOptions: WriteOptions = { sheets: workbook.sheets.map((sheet) => ({ name: sheet.name, rows: sheet.rows })), properties: workbook.properties, } - return write({ ...writeOptions, format }) + + // Every row goes through writeCsv, including the first. It used to be + // pulled out as `headers` and stringified separately, so a Date in row 0 + // came out ISO while the same Date in row 1 came out in writeCsv's + // format — one column, two formats, decided by which row the value + // happened to land in. + return write({ + ...writeOptions, + format, + csv: { delimiter: delimiterForExtension(outputPath), bom }, + }) } // ── Inspect Command ───────────────────────────────────────────────── diff --git a/src/defter.ts b/src/defter.ts index 883199d..8c86129 100644 --- a/src/defter.ts +++ b/src/defter.ts @@ -13,7 +13,12 @@ import type { ReadInput, TableDefinition, TableColumn, + CsvWriteOptions, } from "./_types" +import type { JsonWriteOptions } from "./json/writer" +import type { XmlWriteOptions } from "./xml/data-writer" +import type { HtmlExportOptions } from "./export/html" +import type { MarkdownExportOptions } from "./export/markdown" import { collectHeaders, rowsToObjects, selectSheet } from "./_objects" import { readXlsx } from "./xlsx/reader" import { readXlsb, looksLikeXlsb } from "./xlsx/xlsb/reader" @@ -207,8 +212,25 @@ export type WriteFormat = * `hucre convert` documents. The return is always bytes, so a caller can * hand the result to `Response` or `writeFile` without branching. */ +export interface TextFormatOptions { + /** Options for `format: "csv"`. */ + csv?: CsvWriteOptions + /** Options for `format: "tsv"`. The delimiter is the tab and not yours. */ + tsv?: Omit + /** Options for `format: "json"`. */ + json?: JsonWriteOptions + /** Options for `format: "ndjson"`. */ + ndjson?: Pick + /** Options for `format: "xml"`. */ + xml?: XmlWriteOptions + /** Options for `format: "html"`. */ + html?: HtmlExportOptions + /** Options for `format: "markdown"`. */ + markdown?: MarkdownExportOptions +} + export async function write( - options: WriteOptions & { format?: WriteFormat }, + options: WriteOptions & { format?: WriteFormat } & TextFormatOptions, ): Promise { const format = options.format ?? "xlsx" if (format === "xlsx") return writeXlsx(options) @@ -223,22 +245,29 @@ export async function write( // the two spreadsheet writers do. See #433. const rows = toCellValues(sheet.rows ?? []) + // Each text writer already takes an options bag; this function used to + // call every one of them with none, so `write` — the entry #469 added + // precisely so one call could reach all nine formats — was the only way + // to reach seven of them that could not configure any. `bom: true` was + // the one that mattered: it is what makes Excel open a UTF-8 CSV on a + // non-UTF-8 locale, and #475 documents it as the answer while `write` + // gave no way to ask for it. const encoder = new TextEncoder() switch (format) { case "csv": - return encoder.encode(writeCsv(rows)) + return encoder.encode(writeCsv(rows, options.csv)) case "tsv": - return encoder.encode(writeTsv(rows)) + return encoder.encode(writeTsv(rows, options.tsv)) case "json": - return encoder.encode(writeJson(rowsToRecords(rows))) + return encoder.encode(writeJson(rowsToRecords(rows), options.json)) case "ndjson": - return encoder.encode(writeNdjson(rowsToRecords(rows))) + return encoder.encode(writeNdjson(rowsToRecords(rows), options.ndjson)) case "xml": - return encoder.encode(writeXml(rowsToRecords(rows))) + return encoder.encode(writeXml(rowsToRecords(rows), options.xml)) case "html": - return encoder.encode(toHtml({ name: sheet.name, rows })) + return encoder.encode(toHtml({ name: sheet.name, rows }, options.html)) case "markdown": - return encoder.encode(toMarkdown({ name: sheet.name, rows })) + return encoder.encode(toMarkdown({ name: sheet.name, rows }, options.markdown)) } } diff --git a/test/cli.test.ts b/test/cli.test.ts index 1036bf5..4a71f5b 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -230,6 +230,33 @@ describe("tab-separated files", () => { expect(readFileSync(path("out.csv"), "utf-8")).toContain("h1,h2") }) + it("writes a UTF-8 BOM when asked, and not otherwise", async () => { + // Excel on a non-UTF-8 locale reads a UTF-8 CSV as the system code + // page unless the file opens with EF BB BF, so `convert x.xlsx + // out.csv` produced mojibake and there was no way to ask for better. + // See #475 — which answered the read side and documented `bom: true` + // as the write side's remedy, while the CLI could not pass it. + const input = await makeXlsx("in.xlsx", [["Şehir", "Ürün"]]) + + await run(convertCommand, { input, output: path("plain.csv") }) + const plain = readFileSync(path("plain.csv")) + expect(plain[0]).not.toBe(0xef) + + await run(convertCommand, { input, output: path("bom.csv"), bom: true }) + const withBom = readFileSync(path("bom.csv")) + expect([withBom[0], withBom[1], withBom[2]]).toEqual([0xef, 0xbb, 0xbf]) + // The mark is a prefix, not a replacement — the rows are still there. + expect(new TextDecoder().decode(withBom)).toContain("Şehir") + }) + + it("writes the BOM on .tsv too", async () => { + const input = await makeXlsx("in.xlsx", [["a", "b"]]) + await run(convertCommand, { input, output: path("bom.tsv"), bom: true }) + const bytes = readFileSync(path("bom.tsv")) + expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xef, 0xbb, 0xbf]) + expect(new TextDecoder().decode(bytes)).toContain("a\tb") + }) + it("round-trips a tab-separated file without changing its separator", async () => { writeFileSync(path("in.tsv"), "a\tb\nc\td", "utf-8") await run(convertCommand, { input: path("in.tsv"), output: path("out.tsv") }) diff --git a/test/unified-text-formats.test.ts b/test/unified-text-formats.test.ts index 3606b15..60f9546 100644 --- a/test/unified-text-formats.test.ts +++ b/test/unified-text-formats.test.ts @@ -211,3 +211,114 @@ describe("the header-row convention is the same in both directions", () => { expect(dec(await write({ sheets, format: "json" }))).toContain("column2") }) }) + +// ═══════════════════════════════════════════════════════════════════════ +// Every text writer takes an options bag; `write` called all seven with +// none. So the entry #469 added precisely so one call could reach all +// nine formats was the only way to reach seven of them that could not +// configure any of them — including `bom: true`, which #475 documents as +// the answer to Excel opening a UTF-8 CSV as its system code page. +// ═══════════════════════════════════════════════════════════════════════ + +describe("text-format options reach their writer", () => { + const sheet = { + name: "S", + rows: [ + ["Şehir", "Ürün"], + ["İzmir", 3], + ], + } + + it("csv: delimiter and bom", async () => { + const bytes = (await write({ + sheets: [sheet], + format: "csv", + csv: { delimiter: ";", bom: true }, + })) as Uint8Array + + expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xef, 0xbb, 0xbf]) + expect(dec(bytes)).toContain("Şehir;Ürün") + }) + + it("csv: escapeFormulae", async () => { + const out = dec( + (await write({ + sheets: [{ name: "S", rows: [["=1+1"]] }], + format: "csv", + csv: { escapeFormulae: true }, + })) as Uint8Array, + ) + expect(out).toContain("'=1+1") + }) + + it("tsv: bom, with the tab still the delimiter", async () => { + const bytes = (await write({ + sheets: [sheet], + format: "tsv", + tsv: { bom: true }, + })) as Uint8Array + expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xef, 0xbb, 0xbf]) + expect(dec(bytes)).toContain("Şehir\tÜrün") + }) + + it("json: pretty and indent", async () => { + const out = dec( + (await write({ + sheets: [sheet], + format: "json", + json: { pretty: true, indent: " " }, + })) as Uint8Array, + ) + expect(out).toContain("\n ") + }) + + it("xml: rootTag and rowTag", async () => { + // ASCII headers: `writeXml` rejects a non-ASCII element name, which + // XML 1.0 §2.3 allows. Tracked separately. + const out = dec( + (await write({ + sheets: [ + { + name: "S", + rows: [ + ["city", "qty"], + ["Izmir", 3], + ], + }, + ], + format: "xml", + xml: { rootTag: "cities", rowTag: "city_row" }, + })) as Uint8Array, + ) + expect(out).toContain("") + expect(out).toContain("") + }) + + it("html: caption and header row", async () => { + const out = dec( + (await write({ + sheets: [sheet], + format: "html", + html: { caption: "Şehirler", hasHeaderRow: true }, + })) as Uint8Array, + ) + expect(out).toContain("Şehirler") + expect(out).toContain("") + }) + + it("markdown: alignment", async () => { + const out = dec( + (await write({ + sheets: [sheet], + format: "markdown", + markdown: { alignment: ["right", "right"] }, + })) as Uint8Array, + ) + expect(out).toContain("--:") + }) + + it("leaves the defaults alone when no bag is passed", async () => { + const out = dec((await write({ sheets: [sheet], format: "csv" })) as Uint8Array) + expect(out.startsWith("Şehir,Ürün")).toBe(true) + }) +})