feat: the write side of #475 — text-format options, and a CLI --bom

README.md

@@ -1854,8 +1854,24 @@ without being asked:
 hucre convert veriler.csv out.xlsx --encoding windows-1254
 ```
 
-On the write side there is nothing to configure: `writeCsv` emits UTF-8,
-and `bom: true` is what makes Excel open it correctly on every locale.
+On the write side the output is always UTF-8 — `TextEncoder` encodes
+nothing else, by specification, so writing windows-1254 is not something a
+Web-API-only library can do. What matters instead is the mark, because
+Excel on a non-UTF-8 locale reads a UTF-8 CSV as its system code page
+without one:
+
+```ts
+writeCsv(rows, { bom: true })
+await write({ sheets, format: "csv", csv: { delimiter: ";", bom: true } })
+```
+
+```bash
+hucre convert veriler.xlsx out.csv --bom
+```
+
+Each text format takes its own bag through `write` — `csv`, `tsv`, `json`,
+`ndjson`, `xml`, `html`, `markdown` — so the format-agnostic entry can
+configure the writer it dispatches to.
 
 ### Schema Validation
 

src/_inline-cells.ts

@@ -108,17 +108,17 @@ export function splitInlineCells(sheet: WriteSheet): WriteSheet {
 
   for (let r = 0; r < rows.length; r++) {
     const row = rows[r]!
-    const plain: CellValue[] = new Array(row.length)
+    const plain: CellValue[] = []
     for (let c = 0; c < row.length; c++) {
       const v = row[c]
       if (isInlineCell(v)) {
         lifted.set(`${r},${c}`, v)
         // The value stays in the grid too, so everything that reads only
         // `rows` — auto-width, a pivot's source range, a table's extent —
         // sees the cell rather than a hole.
-        plain[c] = v.value ?? null
+        plain.push(v.value ?? null)
       } else {
-        plain[c] = v as CellValue
+        plain.push(v as CellValue)
       }
     }
     plainRows[r] = plain

src/cli/commands.ts

@@ -17,7 +17,6 @@ import { readXlsb } from "../xlsx/xlsb/reader"
 import { readXls } from "../xls/reader"
 import { readOds } from "../ods/reader"
 import { parseCsv } from "../csv/reader"
-import { writeCsv } from "../csv/writer"
 import { validateWithSchema } from "../_schema"
 import { read, write } from "../defter"
 import type { Workbook, CellValue, WriteOptions, SchemaDefinition } from "../_types"
@@ -238,6 +237,13 @@ export const convertCommand = defineCommand({
         "Output format when writing to stdout (`-`), which has no " +
         "extension to read. E.g. `--to csv`.",
     },
+    bom: {
+      type: "boolean",
+      description:
+        "Start CSV/TSV output with a UTF-8 byte-order mark. Excel needs " +
+        "it to read a UTF-8 CSV as UTF-8 on a non-UTF-8 locale; without " +
+        "it the accented characters arrive as mojibake.",
+    },
   },
   async run({ args }) {
     const inputPath = args.input as string
@@ -262,7 +268,7 @@ export const convertCommand = defineCommand({
 
     say(consola.start, `Writing ${outputPath}...`)
 
-    const output = await renderWorkbook(workbook, outputFormat, outputPath)
+    const output = await renderWorkbook(workbook, outputFormat, outputPath, args.bom === true)
     if (toStdout) writeFileSync(1, output)
     else writeFileSync(outputPath, output)
 
@@ -303,25 +309,27 @@ async function renderWorkbook(
   workbook: Workbook,
   format: WritableFormat,
   outputPath: string,
+  bom: boolean,
 ): Promise<Uint8Array> {
-  const encoder = new TextEncoder()
-
-  if (format === "csv") {
-    const sheet = workbook.sheets[0]
-    if (!sheet) throw new CliError("No sheets found in input file")
-    // Every row goes through writeCsv, including the first. It used to
-    // be pulled out as `headers` and stringified separately, so a Date
-    // in row 0 came out ISO while the same Date in row 1 came out in
-    // writeCsv's format — one column, two formats, decided by which
-    // row the value happened to land in.
-    return encoder.encode(writeCsv(sheet.rows, { delimiter: delimiterForExtension(outputPath) }))
+  if (format === "csv" && !workbook.sheets[0]) {
+    throw new CliError("No sheets found in input file")
   }
 
   const writeOptions: WriteOptions = {
     sheets: workbook.sheets.map((sheet) => ({ name: sheet.name, rows: sheet.rows })),
     properties: workbook.properties,
   }
-  return write({ ...writeOptions, format })
+
+  // Every row goes through writeCsv, including the first. It used to be
+  // pulled out as `headers` and stringified separately, so a Date in row 0
+  // came out ISO while the same Date in row 1 came out in writeCsv's
+  // format — one column, two formats, decided by which row the value
+  // happened to land in.
+  return write({
+    ...writeOptions,
+    format,
+    csv: { delimiter: delimiterForExtension(outputPath), bom },
+  })
 }
 
 // ── Inspect Command ─────────────────────────────────────────────────

src/defter.ts

@@ -13,7 +13,12 @@ import type {
   ReadInput,
   TableDefinition,
   TableColumn,
+  CsvWriteOptions,
 } from "./_types"
+import type { JsonWriteOptions } from "./json/writer"
+import type { XmlWriteOptions } from "./xml/data-writer"
+import type { HtmlExportOptions } from "./export/html"
+import type { MarkdownExportOptions } from "./export/markdown"
 import { collectHeaders, rowsToObjects, selectSheet } from "./_objects"
 import { readXlsx } from "./xlsx/reader"
 import { readXlsb, looksLikeXlsb } from "./xlsx/xlsb/reader"
@@ -207,8 +212,25 @@ export type WriteFormat =
  * `hucre convert` documents. The return is always bytes, so a caller can
  * hand the result to `Response` or `writeFile` without branching.
  */
+export interface TextFormatOptions {
+  /** Options for `format: "csv"`. */
+  csv?: CsvWriteOptions
+  /** Options for `format: "tsv"`. The delimiter is the tab and not yours. */
+  tsv?: Omit<CsvWriteOptions, "delimiter">
+  /** Options for `format: "json"`. */
+  json?: JsonWriteOptions
+  /** Options for `format: "ndjson"`. */
+  ndjson?: Pick<JsonWriteOptions, "unflatten">
+  /** Options for `format: "xml"`. */
+  xml?: XmlWriteOptions
+  /** Options for `format: "html"`. */
+  html?: HtmlExportOptions
+  /** Options for `format: "markdown"`. */
+  markdown?: MarkdownExportOptions
+}
+
 export async function write(
-  options: WriteOptions & { format?: WriteFormat },
+  options: WriteOptions & { format?: WriteFormat } & TextFormatOptions,
 ): Promise<WriteOutput> {
   const format = options.format ?? "xlsx"
   if (format === "xlsx") return writeXlsx(options)
@@ -223,22 +245,29 @@ export async function write(
   // the two spreadsheet writers do. See #433.
   const rows = toCellValues(sheet.rows ?? [])
 
+  // Each text writer already takes an options bag; this function used to
+  // call every one of them with none, so `write` — the entry #469 added
+  // precisely so one call could reach all nine formats — was the only way
+  // to reach seven of them that could not configure any. `bom: true` was
+  // the one that mattered: it is what makes Excel open a UTF-8 CSV on a
+  // non-UTF-8 locale, and #475 documents it as the answer while `write`
+  // gave no way to ask for it.
   const encoder = new TextEncoder()
   switch (format) {
     case "csv":
-      return encoder.encode(writeCsv(rows))
+      return encoder.encode(writeCsv(rows, options.csv))
     case "tsv":
-      return encoder.encode(writeTsv(rows))
+      return encoder.encode(writeTsv(rows, options.tsv))
     case "json":
-      return encoder.encode(writeJson(rowsToRecords(rows)))
+      return encoder.encode(writeJson(rowsToRecords(rows), options.json))
     case "ndjson":
-      return encoder.encode(writeNdjson(rowsToRecords(rows)))
+      return encoder.encode(writeNdjson(rowsToRecords(rows), options.ndjson))
     case "xml":
-      return encoder.encode(writeXml(rowsToRecords(rows)))
+      return encoder.encode(writeXml(rowsToRecords(rows), options.xml))
     case "html":
-      return encoder.encode(toHtml({ name: sheet.name, rows }))
+      return encoder.encode(toHtml({ name: sheet.name, rows }, options.html))
     case "markdown":
-      return encoder.encode(toMarkdown({ name: sheet.name, rows }))
+      return encoder.encode(toMarkdown({ name: sheet.name, rows }, options.markdown))
   }
 }
 

test/cli.test.ts

@@ -230,6 +230,33 @@ describe("tab-separated files", () => {
     expect(readFileSync(path("out.csv"), "utf-8")).toContain("h1,h2")
   })
 
+  it("writes a UTF-8 BOM when asked, and not otherwise", async () => {
+    // Excel on a non-UTF-8 locale reads a UTF-8 CSV as the system code
+    // page unless the file opens with EF BB BF, so `convert x.xlsx
+    // out.csv` produced mojibake and there was no way to ask for better.
+    // See #475 — which answered the read side and documented `bom: true`
+    // as the write side's remedy, while the CLI could not pass it.
+    const input = await makeXlsx("in.xlsx", [["Şehir", "Ürün"]])
+
+    await run(convertCommand, { input, output: path("plain.csv") })
+    const plain = readFileSync(path("plain.csv"))
+    expect(plain[0]).not.toBe(0xef)
+
+    await run(convertCommand, { input, output: path("bom.csv"), bom: true })
+    const withBom = readFileSync(path("bom.csv"))
+    expect([withBom[0], withBom[1], withBom[2]]).toEqual([0xef, 0xbb, 0xbf])
+    // The mark is a prefix, not a replacement — the rows are still there.
+    expect(new TextDecoder().decode(withBom)).toContain("Şehir")
+  })
+
+  it("writes the BOM on .tsv too", async () => {
+    const input = await makeXlsx("in.xlsx", [["a", "b"]])
+    await run(convertCommand, { input, output: path("bom.tsv"), bom: true })
+    const bytes = readFileSync(path("bom.tsv"))
+    expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xef, 0xbb, 0xbf])
+    expect(new TextDecoder().decode(bytes)).toContain("a\tb")
+  })
+
   it("round-trips a tab-separated file without changing its separator", async () => {
     writeFileSync(path("in.tsv"), "a\tb\nc\td", "utf-8")
     await run(convertCommand, { input: path("in.tsv"), output: path("out.tsv") })

test/unified-text-formats.test.ts

@@ -211,3 +211,114 @@ describe("the header-row convention is the same in both directions", () => {
     expect(dec(await write({ sheets, format: "json" }))).toContain("column2")
   })
 })
+
+// ═══════════════════════════════════════════════════════════════════════
+// Every text writer takes an options bag; `write` called all seven with
+// none. So the entry #469 added precisely so one call could reach all
+// nine formats was the only way to reach seven of them that could not
+// configure any of them — including `bom: true`, which #475 documents as
+// the answer to Excel opening a UTF-8 CSV as its system code page.
+// ═══════════════════════════════════════════════════════════════════════
+
+describe("text-format options reach their writer", () => {
+  const sheet = {
+    name: "S",
+    rows: [
+      ["Şehir", "Ürün"],
+      ["İzmir", 3],
+    ],
+  }
+
+  it("csv: delimiter and bom", async () => {
+    const bytes = (await write({
+      sheets: [sheet],
+      format: "csv",
+      csv: { delimiter: ";", bom: true },
+    })) as Uint8Array
+
+    expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xef, 0xbb, 0xbf])
+    expect(dec(bytes)).toContain("Şehir;Ürün")
+  })
+
+  it("csv: escapeFormulae", async () => {
+    const out = dec(
+      (await write({
+        sheets: [{ name: "S", rows: [["=1+1"]] }],
+        format: "csv",
+        csv: { escapeFormulae: true },
+      })) as Uint8Array,
+    )
+    expect(out).toContain("'=1+1")
+  })
+
+  it("tsv: bom, with the tab still the delimiter", async () => {
+    const bytes = (await write({
+      sheets: [sheet],
+      format: "tsv",
+      tsv: { bom: true },
+    })) as Uint8Array
+    expect([bytes[0], bytes[1], bytes[2]]).toEqual([0xef, 0xbb, 0xbf])
+    expect(dec(bytes)).toContain("Şehir\tÜrün")
+  })
+
+  it("json: pretty and indent", async () => {
+    const out = dec(
+      (await write({
+        sheets: [sheet],
+        format: "json",
+        json: { pretty: true, indent: "    " },
+      })) as Uint8Array,
+    )
+    expect(out).toContain("\n    ")
+  })
+
+  it("xml: rootTag and rowTag", async () => {
+    // ASCII headers: `writeXml` rejects a non-ASCII element name, which
+    // XML 1.0 §2.3 allows. Tracked separately.
+    const out = dec(
+      (await write({
+        sheets: [
+          {
+            name: "S",
+            rows: [
+              ["city", "qty"],
+              ["Izmir", 3],
+            ],
+          },
+        ],
+        format: "xml",
+        xml: { rootTag: "cities", rowTag: "city_row" },
+      })) as Uint8Array,
+    )
+    expect(out).toContain("<cities>")
+    expect(out).toContain("<city_row>")
+  })
+
+  it("html: caption and header row", async () => {
+    const out = dec(
+      (await write({
+        sheets: [sheet],
+        format: "html",
+        html: { caption: "Şehirler", hasHeaderRow: true },
+      })) as Uint8Array,
+    )
+    expect(out).toContain("<caption>Şehirler</caption>")
+    expect(out).toContain("<thead>")
+  })
+
+  it("markdown: alignment", async () => {
+    const out = dec(
+      (await write({
+        sheets: [sheet],
+        format: "markdown",
+        markdown: { alignment: ["right", "right"] },
+      })) as Uint8Array,
+    )
+    expect(out).toContain("--:")
+  })
+
+  it("leaves the defaults alone when no bag is passed", async () => {
+    const out = dec((await write({ sheets: [sheet], format: "csv" })) as Uint8Array)
+    expect(out.startsWith("Şehir,Ürün")).toBe(true)
+  })
+})