From d2ea437565de80f12658c11a6a4dd35461ef396b Mon Sep 17 00:00:00 2001 From: Chimi6 <43803253+Chimi6@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:36:24 -0600 Subject: [PATCH 1/2] feat: Japanese in every document and data format Legacy encodings read everywhere: text, Markdown, and HTML in Shift_JIS, EUC-JP, or ISO-2022-JP (declared or detected), CSV/TSV and XML decoded as they stream, and Word's RTF in code page 932. The PDF writer embeds CID-keyed CFF fonts (Hiragino, Noto Sans CJK), prefers Japanese fonts for Japanese, and breaks Japanese lines with kinsoku. --- DOCS/CHANGELOG.md | 6 + DOCS/STATE.md | 3 +- DOCS/formats/csv.md | 5 +- DOCS/formats/html.md | 4 +- DOCS/formats/pdf.md | 13 +- DOCS/formats/rtf.md | 7 +- src/converters/input.rs | 172 +++- src/converters/to_pdf.rs | 4 +- src/converters/xml_to_json.rs | 5 +- src/io/cjk/mod.rs | 342 ++++++++ src/io/font/cff.rs | 786 ++++++++++++++++++ src/io/font/mod.rs | 37 +- src/io/font/system.rs | 30 +- src/io/pdf/compose.rs | 79 +- src/io/pdf/writer.rs | 36 +- src/io/rtf/reader.rs | 22 +- tests/fixtures/fonts/NotoSansCJKjp-Subset.otf | Bin 0 -> 9640 bytes tests/fixtures/fonts/OFL-NotoSansCJK.txt | 92 ++ tests/fixtures/fonts/README.md | 9 + tests/fixtures/japanese-input/ansicpg932.rtf | 1 + .../japanese-input/ansicpg932.rtf.expected | 1 + tests/fixtures/japanese-input/euc-jp.csv | 3 + .../japanese-input/euc-jp.csv.expected | 3 + tests/fixtures/japanese-input/euc-jp.txt | 1 + .../japanese-input/euc-jp.txt.expected | 1 + tests/fixtures/japanese-input/fcharset128.rtf | 1 + .../japanese-input/fcharset128.rtf.expected | 1 + tests/fixtures/japanese-input/make.py | 48 ++ tests/fixtures/japanese-input/shift-jis.csv | 3 + .../japanese-input/shift-jis.csv.expected | 3 + tests/fixtures/japanese-input/shift-jis.html | 1 + .../japanese-input/shift-jis.html.expected | 1 + tests/fixtures/japanese-input/shift-jis.txt | 1 + .../japanese-input/shift-jis.txt.expected | 1 + tests/fixtures/japanese-input/shift-jis.xml | 2 + .../japanese-input/shift-jis.xml.expected | 1 + tests/japanese_input.rs | 64 ++ tests/to_pdf.rs | 30 + 38 files changed, 1753 insertions(+), 66 deletions(-) create mode 100644 src/io/font/cff.rs create mode 100644 tests/fixtures/fonts/NotoSansCJKjp-Subset.otf create mode 100644 tests/fixtures/fonts/OFL-NotoSansCJK.txt create mode 100644 tests/fixtures/japanese-input/ansicpg932.rtf create mode 100644 tests/fixtures/japanese-input/ansicpg932.rtf.expected create mode 100644 tests/fixtures/japanese-input/euc-jp.csv create mode 100644 tests/fixtures/japanese-input/euc-jp.csv.expected create mode 100644 tests/fixtures/japanese-input/euc-jp.txt create mode 100644 tests/fixtures/japanese-input/euc-jp.txt.expected create mode 100644 tests/fixtures/japanese-input/fcharset128.rtf create mode 100644 tests/fixtures/japanese-input/fcharset128.rtf.expected create mode 100644 tests/fixtures/japanese-input/make.py create mode 100644 tests/fixtures/japanese-input/shift-jis.csv create mode 100644 tests/fixtures/japanese-input/shift-jis.csv.expected create mode 100644 tests/fixtures/japanese-input/shift-jis.html create mode 100644 tests/fixtures/japanese-input/shift-jis.html.expected create mode 100644 tests/fixtures/japanese-input/shift-jis.txt create mode 100644 tests/fixtures/japanese-input/shift-jis.txt.expected create mode 100644 tests/fixtures/japanese-input/shift-jis.xml create mode 100644 tests/fixtures/japanese-input/shift-jis.xml.expected create mode 100644 tests/japanese_input.rs diff --git a/DOCS/CHANGELOG.md b/DOCS/CHANGELOG.md index 352dcf84..daa1baef 100644 --- a/DOCS/CHANGELOG.md +++ b/DOCS/CHANGELOG.md @@ -6,6 +6,12 @@ section under a version heading. ## [Unreleased] +### Added + +- Japanese in the legacy encodings reads everywhere. Plain text, Markdown, and HTML in Shift_JIS, EUC-JP, or ISO-2022-JP (an HTML `charset` or XML `encoding` is followed; otherwise the encoding is detected), CSV and TSV in Shift_JIS (as Excel saves a Japanese CSV) or EUC-JP, decoded as they stream, and XML declaring Shift_JIS. Detection takes bytes for Japanese only when they read as Japanese does (common kana and kanji), so a Western file is still refused with an error rather than turned into kanji. Word's RTF with Japanese as Shift_JIS byte pairs (`\ansicpg932`, `\fcharset128`) reads in full; it read as replacement characters before. +- PDFs embed the fonts CJK systems ship: CID-keyed CFF OpenType (Hiragino on the Mac, Noto Sans CJK and Source Han Sans on Linux), subset to the glyphs and subroutines used (a Japanese page with Noto Sans CJK is 264 KB from a 16 MB font). Japanese text prefers a Japanese font (Hiragino, Noto Sans CJK, Yu Gothic, Meiryo, MS Gothic) over the broad fallbacks such as Arial Unicode. `--font` takes a CID-keyed CFF font too. +- Japanese in PDFs breaks lines between characters with the kinsoku rules: no line starts with closing brackets, punctuation, small kana, or the long-vowel mark, and none ends with an opening bracket. + ## [1.0.1] - 2026-10-30 ### Fixed diff --git a/DOCS/STATE.md b/DOCS/STATE.md index 5fef70c7..d7ca0f6a 100644 --- a/DOCS/STATE.md +++ b/DOCS/STATE.md @@ -6,6 +6,7 @@ describes. ## Now +- 2026-10-31: Japanese everywhere (branch `japanese-input`). Legacy encodings read in every text format: `read_text_document` decodes UTF-16, a declared charset, or detected Shift_JIS, EUC-JP, or ISO-2022-JP (`src/io/cjk`: common-row plausibility keeps Western bytes out), CSV/TSV and XML decode as they stream (`Utf8Reader`), RTF decodes code page 932. The PDF writer embeds CID-keyed CFF fonts (`src/io/font/cff.rs`: unused glyphs and subroutines blanked in place, CID = glyph id; checked in poppler, Quartz, and MuPDF with Hiragino and Noto Sans CJK), prefers Japanese fonts for Japanese, and breaks Japanese lines with kinsoku. A matrix of 20 Japanese document and 5 data sources to every format reads 100%. - 2026-10-30: Japanese PDF text (branch `japanese`, 1.0.1). A user's project found Mac-made PDFs reading 1 Japanese character in 360 and PDFs in the standard Japanese CMaps reading as nothing. Composite fonts now read through their encoding CMap (`src/io/pdf/font.rs`): Identity and embedded CMaps give CIDs, read through Adobe-Japan1 (`src/io/cjk`, Adobe's table) when the font is Japan1; the Unicode CMaps give Unicode; Shift_JIS, EUC-JP, and JIS CMaps are decoded (WHATWG's JIS X 0208 index). Also: radicals normalised to ideographs (Chrome), `/ActualText` (LibreOffice without a Japanese font), vertical writing read by column (a `-V` font, and the Mac's glyph-by-glyph columns), and Japanese lines joined without spaces in documents. On twelve test PDFs from seven producers every character reads as PyMuPDF reads it; on 70 to 100-page documents a whole run is 0.01 to 0.05 s at 8 to 13 MB, against PyMuPDF's extraction alone at 0.06 to 0.14 s and 186 MB. Tables add about 75 KB to the binary. - 2026-10-21: v1 wrap-up and 1.0.0 (branch `v1-wrapup`). The converter registry is data: each converter has a `const fn entry()` and the registry is a static table of entries, so planning runs no converter's code, and the 195 image pairs are built only when an image format is an end (`registry::converters_for`). Starting memory 2.24 -> 1.95 MB, a ROM conversion 2.66 -> 2.42 MB, startup 0.15 ms faster; sizes move by a few KB. Every benchmark pair was rerun and recorded (first run on the M1 Max for many pairs, under load); the failing lines are under Tech Debt. The bench harness was fixed for this Mac (`gtime` in `csv-json`, awk helpers in `ico-png`, a bash 5 guard in `jpeg-pdf`). ROADMAP item 9 done; the next cycle is audio and video. - 2026-10-20: N64 ROMs between `z64`, `v64`, and `n64` (`src/io/n64.rs`, branch `n64`): six lossless converters permute each 32-bit word in a 256 KB buffer, vectorized as an array shuffle, reading the input's order from where the header's `0x80` lies whatever the name. Detection on the command line now lets content overrule an extension whose signature it fails when another format's matches (`format::detect`, for every format), with a warning; errors without a position drop `line 0, column 0`. Checked on Paper Mario (USA), a 40 MB No-Intro dump in `~/Downloads` (not committed): round trips return its SHA-1, outputs byte-identical to ucon64, `n64romconvert`, and `dd conv=swab`. Pair `z64-v64`: every speed line passes (1.2 to 1.35 times `dd`); the four memory lines against ucon64 and `dd` fail by 0.1 to 0.6 MB, the binary's startup floor (Tech Debt). The bench harness takes `bench_runs` for millisecond pairs. The wrap-up is next. @@ -78,7 +79,7 @@ describes. ## Tech Debt -- 2026-10-30: Japanese beyond PDF reading. Plain text and HTML in Shift_JIS, EUC-JP, or ISO-2022-JP are refused as invalid UTF-8 (`src/converters/input.rs` reads UTF-8 only; the decoders are in `src/io/cjk`); RTF from Word with `\fcharset128` and `\'xx` byte pairs is untested (the code pages in `src/io/rtf/codepage.rs` are single-byte); the PDF writer embeds TrueType only, so the CFF fonts most CJK systems ship (Noto Sans CJK) are skipped, and Japanese wraps by character without the line-breaking rules (kinsoku). Every other document pair carries Japanese whole (pandoc, textutil, and LibreOffice sources to every document format, 100%). Chinese and Korean CID collections (GB1, CNS1, Korea1) have no CID table yet, so their fonts without ToUnicode read only through Unicode CMaps. +- 2026-10-30 (updated 2026-10-31): Japanese, what is left. Read and written across every document and data pair (sources from pandoc, textutil, LibreOffice, Word-style RTF, and Shift_JIS/EUC-JP files; `tests/japanese_input.rs`, `tests/pdf_japanese.rs`). Left: Chinese and Korean CID collections (GB1, CNS1, Korea1) have no CID table, so their fonts without ToUnicode read only through Unicode CMaps, and their RTF double-byte pages only through `\uN`; name-keyed CFF fonts (Latin `.otf`) are not embedded; the PDF writer sets Japanese horizontally only and bold Japanese in the regular weight (one face per file: Hiragino W6 is a separate file); Western code pages (Latin-1, Windows-1252) in text, HTML, and CSV are still refused rather than detected. - 2026-09-24 (updated 2026-10-21, moved from Blockers): The Pages document paths' throughput against the shared Pages goal of 50 MB/s of input (`DOCS/benchmarks/pages-markdown.md`, `pages-html.md`, `pages-text.md`). The 1.0.0 run, on an M1 Max under load: memory passes on every document row (26 to 47 MB against 64), Word passes both ways (Pages -> Word 129 and 140 MB/s of input plus uncompressed output, Word -> Pages 67 and 89), and the text paths fail throughput at 26 to 33 MB/s (styled 29 to 33, prose 26 to 28). A real resume converts in 3 ms at 4.8 MB, inside both goals. Where the time goes on the dense shape, in-process (2026-09-24): package decode 10 ms (7.5 of it Snappy), document build 25 ms, the text writers 11 ms; wall clock adds about 10 ms of process start, file I/O, and first-touch page faults. Levers under Spikes. Pages -> Word's peak memory rose on every row since 2026-10-06 (prose 33.8 -> 46.8 MB), still inside the goal. - 2026-09-24 (updated 2026-10-21, moved from Blockers): `pages -> pages-json` misses the 50 MB/s goal at 24 and 26 MB/s of package bytes (`DOCS/benchmarks/pages-json.md`); the reverse direction passes at 146 and 239. Levers: the typed decode of the attribute tables written as JSON directly (the Spikes entry below), then `Tree::decode` itself and the JSON writer's per-field work. - 2026-10-21: Benchmark lines failing in the 1.0.0 run that passed in their previous blocks. The run was on the M1 Max with the machine in use (load average 7 to 26 on 10 cores); for several pairs every earlier block was on the Linux i7, so these are not like for like. An alternated A/B of ec68950 against the commit before it showed the registry change caused none of those it covered. Rerun on an idle machine before acting on any of them. diff --git a/DOCS/formats/csv.md b/DOCS/formats/csv.md index b916a2b8..c6038589 100644 --- a/DOCS/formats/csv.md +++ b/DOCS/formats/csv.md @@ -3,7 +3,10 @@ Delimited rows, read and written a record at a time in constant memory (`src/io/csv`): RFC 4180 quoting (a field with the delimiter, a quote, or a line break is quoted, a quote doubled), any line ending read, `\n` -written, a UTF-8 byte order mark skipped. TSV is the same with a tab. +written, a UTF-8 byte order mark skipped. Input that is not UTF-8 but +reads as Japanese (Shift_JIS, as Excel saves a Japanese CSV, or EUC-JP) +is decoded as it is read (`src/io/cjk`); a Western code page is still +refused. TSV is the same with a tab. Every row format reaches every other (JSON, JSON Lines, Excel, Numbers, Markdown tables and the documents through them). diff --git a/DOCS/formats/html.md b/DOCS/formats/html.md index ace048ee..75be2430 100644 --- a/DOCS/formats/html.md +++ b/DOCS/formats/html.md @@ -75,4 +75,6 @@ gets one. table cells. - Character references are decoded only with their `;`, as the writer produces them; legacy forms without it stay as written. -- Encodings other than UTF-8 are not sniffed; the input is read as text. +- Encodings: UTF-8 and UTF-16 (by its byte order mark), and Japanese + (Shift_JIS, EUC-JP, ISO-2022-JP): the `charset` the page declares, else + detected when the input is not UTF-8. Western code pages are refused. diff --git a/DOCS/formats/pdf.md b/DOCS/formats/pdf.md index 30989ed3..41421da7 100644 --- a/DOCS/formats/pdf.md +++ b/DOCS/formats/pdf.md @@ -71,8 +71,8 @@ are at any effort). Fast is the same as balanced (deflate's default is near the |---|---| | Input | any reader's Markdown event stream (markdown, html, text directly; docx and pages through the document model) | | Page | US Letter, one-inch margins; a page is written when the next line would pass the bottom margin, so one page is held | -| Fonts | the base-14 Helvetica, Helvetica-Bold, -Oblique, -BoldOblique, Courier, Courier-Bold, WinAnsi, not embedded; widths from their AFM metrics. A character outside WinAnsi is set in a font installed on the machine that has it (`src/io/font/system.rs`: broad sans serifs first, then one per script, symbols, the large CJK fallbacks, then every other font, each probed by its character map alone), embedded as a subset (Type0, Identity-H, CIDFontType2 over a TrueType program holding only the glyphs used, with `/W` widths and a ToUnicode map). `--font file.ttf` (or `ConvertOptions::font`, which the WebAssembly module takes as bytes) sets all body text in that font instead, code staying in Courier. Only fonts with TrueType outlines are embedded; a character no such font has is `?` and reported | -| Text | paragraphs 11/15 pt, broken first fit on real widths (a word wider than the line is split by characters); headings 20, 16, 13 pt bold (4 to 6 at 11), kept with the two lines after them; soft breaks are spaces, hard breaks end the line | +| Fonts | the base-14 Helvetica, Helvetica-Bold, -Oblique, -BoldOblique, Courier, Courier-Bold, WinAnsi, not embedded; widths from their AFM metrics. A character outside WinAnsi is set in a font installed on the machine that has it (`src/io/font/system.rs`: broad sans serifs first, then the Japanese fonts (Hiragino Kaku Gothic, Noto Sans CJK, Source Han Sans, Yu Gothic, Meiryo, MS Gothic) and one per script, symbols, the large CJK fallbacks, then every other font, each probed by its character map alone), embedded as a subset (Type0, Identity-H, a CIDFontType2 over a TrueType program, or a CIDFontType0 over a CID-keyed CFF program (`FontFile3 /CIDFontType0C`, `src/io/font/cff.rs`: unused glyphs and subroutines blanked in place, each glyph's CID its id), holding only the glyphs used, with `/W` widths and a ToUnicode map). `--font file.ttf` (or `ConvertOptions::font`, which the WebAssembly module takes as bytes) sets all body text in that font instead, code staying in Courier. Fonts with TrueType outlines and CID-keyed CFF ones (every CJK `.otf`/`.ttc`) are embedded; name-keyed CFF fonts (Latin `.otf`) are not, and a character no embeddable font has is `?` and reported | +| Text | paragraphs 11/15 pt, broken first fit on real widths (a word wider than the line is split by characters; Japanese breaks between characters, kept from starting a line with closing brackets, punctuation, small kana, or the long-vowel mark, and from ending one with an opening bracket: kinsoku); headings 20, 16, 13 pt bold (4 to 6 at 11), kept with the two lines after them; soft breaks are spaces, hard breaks end the line | | Inline | bold, italic, and code change the font; links are blue and carry a URI annotation over each piece; footnote references as `[n]`; task markers as `[x]`/`[ ]` | | Blocks | lists indented 18 pt a level with `•` or `n.`, tight ones close; block quotes indented 16 pt with a gray bar; code 9.5 pt Courier on a gray band, lines kept, long ones wrapped; rules; tables sized from their content (natural widths when they fit, else the longest word plus a share of the rest), cells wrapped and aligned, header bold, a grid, rows kept whole across pages | | Images | each picture a block of its own at its size, scaled down to the text width and the page, on a new page when it does not fit: Word, Pages, and RTF pictures from the document, at the size it shows each (a picture shown twice at two sizes takes each); Markdown and HTML pictures by path from the input's folder (on the command line; `ConvertOptions::base`) or by `data:` URI, at their recorded resolution or 96 pixels to the inch. A JPEG is embedded unchanged (`/DCTDecode`); PNG, WebP, BMP, TIFF, QOI, and ICO are read whole first (a picture that fails leaves nothing behind) and deflated with PNG predictors, alpha as a soft mask. A picture not found, unreadable (GIF has no reader yet), in a table cell, or by web address (not fetched) keeps its alt text in brackets, and is counted | @@ -149,10 +149,11 @@ Markdown. headers, or footers. Complex scripts are set glyph by glyph in logical order: Hebrew and Arabic read left to right (no bidirectional reordering), Arabic letters take their isolated forms, and Indic - conjuncts are not formed (no shaping). Fonts with CFF outlines (most - CJK `.otf`/`.ttc`) are not embedded yet; another installed TrueType - font is used when one covers the text. A tab in running text is a - space. With `--font`, bold and italic body text is set in the one font + conjuncts are not formed (no shaping). Name-keyed CFF fonts (Latin + `.otf`) are not embedded; another installed font is used when one + covers the text. Japanese is set horizontally only (no vertical + writing), and bold Japanese in the regular weight (one face per font + file). A tab in running text is a space. With `--font`, bold and italic body text is set in the one font given. - WebP, QOI, TGA, Netpbm, and ICO record no resolution a page could use (WebP's lives in EXIF, which is not read), so their pages are a point per pixel. diff --git a/DOCS/formats/rtf.md b/DOCS/formats/rtf.md index a7c3b0a1..53b2d016 100644 --- a/DOCS/formats/rtf.md +++ b/DOCS/formats/rtf.md @@ -38,7 +38,7 @@ as their text, aside). | Feature | RTF | Notes | |---|---|---| | Fonts, colours | `\fonttbl`, `\colortbl` | A font's `\fcharset` picks the code page of its `\'hh` bytes; Symbol and Wingdings bytes map to their private-use characters, and bullets among them to real bullets. | -| Text | text, `\'hh`, `\uN` with `\ucN` | Code pages 437, 850, 874, 1250-1258, and Mac Roman; surrogate pairs; the Asian double-byte pages only through `\uN` (Word always writes it). | +| Text | text, `\'hh`, `\uN` with `\ucN` | Code pages 437, 850, 874, 1250-1258, and Mac Roman; Japanese code page 932 (Shift_JIS byte pairs and half-width katakana, under `\ansicpg932` or a `\fcharset128` font); surrogate pairs; the Chinese and Korean double-byte pages only through `\uN` (Word always writes it). | | Styles | `\stylesheet`, `\sN`, `\csN`, `\sbasedon` | Style 0 is the default; a paragraph naming no style is in it. Built-in heading names give outline levels where the writer left them out. | | Character formatting | `\b \i \ul \strike \f \fs \cf \chcbpat \highlight \super \sub \up \dn \caps \scaps \v \lang \expndtw \charscalex` | Formatting in RTF is absolute: the reader keeps on a run only what differs from its paragraph's style, and gives text RTF's defaults where the style sets something the text does not. | | Paragraph formatting | `\q* \li \ri \fi \sb \sa \sl \slmult \keepn \keep \widctlpar \outlinelevel \pagebb \contextualspace \cbpat \brdr* \tx` | | @@ -65,6 +65,7 @@ the header: LibreOffice spaces paragraphs by older rules without Equations (written as their text), charts, OLE objects (their result picture is kept), drawn shapes with neither fill nor outline, form fields -(their shown text is kept), and text in double-byte code pages without -`\uN` escapes (each such character becomes U+FFFD). RTFD (TextEdit's +(their shown text is kept), and text in the Chinese and Korean +double-byte code pages without `\uN` escapes (each such character +becomes U+FFFD; Japanese, code page 932, is decoded). RTFD (TextEdit's bundle of RTF and attachments) is a later step. diff --git a/src/converters/input.rs b/src/converters/input.rs index 25ddafaa..86ae4894 100644 --- a/src/converters/input.rs +++ b/src/converters/input.rs @@ -4,17 +4,20 @@ use std::io::Read; use crate::converter::{ConvertError, Input, Location}; -/// Reads the whole input as UTF-8, replacing NUL with U+FFFD as the -/// CommonMark specification requires for security. An invalid sequence is -/// reported with the line it starts on. +/// Reads the whole input as text, replacing NUL with U+FFFD as the +/// CommonMark specification requires for security. UTF-8 is read as it +/// is (a byte-order mark dropped), and UTF-16 by its byte-order mark; +/// anything else is Japanese if it reads as Japanese: in the encoding the +/// document declares (an HTML `charset`, an XML `encoding`), else the +/// more plausible of Shift_JIS, EUC-JP, and ISO-2022-JP. Input that is +/// none of these is refused with the line its first bad byte is on. pub fn read_text_document(input: &mut Input<'_>) -> Result { let mut bytes = Vec::new(); input.read_to_end(&mut bytes)?; - let text = match String::from_utf8(bytes) { + let text = match decode_text(bytes) { Ok(text) => text, - Err(error) => { - let offset = error.utf8_error().valid_up_to(); - let newlines = error.as_bytes()[..offset] + Err((bytes, offset)) => { + let newlines = bytes[..offset] .iter() .filter(|byte| **byte == b'\n') .count(); @@ -23,7 +26,8 @@ pub fn read_text_document(input: &mut Input<'_>) -> Result line: newlines as u64 + 1, column: 0, }, - message: "invalid UTF-8".to_string(), + message: "invalid UTF-8 (and not Japanese in Shift_JIS, EUC-JP, or ISO-2022-JP)" + .to_string(), }); } }; @@ -33,6 +37,43 @@ pub fn read_text_document(input: &mut Input<'_>) -> Result Ok(text) } +/// Text from bytes in UTF-8, UTF-16 (by its mark), or a Japanese +/// encoding; the bytes back, with where UTF-8 failed, when none fits. +fn decode_text(bytes: Vec) -> Result, usize)> { + if let Some(rest) = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) + && let Ok(text) = std::str::from_utf8(rest) + { + return Ok(text.to_string()); + } + for (mark, big_endian) in [([0xFE, 0xFF], true), ([0xFF, 0xFE], false)] { + if let Some(rest) = bytes.strip_prefix(&mark) { + let units = rest.chunks(2).map(|pair| { + let pair = [pair[0], *pair.get(1).unwrap_or(&0)]; + if big_endian { + u16::from_be_bytes(pair) + } else { + u16::from_le_bytes(pair) + } + }); + return Ok(char::decode_utf16(units) + .map(|unit| unit.unwrap_or('\u{FFFD}')) + .collect()); + } + } + let (bytes, offset) = match String::from_utf8(bytes) { + Ok(text) => return Ok(text), + Err(error) => { + let offset = error.utf8_error().valid_up_to(); + (error.into_bytes(), offset) + } + }; + let encoding = crate::io::cjk::declared(&bytes).or_else(|| crate::io::cjk::sniff(&bytes)); + match encoding { + Some(encoding) => Ok(encoding.decode(&bytes)), + None => Err((bytes, offset)), + } +} + /// The bytes read to find a CSV input's delimiter: enough for its first /// records. const SNIFF_BYTES: usize = 64 * 1024; @@ -46,13 +87,17 @@ pub fn written_delimiter(format_delimiter: u8, context: &crate::event::Context<' } } -/// A stream's first bytes, read to find its delimiter, then the rest. -pub type HeadAndRest<'a> = std::io::Chain>, Input<'a>>; +/// What delimited input is read through once its first bytes have been +/// looked at: those bytes and then the rest, decoded from Shift_JIS or +/// EUC-JP when they are Japanese. +pub type HeadAndRest<'a> = Box; /// Delimited input and the delimiter to read it with: TSV's tab, or for /// CSV `--delimiter`'s, else the one its first records use (a semicolon -/// export reads as one). A rewindable input is rewound after the look and -/// handed back as it was; a stream comes back as the bytes read first and +/// export reads as one). Input that is not UTF-8 but reads as Japanese +/// (Shift_JIS, as Excel saves a Japanese CSV, or EUC-JP) is decoded as it +/// is read. A rewindable UTF-8 input is rewound after the look and handed +/// back as it was; anything else comes back as the bytes read first and /// then the rest, held in `rest`. Either way the readers see an `Input`, /// the type they are compiled and tuned for. pub fn delimited_input<'a: 'b, 'b>( @@ -61,29 +106,98 @@ pub fn delimited_input<'a: 'b, 'b>( context: &crate::event::Context<'_>, rest: &'b mut Option>, ) -> std::io::Result<(u8, Input<'b>)> { + let mut head = Vec::new(); + let whole = (&mut input) + .take(SNIFF_BYTES as u64) + .read_to_end(&mut head)? + < SNIFF_BYTES; + let encoding = match std::str::from_utf8(&head) { + Ok(_) => None, + // Cut at the end of the sample: still UTF-8. + Err(error) if error.error_len().is_none() && !whole => None, + Err(_) => crate::io::cjk::sniff(&head), + }; let delimiter = match (format_delimiter, context.options.delimiter) { (b',', Some(delimiter)) => delimiter, (b',', None) => { - let mut head = Vec::new(); - let whole = (&mut input) - .take(SNIFF_BYTES as u64) - .read_to_end(&mut head)? - < SNIFF_BYTES; - let delimiter = crate::io::csv::sniff_delimiter(&head, whole); - return Ok(match input { - Input::Rewindable(reader) => { - reader.rewind()?; - (delimiter, shorten(Input::Rewindable(reader))) - } - stream => { - let chained = rest.insert(std::io::Cursor::new(head).chain(stream)); - (delimiter, Input::Stream(chained)) - } - }); + let sample = match encoding { + Some(encoding) => encoding.decode(&head).into_bytes(), + None => head.clone(), + }; + crate::io::csv::sniff_delimiter(&sample, whole) } (other, _) => other, }; - Ok((delimiter, shorten(input))) + let Some(encoding) = encoding else { + return Ok(match input { + Input::Rewindable(reader) => { + reader.rewind()?; + (delimiter, shorten(Input::Rewindable(reader))) + } + stream => { + let chained = rest.insert(Box::new(std::io::Cursor::new(head).chain(stream))); + (delimiter, Input::Stream(&mut **chained)) + } + }); + }; + let source = std::io::Cursor::new(head).chain(input); + let decoded: HeadAndRest<'a> = match encoding { + // ISO-2022-JP keeps a mode across the text: decoded whole. + crate::io::cjk::Encoding::Iso2022Jp => { + let mut bytes = Vec::new(); + let mut source = source; + source.read_to_end(&mut bytes)?; + Box::new(std::io::Cursor::new(encoding.decode(&bytes).into_bytes())) + } + _ => Box::new(crate::io::cjk::Utf8Reader::new(source, encoding)), + }; + let reader = rest.insert(decoded); + Ok((delimiter, Input::Stream(&mut **reader))) +} + +/// `input` decoded as it is read when it is Japanese rather than UTF-8: +/// in the encoding its first bytes declare (an XML `encoding`), else the +/// one they read as. UTF-8 input comes back as it was (rewound, or its +/// first bytes and then the rest). +pub fn decoded_input<'a: 'b, 'b>( + mut input: Input<'a>, + rest: &'b mut Option>, +) -> std::io::Result> { + let mut head = Vec::new(); + let whole = (&mut input) + .take(SNIFF_BYTES as u64) + .read_to_end(&mut head)? + < SNIFF_BYTES; + let encoding = match std::str::from_utf8(&head) { + Ok(_) => crate::io::cjk::declared(&head) + .filter(|_| head.iter().any(|byte| *byte >= 0x80 || *byte == 0x1B)), + Err(error) if error.error_len().is_none() && !whole => None, + Err(_) => crate::io::cjk::declared(&head).or_else(|| crate::io::cjk::sniff(&head)), + }; + let Some(encoding) = encoding else { + return Ok(match input { + Input::Rewindable(reader) => { + reader.rewind()?; + shorten(Input::Rewindable(reader)) + } + stream => { + let chained = rest.insert(Box::new(std::io::Cursor::new(head).chain(stream))); + Input::Stream(&mut **chained) + } + }); + }; + let source = std::io::Cursor::new(head).chain(input); + let decoded: HeadAndRest<'a> = match encoding { + crate::io::cjk::Encoding::Iso2022Jp => { + let mut bytes = Vec::new(); + let mut source = source; + source.read_to_end(&mut bytes)?; + Box::new(std::io::Cursor::new(encoding.decode(&bytes).into_bytes())) + } + _ => Box::new(crate::io::cjk::Utf8Reader::new(source, encoding)), + }; + let reader = rest.insert(decoded); + Ok(Input::Stream(&mut **reader)) } /// `input` for a shorter lifetime (a reborrow of what it reads from): the diff --git a/src/converters/to_pdf.rs b/src/converters/to_pdf.rs index 6dcbb29a..181567d8 100644 --- a/src/converters/to_pdf.rs +++ b/src/converters/to_pdf.rs @@ -105,10 +105,10 @@ impl Converter for ToPdf { )?)), None => None, }; - if forced.as_ref().is_some_and(|font| !font.has_glyf()) { + if forced.as_ref().is_some_and(|font| !font.embeddable()) { return Err(ConvertError::Malformed { location: Location::default(), - message: "the font has CFF outlines; only TrueType-outline fonts (.ttf) are embedded so far".to_string(), + message: "the font has name-keyed CFF outlines; TrueType and CID-keyed CFF fonts (every CJK font) are embedded so far".to_string(), }); } let faces = Faces::new(forced, true); diff --git a/src/converters/xml_to_json.rs b/src/converters/xml_to_json.rs index 824c58ad..7509d05a 100644 --- a/src/converters/xml_to_json.rs +++ b/src/converters/xml_to_json.rs @@ -18,10 +18,13 @@ pub struct XmlToJson; impl Converter for XmlToJson { fn convert( &self, - mut input: Input<'_>, + input: Input<'_>, output: &mut dyn Write, context: &mut Context<'_>, ) -> Result<(), ConvertError> { + // Japanese XML (`encoding="Shift_JIS"`) is decoded as it is read. + let mut rest = None; + let mut input = crate::converters::input::decoded_input(input, &mut rest)?; let parsed = tree::parse_reader(&mut input)?; for note in parsed.notes { context.loss(NAME, Location::default(), note); diff --git a/src/io/cjk/mod.rs b/src/io/cjk/mod.rs index a5ed23fa..f147df92 100644 --- a/src/io/cjk/mod.rs +++ b/src/io/cjk/mod.rs @@ -140,6 +140,283 @@ pub fn decode_euc_jp(bytes: &[u8]) -> String { decode_with(bytes, euc_jp_char) } +/// ISO-2022-JP bytes as text: ASCII, JIS X 0201 Roman and katakana, and +/// JIS X 0208, switched by their escape sequences. +pub fn decode_iso_2022_jp(bytes: &[u8]) -> String { + #[derive(Clone, Copy, PartialEq)] + enum Mode { + Ascii, + Katakana, + Jis0208, + } + let mut out = String::with_capacity(bytes.len()); + let mut mode = Mode::Ascii; + let mut at = 0; + while at < bytes.len() { + let byte = bytes[at]; + if byte == 0x1B { + let switched = match (bytes.get(at + 1), bytes.get(at + 2)) { + (Some(b'('), Some(b'B' | b'J')) => Some(Mode::Ascii), + (Some(b'('), Some(b'I')) => Some(Mode::Katakana), + (Some(b'$'), Some(b'@' | b'B')) => Some(Mode::Jis0208), + _ => None, + }; + if let Some(switched) = switched { + mode = switched; + at += 3; + continue; + } + } + if byte == b'\n' || byte == b'\r' || mode == Mode::Ascii { + out.push(char::from(byte)); + at += 1; + continue; + } + if mode == Mode::Katakana { + let kana = (0x21..=0x5F) + .contains(&byte) + .then(|| char::from_u32(0xFF61 + u32::from(byte - 0x21))) + .flatten(); + out.push(kana.unwrap_or('\u{FFFD}')); + at += 1; + continue; + } + let code = u32::from(byte) << 8 | u32::from(bytes.get(at + 1).copied().unwrap_or(0)); + out.push(jis_code(code).unwrap_or('\u{FFFD}')); + at += 2; + } + out +} + +/// A Japanese encoding text may be in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Encoding { + ShiftJis, + EucJp, + Iso2022Jp, +} + +impl Encoding { + /// The encoding a charset name (`Shift_JIS`, `EUC-JP`, ...) names. + pub fn named(name: &str) -> Option { + match name.trim().to_ascii_lowercase().as_str() { + "shift_jis" | "shift-jis" | "sjis" | "x-sjis" | "windows-31j" | "cp932" + | "ms_kanji" | "csshiftjis" | "ms932" => Some(Encoding::ShiftJis), + "euc-jp" | "euc_jp" | "x-euc-jp" | "eucjp" | "cseucpkdfmtjapanese" => { + Some(Encoding::EucJp) + } + "iso-2022-jp" | "csiso2022jp" | "jis" => Some(Encoding::Iso2022Jp), + _ => None, + } + } + + pub fn label(self) -> &'static str { + match self { + Encoding::ShiftJis => "Shift_JIS", + Encoding::EucJp => "EUC-JP", + Encoding::Iso2022Jp => "ISO-2022-JP", + } + } + + /// The bytes as text in this encoding, a bad sequence as U+FFFD. + pub fn decode(self, bytes: &[u8]) -> String { + match self { + Encoding::ShiftJis => decode_shift_jis(bytes), + Encoding::EucJp => decode_euc_jp(bytes), + Encoding::Iso2022Jp => decode_iso_2022_jp(bytes), + } + } +} + +/// The Japanese encoding a document's first bytes declare: an HTML or +/// CSS `charset`, or an XML declaration's `encoding`. +pub fn declared(head: &[u8]) -> Option { + let head = String::from_utf8_lossy(&head[..head.len().min(2048)]).to_ascii_lowercase(); + for key in ["charset=", "encoding="] { + let mut from = 0; + while let Some(found) = head[from..].find(key) { + let at = from + found + key.len(); + let value: String = head[at..] + .trim_start_matches(['"', '\'', ' ']) + .chars() + .take_while(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + .collect(); + if let Some(encoding) = Encoding::named(&value) { + return Some(encoding); + } + from = at; + } + } + None +} + +/// The Japanese encoding bytes that are not UTF-8 are most plausibly in: +/// ISO-2022-JP when its escape is there, else Shift_JIS or EUC-JP, +/// whichever reads with fewer bad sequences (then fewer half-width +/// katakana, what EUC-JP turns into when misread as Shift_JIS). Either +/// must read with fewer than one bad sequence in a hundred characters, +/// and with most of its two-byte characters (seven in ten) in JIS rows 1 +/// to 47: punctuation, kana, and the common (Level 1) kanji, where real +/// Japanese lives; Western bytes misread as Japanese land in the rare +/// rows instead, and are not taken for it. A sequence cut off at the end +/// of `bytes` (a sample of a longer input) does not count against it. +pub fn sniff(bytes: &[u8]) -> Option { + if bytes + .windows(3) + .any(|window| window == b"\x1b$B" || window == b"\x1b$@") + { + return Some(Encoding::Iso2022Jp); + } + let shift_jis = tally(bytes, Encoding::ShiftJis); + let euc = tally(bytes, Encoding::EucJp); + let best = if (euc.bad, euc.half_kana) < (shift_jis.bad, shift_jis.half_kana) { + euc + } else { + shift_jis + }; + let pairs = best.common + best.rare; + let plausible = + best.bad * 100 < best.characters.max(1) && pairs > 0 && best.common * 10 >= pairs * 7; + plausible.then_some(best.encoding) +} + +/// How bytes read in one encoding: characters, bad sequences, half-width +/// katakana, and two-byte characters in the common rows and the rest. +#[derive(Clone, Copy)] +struct Tally { + encoding: Encoding, + characters: usize, + bad: usize, + half_kana: usize, + common: usize, + rare: usize, +} + +fn tally(bytes: &[u8], encoding: Encoding) -> Tally { + // Rows 1 to 47 of JIS X 0208: pointers below 47 times 94. + const COMMON_ROWS: usize = 47 * 94; + let mut tally = Tally { + encoding, + characters: 0, + bad: 0, + half_kana: 0, + common: 0, + rare: 0, + }; + let mut at = 0; + while at < bytes.len() { + let rest = &bytes[at..]; + let (ch, length) = match encoding { + Encoding::ShiftJis => shift_jis_char(rest), + _ => euc_jp_char(rest), + }; + let length = length.max(1); + let cut = + at + length > bytes.len() || (ch.is_none() && length == 1 && at + 1 == bytes.len()); + tally.characters += 1; + match ch { + None if !cut => tally.bad += 1, + None => {} + Some(ch) if ('\u{FF61}'..='\u{FF9F}').contains(&ch) => tally.half_kana += 1, + Some(_) if length == 2 => { + let pointer = match encoding { + Encoding::ShiftJis => { + let (lead, trail) = (rest[0], rest[1]); + let lead_offset = if lead < 0xA0 { 0x81 } else { 0xC1 }; + let trail_offset = if trail < 0x7F { 0x40 } else { 0x41 }; + usize::from(lead - lead_offset) * 188 + usize::from(trail - trail_offset) + } + _ => { + usize::from(rest[0] - 0xA1) * 94 + usize::from(rest[1].saturating_sub(0xA1)) + } + }; + if pointer < COMMON_ROWS { + tally.common += 1; + } else { + tally.rare += 1; + } + } + Some(_) => {} + } + at += length; + } + tally +} + +/// A reader that turns Shift_JIS or EUC-JP bytes into UTF-8 as they are +/// read, carrying a character cut between two reads over to the next. +pub struct Utf8Reader { + inner: R, + encoding: Encoding, + /// Bytes read but not yet decoded: the start of a cut character. + pending: Vec, + /// Decoded UTF-8 not yet handed out. + decoded: Vec, + at: usize, + done: bool, +} + +impl Utf8Reader { + pub fn new(inner: R, encoding: Encoding) -> Utf8Reader { + Utf8Reader { + inner, + encoding, + pending: Vec::new(), + decoded: Vec::new(), + at: 0, + done: false, + } + } + + fn refill(&mut self) -> std::io::Result<()> { + let mut chunk = vec![0u8; 64 * 1024]; + let count = self.inner.read(&mut chunk)?; + if count == 0 { + self.done = true; + } + self.pending.extend_from_slice(&chunk[..count]); + // Decode up to the last whole character; keep a cut one. + let mut end = self.pending.len(); + if !self.done { + let mut at = 0; + let mut last_whole = 0; + while at < self.pending.len() { + let length = match self.encoding { + Encoding::ShiftJis => shift_jis_length(self.pending[at]), + _ => euc_jp_length(self.pending[at]), + }; + if at + length > self.pending.len() { + break; + } + at += length; + last_whole = at; + } + end = last_whole; + } + let text = self.encoding.decode(&self.pending[..end]); + self.pending.drain(..end); + self.decoded.clear(); + self.decoded.extend_from_slice(text.as_bytes()); + self.at = 0; + Ok(()) + } +} + +impl std::io::Read for Utf8Reader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + while self.at >= self.decoded.len() { + if self.done { + return Ok(0); + } + self.refill()?; + } + let count = buffer.len().min(self.decoded.len() - self.at); + buffer[..count].copy_from_slice(&self.decoded[self.at..self.at + count]); + self.at += count; + Ok(count) + } +} + fn decode_with(bytes: &[u8], next: fn(&[u8]) -> (Option, usize)) -> String { let mut out = String::with_capacity(bytes.len() * 3 / 2); let mut at = 0; @@ -218,6 +495,71 @@ mod tests { assert!(!japan1(999_999, &mut text)); } + #[test] + fn decodes_iso_2022_jp() { + // ESC $ B, 日 (0x467C), ESC ( B, then ASCII. + let bytes = [0x1B, b'$', b'B', 0x46, 0x7C, 0x1B, b'(', b'B', b'!']; + assert_eq!(decode_iso_2022_jp(&bytes), "日!"); + } + + #[test] + fn tells_shift_jis_from_euc_jp() { + let text = "日本語の文章です。ひらがなとカタカナ。"; + let shift_jis = encode_shift_jis(text); + assert_eq!(sniff(&shift_jis), Some(Encoding::ShiftJis)); + // EUC-JP: each JIS X 0208 pointer as two bytes from 0xA1. + let mut euc = Vec::new(); + for ch in text.chars() { + let pointer = tables::JIS0208 + .iter() + .position(|code| u32::from(*code) == ch as u32) + .expect("in JIS X 0208"); + euc.push((pointer / 94 + 0xA1) as u8); + euc.push((pointer % 94 + 0xA1) as u8); + } + assert_eq!(sniff(&euc), Some(Encoding::EucJp)); + assert_eq!(decode_euc_jp(&euc), text); + // Latin-1 bytes are not Japanese. + assert_eq!( + sniff( + "café crème brûlée" + .as_bytes() + .iter() + .map(|b| if *b > 0x7F { 0xE9 } else { *b }) + .collect::>() + .as_slice() + ), + None + ); + assert_eq!( + declared(br#""#), + Some(Encoding::ShiftJis) + ); + } + + #[test] + fn decodes_a_stream_cut_between_bytes() { + use std::io::Read; + let bytes = encode_shift_jis("日本語ｶﾅ、English"); + // One byte at a time: every two-byte character is cut. + struct Trickle<'a>(&'a [u8]); + impl Read for Trickle<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let Some((first, rest)) = self.0.split_first() else { + return Ok(0); + }; + buffer[0] = *first; + self.0 = rest; + Ok(1) + } + } + let mut text = String::new(); + Utf8Reader::new(Trickle(&bytes), Encoding::ShiftJis) + .read_to_string(&mut text) + .expect("reads"); + assert_eq!(text, "日本語ｶﾅ、English"); + } + #[test] fn radicals_read_as_ideographs() { assert_eq!(unify_radical('\u{2F47}'), '日'); diff --git a/src/io/font/cff.rs b/src/io/font/cff.rs new file mode 100644 index 00000000..b1ff0f08 --- /dev/null +++ b/src/io/font/cff.rs @@ -0,0 +1,786 @@ +//! CFF outlines (the `CFF ` table of an OpenType font) subset for a PDF: +//! a CID-keyed font (every CJK font: Hiragino, Noto Sans CJK, Source Han) +//! keeps only the glyphs a document uses and the subroutines they call. +//! +//! Nothing is renumbered: an unused glyph's charstring becomes a bare +//! `endchar` and an unused subroutine a bare `return`, each in its place, +//! so glyph ids, subroutine numbers, and the font's FDSelect stay as they +//! were. The charset is rewritten so that every glyph's CID is its glyph +//! id, which lets a PDF address glyphs by id through `Identity-H`, as it +//! does TrueType fonts. The subroutines a glyph calls are found by +//! walking its Type 2 charstring (counting stem hints to step over hint +//! masks), so the subset holds exactly what drawing the glyphs needs. + +use std::collections::BTreeSet; + +use super::{FontError, fail}; + +/// A CFF INDEX: its items' byte ranges in the table. +struct Index { + items: Vec>, + end: usize, +} + +fn read_index(data: &[u8], at: usize) -> Result { + let count = usize::from(u16::from_be_bytes([ + *data.get(at).ok_or_else(|| fail("CFF INDEX cut short"))?, + *data + .get(at + 1) + .ok_or_else(|| fail("CFF INDEX cut short"))?, + ])); + if count == 0 { + return Ok(Index { + items: Vec::new(), + end: at + 2, + }); + } + let size = usize::from( + *data + .get(at + 2) + .ok_or_else(|| fail("CFF INDEX cut short"))?, + ); + if !(1..=4).contains(&size) { + return Err(fail("CFF INDEX offset size out of range")); + } + let offsets_at = at + 3; + let offset = |index: usize| -> Result { + let start = offsets_at + index * size; + let bytes = data + .get(start..start + size) + .ok_or_else(|| fail("CFF INDEX cut short"))?; + Ok(bytes + .iter() + .fold(0usize, |value, byte| value << 8 | usize::from(*byte))) + }; + let data_start = offsets_at + (count + 1) * size - 1; + let mut items = Vec::with_capacity(count); + for index in 0..count { + let (start, end) = (offset(index)?, offset(index + 1)?); + if end < start || data_start + end > data.len() { + return Err(fail("CFF INDEX offsets out of range")); + } + items.push(data_start + start..data_start + end); + } + let end = data_start + offset(count)?; + Ok(Index { items, end }) +} + +fn write_index(items: &[&[u8]], out: &mut Vec) { + out.extend_from_slice(&(items.len() as u16).to_be_bytes()); + if items.is_empty() { + return; + } + let total: usize = items.iter().map(|item| item.len()).sum::() + 1; + let size: usize = match total { + 0..=0xFF => 1, + 0x100..=0xFFFF => 2, + 0x1_0000..=0xFF_FFFF => 3, + _ => 4, + }; + out.push(size as u8); + let mut offset = 1usize; + let put = |value: usize, out: &mut Vec| { + out.extend_from_slice(&value.to_be_bytes()[std::mem::size_of::() - size..]); + }; + put(offset, out); + for item in items { + offset += item.len(); + put(offset, out); + } + for item in items { + out.extend_from_slice(item); + } +} + +/// A DICT operand. +#[derive(Debug, Clone, PartialEq)] +enum Operand { + Integer(i32), + /// A real, kept as its encoded bytes (nibbles), which are copied out. + Real(Vec), +} + +impl Operand { + fn integer(&self) -> Option { + match self { + Operand::Integer(value) => Some(*value), + Operand::Real(_) => None, + } + } +} + +/// A DICT: its entries in order, each operator (12 x as 1200 + x) with +/// its operands. +type Dict = Vec<(u16, Vec)>; + +fn read_dict(data: &[u8]) -> Result { + let mut entries = Vec::new(); + let mut operands = Vec::new(); + let mut at = 0; + while at < data.len() { + let byte = data[at]; + match byte { + 0..=21 => { + let operator = if byte == 12 { + at += 1; + 1200 + u16::from(*data.get(at).ok_or_else(|| fail("CFF DICT cut short"))?) + } else { + u16::from(byte) + }; + at += 1; + entries.push((operator, std::mem::take(&mut operands))); + } + 28 => { + let value = i16::from_be_bytes([ + *data.get(at + 1).ok_or_else(|| fail("CFF DICT cut short"))?, + *data.get(at + 2).ok_or_else(|| fail("CFF DICT cut short"))?, + ]); + operands.push(Operand::Integer(i32::from(value))); + at += 3; + } + 29 => { + let bytes = data + .get(at + 1..at + 5) + .ok_or_else(|| fail("CFF DICT cut short"))?; + operands.push(Operand::Integer(i32::from_be_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], + ]))); + at += 5; + } + 30 => { + let start = at; + at += 1; + loop { + let nibbles = *data.get(at).ok_or_else(|| fail("CFF DICT cut short"))?; + at += 1; + if nibbles & 0x0F == 0x0F || nibbles >> 4 == 0x0F { + break; + } + } + operands.push(Operand::Real(data[start..at].to_vec())); + } + 32..=246 => { + operands.push(Operand::Integer(i32::from(byte) - 139)); + at += 1; + } + 247..=250 => { + let next = i32::from(*data.get(at + 1).ok_or_else(|| fail("CFF DICT cut short"))?); + operands.push(Operand::Integer((i32::from(byte) - 247) * 256 + next + 108)); + at += 2; + } + 251..=254 => { + let next = i32::from(*data.get(at + 1).ok_or_else(|| fail("CFF DICT cut short"))?); + operands.push(Operand::Integer( + -(i32::from(byte) - 251) * 256 - next - 108, + )); + at += 2; + } + _ => return Err(fail("CFF DICT holds a reserved byte")), + } + } + Ok(entries) +} + +/// A DICT encoded; every integer an offset could be is written in the +/// fixed five-byte form, so a DICT's size does not depend on where things +/// end up. +fn write_dict(dict: &Dict) -> Vec { + let mut out = Vec::new(); + for (operator, operands) in dict { + for operand in operands { + match operand { + Operand::Integer(value) => { + out.push(29); + out.extend_from_slice(&value.to_be_bytes()); + } + Operand::Real(bytes) => out.extend_from_slice(bytes), + } + } + if *operator >= 1200 { + out.push(12); + out.push((operator - 1200) as u8); + } else { + out.push(*operator as u8); + } + } + out +} + +fn get(dict: &Dict, operator: u16) -> Option<&[Operand]> { + dict.iter() + .find(|(known, _)| *known == operator) + .map(|(_, operands)| operands.as_slice()) +} + +fn set(dict: &mut Dict, operator: u16, operands: Vec) { + match dict.iter_mut().find(|(known, _)| *known == operator) { + Some(entry) => entry.1 = operands, + None => dict.push((operator, operands)), + } +} + +const CHARSET: u16 = 15; +const CHAR_STRINGS: u16 = 17; +const PRIVATE: u16 = 18; +const SUBRS: u16 = 19; +const ROS: u16 = 1230; +const CHARSTRING_TYPE: u16 = 1206; +const FD_ARRAY: u16 = 1236; +const FD_SELECT: u16 = 1237; + +/// The subroutine bias Type 2 charstrings add to an index. +fn bias(count: usize) -> i32 { + if count < 1240 { + 107 + } else if count < 33900 { + 1131 + } else { + 32768 + } +} + +/// What walking a charstring tracks: the operand stack (its numbers, for +/// the subroutine a call names), the stem hints seen (for the bytes a +/// hint mask takes), and which subroutines were called. +struct Walk<'a> { + data: &'a [u8], + global: &'a [std::ops::Range], + local: &'a [std::ops::Range], + used_global: &'a mut BTreeSet, + used_local: &'a mut BTreeSet, + stack: Vec, + stems: usize, + done: bool, +} + +impl Walk<'_> { + fn run(&mut self, range: std::ops::Range, depth: usize) -> Result<(), FontError> { + if depth > 10 { + return Err(fail("CFF subroutines nest too deep")); + } + let code = &self.data[range]; + let mut at = 0; + while at < code.len() && !self.done { + let byte = code[at]; + match byte { + 32..=246 => { + self.stack.push(i32::from(byte) - 139); + at += 1; + } + 247..=250 => { + let next = i32::from(*code.get(at + 1).unwrap_or(&0)); + self.stack.push((i32::from(byte) - 247) * 256 + next + 108); + at += 2; + } + 251..=254 => { + let next = i32::from(*code.get(at + 1).unwrap_or(&0)); + self.stack.push(-(i32::from(byte) - 251) * 256 - next - 108); + at += 2; + } + 28 => { + let value = i16::from_be_bytes([ + *code.get(at + 1).unwrap_or(&0), + *code.get(at + 2).unwrap_or(&0), + ]); + self.stack.push(i32::from(value)); + at += 3; + } + 255 => { + // A 16.16 fixed number: its integer part is enough here. + let value = i32::from_be_bytes([ + *code.get(at + 1).unwrap_or(&0), + *code.get(at + 2).unwrap_or(&0), + *code.get(at + 3).unwrap_or(&0), + *code.get(at + 4).unwrap_or(&0), + ]); + self.stack.push(value >> 16); + at += 5; + } + // hstem, vstem, hstemhm, vstemhm. + 1 | 3 | 18 | 23 => { + self.stems += self.stack.len() / 2; + self.stack.clear(); + at += 1; + } + // hintmask, cntrmask: stems first (an implied vstem), then + // a byte for every eight stems. + 19 | 20 => { + self.stems += self.stack.len() / 2; + self.stack.clear(); + at += 1 + self.stems.div_ceil(8); + } + 10 | 29 => { + let local = byte == 10; + let Some(number) = self.stack.pop() else { + return Err(fail("CFF subroutine call with no number")); + }; + let subrs = if local { self.local } else { self.global }; + let index = number + bias(subrs.len()); + let Some(subr) = usize::try_from(index) + .ok() + .and_then(|index| subrs.get(index).cloned()) + else { + return Err(fail("CFF subroutine number out of range")); + }; + if local { + self.used_local.insert(index as usize); + } else { + self.used_global.insert(index as usize); + } + self.run(subr, depth + 1)?; + at += 1; + } + 11 => return Ok(()), + 14 => { + self.done = true; + return Ok(()); + } + 12 => { + self.stack.clear(); + at += 2; + } + _ => { + self.stack.clear(); + at += 1; + } + } + } + Ok(()) + } +} + +/// Whether CFF table bytes (its first few kilobytes suffice) hold a +/// CID-keyed font: its Top DICT has the ROS operator. +pub fn is_cid_keyed(cff: &[u8]) -> bool { + let parse = || -> Result { + let header_size = usize::from(*cff.get(2).ok_or_else(|| fail("short"))?); + let names = read_index(cff, header_size)?; + let tops = read_index(cff, names.end)?; + let top = tops.items.first().ok_or_else(|| fail("no Top DICT"))?; + Ok(get(&read_dict(&cff[top.clone()])?, ROS).is_some()) + }; + parse().unwrap_or(false) +} + +/// The Registry, Ordering, and Supplement a CID-keyed font names. +pub fn registry_ordering(cff: &[u8]) -> Option<(String, String, i32)> { + let header_size = usize::from(*cff.get(2)?); + let names = read_index(cff, header_size).ok()?; + let tops = read_index(cff, names.end).ok()?; + let strings = read_index(cff, tops.end).ok()?; + let top = read_dict(&cff[tops.items.first()?.clone()]).ok()?; + let ros = get(&top, ROS)?; + let string = |sid: i32| -> Option { + let index = usize::try_from(sid).ok()?.checked_sub(391)?; + let range = strings.items.get(index)?.clone(); + Some(String::from_utf8_lossy(&cff[range]).into_owned()) + }; + Some(( + string(ros.first()?.integer()?)?, + string(ros.get(1)?.integer()?)?, + ros.get(2)?.integer()?, + )) +} + +/// The CFF table of a CID-keyed font holding only `glyphs` (and glyph 0), +/// every glyph's CID set to its glyph id. +pub fn subset(cff: &[u8], glyphs: &BTreeSet) -> Result, FontError> { + let header_size = usize::from(*cff.get(2).ok_or_else(|| fail("CFF cut short"))?); + let names = read_index(cff, header_size)?; + let tops = read_index(cff, names.end)?; + let strings = read_index(cff, tops.end)?; + let global = read_index(cff, strings.end)?; + let top_range = tops + .items + .first() + .ok_or_else(|| fail("CFF has no Top DICT"))? + .clone(); + let mut top = read_dict(&cff[top_range])?; + if get(&top, ROS).is_none() { + return Err(fail("only CID-keyed CFF fonts are subset")); + } + if get(&top, CHARSTRING_TYPE) + .and_then(|operands| operands.first()?.integer()) + .unwrap_or(2) + != 2 + { + return Err(fail("CFF charstrings are not Type 2")); + } + let offset_of = |dict: &Dict, operator: u16| -> Result { + get(dict, operator) + .and_then(|operands| operands.last()?.integer()) + .and_then(|value| usize::try_from(value).ok()) + .ok_or_else(|| fail("CFF DICT lacks an offset")) + }; + let char_strings = read_index(cff, offset_of(&top, CHAR_STRINGS)?)?; + let glyph_count = char_strings.items.len(); + let fd_select_at = offset_of(&top, FD_SELECT)?; + let fd_array = read_index(cff, offset_of(&top, FD_ARRAY)?)?; + // Each font DICT, its Private DICT, and that DICT's local subroutines. + struct FontDict { + dict: Dict, + private: Dict, + local: Vec>, + used: BTreeSet, + } + let mut fonts = Vec::with_capacity(fd_array.items.len()); + for range in &fd_array.items { + let dict = read_dict(&cff[range.clone()])?; + let private_operands = + get(&dict, PRIVATE).ok_or_else(|| fail("CFF font DICT lacks a Private DICT"))?; + let size = private_operands + .first() + .and_then(Operand::integer) + .unwrap_or(0) + .max(0) as usize; + let offset = private_operands + .get(1) + .and_then(Operand::integer) + .unwrap_or(0) + .max(0) as usize; + let private_bytes = cff + .get(offset..offset + size) + .ok_or_else(|| fail("CFF Private DICT outside the table"))?; + let private = read_dict(private_bytes)?; + let local = match get(&private, SUBRS).and_then(|operands| operands.first()?.integer()) { + Some(relative) => read_index(cff, offset + relative.max(0) as usize)?.items, + None => Vec::new(), + }; + fonts.push(FontDict { + dict, + private, + local, + used: BTreeSet::new(), + }); + } + let font_of = |glyph: usize| -> Result { + let format = *cff + .get(fd_select_at) + .ok_or_else(|| fail("CFF FDSelect cut short"))?; + match format { + 0 => Ok(usize::from( + *cff.get(fd_select_at + 1 + glyph) + .ok_or_else(|| fail("CFF FDSelect cut short"))?, + )), + 3 => { + let count = usize::from(u16::from_be_bytes([ + cff[fd_select_at + 1], + cff[fd_select_at + 2], + ])); + let mut chosen = 0; + for range in 0..count { + let at = fd_select_at + 3 + range * 3; + let first = usize::from(u16::from_be_bytes([cff[at], cff[at + 1]])); + if first > glyph { + break; + } + chosen = usize::from(cff[at + 2]); + } + Ok(chosen) + } + _ => Err(fail("CFF FDSelect format not supported")), + } + }; + // The glyphs kept, and every subroutine they call. + let mut keep: BTreeSet = glyphs + .iter() + .map(|glyph| usize::from(*glyph)) + .filter(|glyph| *glyph < glyph_count) + .collect(); + keep.insert(0); + let mut used_global = BTreeSet::new(); + for glyph in &keep { + let font = font_of(*glyph)?; + let Some(fd) = fonts.get_mut(font) else { + return Err(fail("CFF FDSelect names a font that is not there")); + }; + let mut walk = Walk { + data: cff, + global: &global.items, + local: &fd.local, + used_global: &mut used_global, + used_local: &mut fd.used, + stack: Vec::new(), + stems: 0, + done: false, + }; + walk.run(char_strings.items[*glyph].clone(), 0)?; + } + // Rebuilt pieces. + const ENDCHAR: &[u8] = &[14]; + const RETURN: &[u8] = &[11]; + let pick = |items: &[std::ops::Range], + used: &dyn Fn(usize) -> bool, + blank: &'static [u8]| + -> Vec> { + items + .iter() + .enumerate() + .map(|(index, range)| { + if used(index) { + cff[range.clone()].to_vec() + } else { + blank.to_vec() + } + }) + .collect() + }; + let char_string_data = pick(&char_strings.items, &|index| keep.contains(&index), ENDCHAR); + let global_data = pick(&global.items, &|index| used_global.contains(&index), RETURN); + let mut charset = vec![2u8]; + if glyph_count > 1 { + charset.extend_from_slice(&1u16.to_be_bytes()); + charset.extend_from_slice(&((glyph_count - 2) as u16).to_be_bytes()); + } + let fd_select_bytes = { + let format = cff[fd_select_at]; + let length = if format == 0 { + 1 + glyph_count + } else { + let count = usize::from(u16::from_be_bytes([ + cff[fd_select_at + 1], + cff[fd_select_at + 2], + ])); + 3 + count * 3 + 2 + }; + cff.get(fd_select_at..fd_select_at + length) + .ok_or_else(|| fail("CFF FDSelect cut short"))? + .to_vec() + }; + let mut private_blocks: Vec<(Vec, Vec)> = Vec::new(); + for fd in &mut fonts { + let local_data = pick(&fd.local, &|index| fd.used.contains(&index), RETURN); + let mut local_index = Vec::new(); + if !fd.local.is_empty() { + let refs: Vec<&[u8]> = local_data.iter().map(Vec::as_slice).collect(); + write_index(&refs, &mut local_index); + // Subrs points just past the Private DICT, whose size is fixed. + set(&mut fd.private, SUBRS, vec![Operand::Integer(0)]); + let size = write_dict(&fd.private).len(); + set(&mut fd.private, SUBRS, vec![Operand::Integer(size as i32)]); + } + private_blocks.push((write_dict(&fd.private), local_index)); + } + // Lay out: header, Name, Top DICT, String, global Subrs, then the + // charset, FDSelect, CharStrings, FDArray, and the Private DICTs. + let header = cff[..header_size].to_vec(); + let mut name_index = Vec::new(); + write_index( + &names + .items + .iter() + .map(|range| &cff[range.clone()]) + .collect::>(), + &mut name_index, + ); + let mut string_index = Vec::new(); + write_index( + &strings + .items + .iter() + .map(|range| &cff[range.clone()]) + .collect::>(), + &mut string_index, + ); + let mut global_index = Vec::new(); + write_index( + &global_data.iter().map(Vec::as_slice).collect::>(), + &mut global_index, + ); + let mut char_string_index = Vec::new(); + write_index( + &char_string_data + .iter() + .map(Vec::as_slice) + .collect::>(), + &mut char_string_index, + ); + // Offsets are written five bytes wide: set placeholders, measure, + // then fill in. + for operator in [CHARSET, CHAR_STRINGS, FD_SELECT, FD_ARRAY] { + set(&mut top, operator, vec![Operand::Integer(0)]); + } + let top_index_size = { + let mut index = Vec::new(); + write_index(&[&write_dict(&top)], &mut index); + index.len() + }; + let mut font_dicts: Vec = fonts.iter().map(|fd| fd.dict.clone()).collect(); + for dict in &mut font_dicts { + set( + dict, + PRIVATE, + vec![Operand::Integer(0), Operand::Integer(0)], + ); + } + let fd_array_size = { + let encoded: Vec> = font_dicts.iter().map(write_dict).collect(); + let mut index = Vec::new(); + write_index( + &encoded.iter().map(Vec::as_slice).collect::>(), + &mut index, + ); + index.len() + }; + let start = header.len() + name_index.len(); + let after_top = + |top_index_size: usize| start + top_index_size + string_index.len() + global_index.len(); + let charset_at = after_top(top_index_size); + let fd_select_new = charset_at + charset.len(); + let char_strings_at = fd_select_new + fd_select_bytes.len(); + let fd_array_at = char_strings_at + char_string_index.len(); + let mut private_at = fd_array_at + fd_array_size; + set(&mut top, CHARSET, vec![Operand::Integer(charset_at as i32)]); + set( + &mut top, + FD_SELECT, + vec![Operand::Integer(fd_select_new as i32)], + ); + set( + &mut top, + CHAR_STRINGS, + vec![Operand::Integer(char_strings_at as i32)], + ); + set( + &mut top, + FD_ARRAY, + vec![Operand::Integer(fd_array_at as i32)], + ); + let mut top_index = Vec::new(); + write_index(&[&write_dict(&top)], &mut top_index); + // The five-byte form keeps the size; check rather than assume. + if top_index.len() != top_index_size { + return Err(fail("CFF Top DICT changed size when its offsets were set")); + } + for (dict, (private, local)) in font_dicts.iter_mut().zip(&private_blocks) { + set( + dict, + PRIVATE, + vec![ + Operand::Integer(private.len() as i32), + Operand::Integer(private_at as i32), + ], + ); + private_at += private.len() + local.len(); + } + let encoded: Vec> = font_dicts.iter().map(write_dict).collect(); + let mut fd_array_index = Vec::new(); + write_index( + &encoded.iter().map(Vec::as_slice).collect::>(), + &mut fd_array_index, + ); + let mut out = Vec::with_capacity(private_at); + out.extend_from_slice(&header); + out.extend_from_slice(&name_index); + out.extend_from_slice(&top_index); + out.extend_from_slice(&string_index); + out.extend_from_slice(&global_index); + out.extend_from_slice(&charset); + out.extend_from_slice(&fd_select_bytes); + out.extend_from_slice(&char_string_index); + out.extend_from_slice(&fd_array_index); + for (private, local) in &private_blocks { + out.extend_from_slice(private); + out.extend_from_slice(local); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn indexes_round_trip() { + let mut out = Vec::new(); + write_index(&[b"ab", b"", b"cde"], &mut out); + let index = read_index(&out, 0).expect("reads"); + let items: Vec<&[u8]> = index + .items + .iter() + .map(|range| &out[range.clone()]) + .collect(); + assert_eq!(items, vec![&b"ab"[..], b"", b"cde"]); + assert_eq!(index.end, out.len()); + } + + #[test] + fn dicts_round_trip_with_wide_integers() { + let dict: Dict = vec![ + (CHAR_STRINGS, vec![Operand::Integer(123_456)]), + ( + ROS, + vec![ + Operand::Integer(391), + Operand::Integer(392), + Operand::Integer(6), + ], + ), + ( + PRIVATE, + vec![Operand::Integer(-5), Operand::Integer(70_000)], + ), + ]; + assert_eq!(read_dict(&write_dict(&dict)).expect("reads"), dict); + } + + /// The CFF table of the Noto Sans CJK fixture. + fn fixture_cff() -> Vec { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/fonts/NotoSansCJKjp-Subset.otf" + ); + let data = std::fs::read(path).expect("the fixture font"); + let font = crate::io::font::Font::parse(std::sync::Arc::new(data), 0).expect("parses"); + font.table(b"CFF ").expect("a CFF table").to_vec() + } + + #[test] + fn a_subset_keeps_used_glyphs_whole_and_blanks_the_rest() { + let cff = fixture_cff(); + assert!(is_cid_keyed(&cff)); + let kept: BTreeSet = [3, 5].into_iter().collect(); + let cut_cff = subset(&cff, &kept).expect("subsets"); + let read = |data: &[u8]| -> (Vec>, usize) { + let header_size = usize::from(data[2]); + let names = read_index(data, header_size).unwrap(); + let tops = read_index(data, names.end).unwrap(); + let top = read_dict(&data[tops.items[0].clone()]).unwrap(); + let at = get(&top, CHAR_STRINGS).unwrap()[0].integer().unwrap() as usize; + let index = read_index(data, at).unwrap(); + let strings = index + .items + .iter() + .map(|range| data[range.clone()].to_vec()) + .collect(); + let strings_index = read_index(data, tops.end).unwrap(); + let global = read_index(data, strings_index.end).unwrap(); + (strings, global.items.len()) + }; + let (original, original_globals) = read(&cff); + let (cut, cut_globals) = read(&cut_cff); + assert_eq!(cut.len(), original.len(), "glyph ids keep their places"); + assert_eq!( + cut_globals, original_globals, + "subroutine numbers keep theirs" + ); + for glyph in 0..original.len() { + if glyph == 0 || kept.contains(&(glyph as u16)) { + assert_eq!(cut[glyph], original[glyph], "glyph {glyph} kept whole"); + } else { + assert_eq!(cut[glyph], vec![14], "glyph {glyph} blanked"); + } + } + assert!(cut_cff.len() < cff.len()); + // The subset is itself a CID-keyed font that subsets again. + assert!(is_cid_keyed(&cut_cff)); + subset(&cut_cff, &kept).expect("subsets again"); + } + + #[test] + fn biases_follow_the_subroutine_count() { + assert_eq!(bias(100), 107); + assert_eq!(bias(2000), 1131); + assert_eq!(bias(40_000), 32768); + } +} diff --git a/src/io/font/mod.rs b/src/io/font/mod.rs index 1edf9a62..57c3da01 100644 --- a/src/io/font/mod.rs +++ b/src/io/font/mod.rs @@ -5,9 +5,11 @@ //! outlines (glyf), a subset holding only the glyphs used. //! //! A collection (`.ttc`) holds several faces; each is opened by index. -//! Fonts with CFF outlines (`OTTO`) are read for their metrics and -//! coverage but not subset. +//! Fonts with CFF outlines (`OTTO`) are subset when they are CID-keyed +//! (every CJK font is; `cff`); name-keyed ones are read for their metrics +//! and coverage only. +pub mod cff; #[cfg(not(target_arch = "wasm32"))] pub mod system; @@ -191,6 +193,8 @@ pub struct Coverage { map: CharMap, /// Whether the face has TrueType outlines (and so can be subset). pub glyf: bool, + /// Whether the face has CID-keyed CFF outlines (and so can be subset). + pub cid_cff: bool, } impl Coverage { @@ -211,6 +215,12 @@ impl Coverage { Ok(Coverage { map: CharMap::from_cmap(&cmap)?, glyf: tables.iter().any(|(tag, _, _)| tag == b"glyf"), + // The CFF table's first kilobytes hold its Top DICT. + cid_cff: tables + .iter() + .find(|(tag, _, _)| tag == b"CFF ") + .and_then(|(_, offset, length)| read_at(*offset, (*length).min(8192))) + .is_some_and(|prefix| cff::is_cid_keyed(&prefix)), }) } @@ -332,6 +342,29 @@ impl Font { self.table(b"glyf").is_some() && self.table(b"loca").is_some() } + /// Whether the face has CID-keyed CFF outlines (and so can be subset). + pub fn has_cid_cff(&self) -> bool { + self.table(b"CFF ").is_some_and(cff::is_cid_keyed) + } + + /// Whether the face can be embedded in a PDF as a subset. + pub fn embeddable(&self) -> bool { + self.has_glyf() || self.has_cid_cff() + } + + /// The CFF table holding only `glyphs`, each glyph's CID its id. + pub fn cff_subset(&self, glyphs: &BTreeSet) -> Result, FontError> { + let table = self + .table(b"CFF ") + .ok_or_else(|| fail("the font has no CFF table"))?; + cff::subset(table, glyphs) + } + + /// A CID-keyed font's Registry, Ordering, and Supplement. + pub fn cff_ordering(&self) -> Option<(String, String, i32)> { + cff::registry_ordering(self.table(b"CFF ")?) + } + /// The glyph for a character; `None` when the font lacks it. pub fn glyph(&self, char: char) -> Option { self.map.glyph(char) diff --git a/src/io/font/system.rs b/src/io/font/system.rs index 08f76333..7ad0927e 100644 --- a/src/io/font/system.rs +++ b/src/io/font/system.rs @@ -35,6 +35,22 @@ const SYMBOLS: &[&str] = &[ ]; /// Fonts that cover whole large scripts (CJK), tried last of the known. +/// Japanese fonts, preferred for Japanese text over the broad fallbacks: +/// the Mac's Hiragino (whose files are named in Japanese), Noto Sans CJK +/// and Source Han Sans, and Windows' Yu Gothic, Meiryo, and MS Gothic. +const JAPANESE: &[&str] = &[ + "ヒラギノ角ゴシック w3", + "hiraginosans-w3", + "notosanscjk-regular", + "notosanscjkjp-regular", + "notosansjp-regular", + "sourcehansans-regular", + "sourcehansansjp-regular", + "yugothr", + "meiryo", + "msgothic", +]; + const FALLBACKS: &[&str] = &[ "arialuni", "arial unicode", @@ -54,6 +70,14 @@ fn rank(path: &Path) -> (usize, usize) { if let Some(index) = BROAD.iter().position(|name| *name == stem) { return (0, index); } + if let Some(index) = JAPANESE.iter().position(|name| *name == stem) { + return (1, index); + } + // Hiragino Kaku Gothic W3 by its file's Japanese name, matched on the + // parts no Unicode normalization changes (ギ may be stored decomposed). + if stem.starts_with("ヒラ") && stem.contains('角') && stem.ends_with("w3") { + return (1, 0); + } if let Some(index) = SYMBOLS.iter().position(|name| *name == stem) { return (2, index); } @@ -65,7 +89,7 @@ fn rank(path: &Path) -> (usize, usize) { || stem.starts_with("droidsans")) && (stem.ends_with("-regular") || !stem.contains('-')); if script_sans { - return (1, 0); + return (1, JAPANESE.len()); } (4, 0) } @@ -166,13 +190,13 @@ impl SystemFonts { } /// A loaded font that covers `char`, the first found in preference - /// order; `None` when no installed font with TrueType outlines does. + /// order; `None` when no installed font that can be embedded does. pub fn font_for(&mut self, char: char) -> Option<(PathBuf, Arc)> { let count = self.files().len(); for index in 0..count { if index == self.probed.len() { let path = self.files()[index].clone(); - let coverage = probe(&path).filter(|coverage| coverage.glyf); + let coverage = probe(&path).filter(|coverage| coverage.glyf || coverage.cid_cff); self.probed.push(coverage); } let covers = self.probed[index] diff --git a/src/io/pdf/compose.rs b/src/io/pdf/compose.rs index 542a0fb6..6bdfea3f 100644 --- a/src/io/pdf/compose.rs +++ b/src/io/pdf/compose.rs @@ -1126,9 +1126,51 @@ fn win_ansi(text: &str) -> (Vec, usize) { (bytes, missing) } +/// Characters a Japanese line may not start with (gyōtō kinsoku): closing +/// brackets and punctuation, small kana, the long-vowel mark, and the +/// iteration marks. +const NO_LINE_START: &str = "、。，．,.・：；？！!?)]}ー‐–〜～…‥」』）〕］｝〉》】〙〗｣’”ゝゞヽヾ々ぁぃぅぇぉっゃゅょゎゕゖァィゥェォッャュョヮヵヶ゛゜｡､ｧｨｩｪｫｬｭｮｯｰ"; + +/// Characters a Japanese line may not end with (gyōmatsu kinsoku): +/// opening brackets and quotes. +const NO_LINE_END: &str = "「『（〔［｛〈《【〘〖｢‘“([{"; + +/// A word split where Japanese allows a line break: between two +/// characters either of which is Japanese (or full width), unless the +/// second may not start a line or the first may not end one. Latin runs +/// stay whole. A word with no Japanese is one segment. +fn japanese_segments(word: &str) -> Vec<&str> { + if word.is_ascii() { + return if word.is_empty() { + Vec::new() + } else { + vec![word] + }; + } + let mut segments = Vec::new(); + let mut start = 0; + let mut previous: Option = None; + for (at, char) in word.char_indices() { + if let Some(before) = previous { + let japanese = crate::io::pdf::text::is_cjk_char(before) + || crate::io::pdf::text::is_cjk_char(char); + if japanese && !NO_LINE_START.contains(char) && !NO_LINE_END.contains(before) { + segments.push(&word[start..at]); + start = at; + } + } + previous = Some(char); + } + if start < word.len() { + segments.push(&word[start..]); + } + segments +} + /// Breaks runs into lines at most `width` wide, first fit: words go on a -/// line while they fit; a word wider than a line is split by characters; -/// a `"\n"` run ends the line. +/// line while they fit (Japanese breaking between characters by its +/// kinsoku rules); a word wider than a line is split by characters; a +/// `"\n"` run ends the line. fn break_lines(runs: &[Run], width: f64, size: f64, faces: &mut Faces) -> Vec { // Words: pieces of one or more styles with no space between them, // each with the space that follows it. @@ -1168,8 +1210,24 @@ fn break_lines(runs: &[Run], width: f64, size: f64, faces: &mut Faces) -> Vec
  • PdfDocument<'a> { let number = self.embedded[index].number; let invalid = |error: FontError| io::Error::new(io::ErrorKind::InvalidData, error.0); let glyphs: BTreeSet = used.keys().copied().collect(); - let program = font.subset(&glyphs).map_err(invalid)?; + // TrueType outlines go in as FontFile2 under a CIDFontType2; CFF + // outlines (CID-keyed, every glyph's CID its id) as FontFile3 + // under a CIDFontType0. + let truetype = font.has_glyf(); + let program = if truetype { + font.subset(&glyphs).map_err(invalid)? + } else { + font.cff_subset(&glyphs).map_err(invalid)? + }; let name = subset_name(&font.family, &glyphs); let mut compressed = vec![0x78, 0x9c]; self.deflate(&program, &mut compressed); @@ -392,11 +400,18 @@ impl<'a> PdfDocument<'a> { let file = self.allocate(); self.object( file, - &format!( - "<< /Length {} /Length1 {} /Filter /FlateDecode >>", - compressed.len(), - program.len() - ), + &if truetype { + format!( + "<< /Length {} /Length1 {} /Filter /FlateDecode >>", + compressed.len(), + program.len() + ) + } else { + format!( + "<< /Length {} /Subtype /CIDFontType0C /Filter /FlateDecode >>", + compressed.len() + ) + }, Some(&compressed), )?; let scale = |units: i16| number_text(font.to_thousandths(f64::from(units))); @@ -411,7 +426,7 @@ impl<'a> PdfDocument<'a> { self.object( descriptor, &format!( - "<< /Type /FontDescriptor /FontName /{name} /Flags {flags} /FontBBox [{} {} {} {}] /ItalicAngle {} /Ascent {} /Descent {} /CapHeight {} /StemV {} /FontFile2 {file} 0 R >>", + "<< /Type /FontDescriptor /FontName /{name} /Flags {flags} /FontBBox [{} {} {} {}] /ItalicAngle {} /Ascent {} /Descent {} /CapHeight {} /StemV {} /{} {file} 0 R >>", scale(font.bbox[0]), scale(font.bbox[1]), scale(font.bbox[2]), @@ -420,7 +435,8 @@ impl<'a> PdfDocument<'a> { scale(font.ascent), scale(font.descent), scale(font.cap_height), - if font.weight >= 600 { 140 } else { 80 } + if font.weight >= 600 { 140 } else { 80 }, + if truetype { "FontFile2" } else { "FontFile3" } ), None, )?; @@ -433,7 +449,9 @@ impl<'a> PdfDocument<'a> { self.object( cid_font, &format!( - "<< /Type /Font /Subtype /CIDFontType2 /BaseFont /{name} /CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> /FontDescriptor {descriptor} 0 R /CIDToGIDMap /Identity /DW 1000 /W [{widths}] >>" + "<< /Type /Font /Subtype /{} /BaseFont /{name} /CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> /FontDescriptor {descriptor} 0 R {}/DW 1000 /W [{widths}] >>", + if truetype { "CIDFontType2" } else { "CIDFontType0" }, + if truetype { "/CIDToGIDMap /Identity " } else { "" } ), None, )?; diff --git a/src/io/rtf/reader.rs b/src/io/rtf/reader.rs index a0091cf9..e6a7b79e 100644 --- a/src/io/rtf/reader.rs +++ b/src/io/rtf/reader.rs @@ -677,10 +677,15 @@ impl<'a> Reader<'a> { let mut text = String::with_capacity(bytes.len()); for &byte in bytes { if let Some(lead) = self.pending_lead.take() { - // A double-byte character the single-byte tables cannot - // decode; Word writes `\uN` for these, so this is rare. - let _ = (lead, byte); - text.push('\u{FFFD}'); + // A double-byte character: Japanese (code page 932, + // Shift_JIS) is decoded; the Chinese and Korean pages have + // no tables here (Word writes `\uN` for them). + let character = if code_page == 932 { + crate::io::cjk::shift_jis_char(&[lead, byte]).0 + } else { + None + }; + text.push(character.unwrap_or('\u{FFFD}')); continue; } if byte < 0x80 { @@ -699,6 +704,15 @@ impl<'a> Reader<'a> { self.pending_lead = Some(byte); continue; } + // Code page 932's single bytes above 0x80 are half-width katakana. + if code_page == 932 { + text.push( + crate::io::cjk::shift_jis_char(&[byte]) + .0 + .unwrap_or('\u{FFFD}'), + ); + continue; + } let character = match codepage::upper_half(code_page).or_else(|| codepage::upper_half(1252)) { Some(table) => char::from_u32(u32::from(table[usize::from(byte - 0x80)])) diff --git a/tests/fixtures/fonts/NotoSansCJKjp-Subset.otf b/tests/fixtures/fonts/NotoSansCJKjp-Subset.otf new file mode 100644 index 0000000000000000000000000000000000000000..b923af7cdd01b783ae60a5be81ab5f3b5c98b488 GIT binary patch literal 9640 zcmb_?2Ut_tw(w5i6cf!@=4e77i3o~H2^PSHASguv1q4JyL+_z0h>UG$CKSOgHY}(J zHpGT)>m%#WOjBS$Aa~D~si*jH*6UG_onnjt&o(GW;8l9m@ zkMAV!P9bFB4dmT3wK=)(O3)~f-y~l?PjAa074Qds$R;1?^(DQ4zpWjNBM|~x_8(`55Ao(8$Wa#e4I!Af9I2n|XzgVdai=xz2# z@gGf=;!I1XFQpiP#%`xyiW&UfRrp?Bv9xT-7MG^B9Lw>QT-r(*l)UQf_YMcqgZY z3l}qsq;IG_ByZQh{13 z|Cy@ydi^W@FO_%jPR>Y-(d6d*^{h{PMtpX1>|alUR@%gDO=i-+**sm7q512bGqbgs z+U(q9ZH6Wd^%V%rr~z{OiT;7XVUcqe#3!dRVp@#euRBl#C$c!Q~YmxuTMdhjG@fEB%uw+(AN=0f#)~Zw;E9w4%6B8#| zOkANk&|Qh(4CZAp)y)wC-;j-KH7%th7OlWON|>ND}bvK*P`n zWLUTDR5~^{s?h%vb`~CFY$I-rB%p1eLGzf*rxT(}%i@*T1G{1t568nS@Ibd`WRT-@ zjsHx=xp3l^OTZ+Ay$v@pr5DR7`shb%{(%=7cIv&!Spe&oo|^KRh{a>cogZVy~RaVE0S2M!5!tk9G?^(zZYuHIoWE)j1c_ za>??cq9eAVZFZvBDv`CQ)}I<88Z46N@K?p$FwtZzAgaKiWX&99)ejWTaAMvbWbV_o zpHWrxf|D1s+T}$Lxx~l~tx-yRm^pRkl!Hh$T!eRpa2=Nd?mNHqx+L=tzBlXZu{S2l zBsdJ0acE;QsH zNxYWbJXo|R-c}TMJ*!3b_BLtd>c&z#t6HmfRN?{arOCv?gDkQomS3bu|A{Ac;DrbA z5Vq>rk;(@7jn+anK}BTHND&_GB=T80smMz))XH$vz;X_CaZ$76A@dGiyLtNP*&|Jh zqBkjxY4`xAcc*Z_I_$n0+v{{grK=a_Rb|WV$9vcj!IN2E9cZkpZ&1dnokEB#{kESJ zFS&aQkHB(^_N8t42NdOa9JPVQLtmDceWuiPQFF;hIwFx5(Yl+u_4>p5wK6rWTTzvv z%hLNv$?$SA^HXA0&PpD44;>-G;TBjtM)WBWEVs4l9!!*6MBd(*#_%0;WoH`l+v2a~ zHp#G{m5a>Xr)i9Ah?Je(ncEh7HK$1?`CV*?=8oTrY9aJZul=%^z}x8<1)QHJBlYA2 zr;DaSwe#I3#qU3PqW!@AmgbUW`xQ6weQpx1_pY|98pp0qOQ}qj6Kc#XC%YHXSg6E( z&)j^XtZ2QNdOrGCyzE#972Y=W^w@nXv75A_Wz|0YQI;&l*SOKORQc>BGm1jkvbgy2 zM7d^xzB)l!?Ln0vYOOdZZ``ZX?^VttyayBFr~R9zE+b+#g$F5q86&4a%o_4!alq zK>Cr97Q~bEwa;CdP?!KeOQ52IKcwXC&TeLQcTLoY!Xb;$++jX&tE-$ zgJZ5L_PpBi%lf~u5?tF|&5d5|A#zJ0A!L{Zk=GO9R&T}KYN}#w#fCK&73(0GZdh~r z;4ck#WA;l6V{!r)N3$zWm$jamRf zx&5EF9QYJWec$jU=S=Gh^_hhhq%Y%8P6dTkRtGCRn9|E<%Z7@^*@!%RU7fqau!R!K z4?X@;xs5rK(H`3#)plDNe$wAjq&o@U-K=h_erx4c5L?hag{yJhTDiG$i(b}J{j}~( z9XrZyII$r?7GxY|Mq_8Z_W@psYw{I`Cb#*Y4?cTOO6;(70d~*FeJ18fcX(1p=B5kz zloZNs9Z%uGScC1c_l*mJ8_Q+K52WO$XZrnN9pUzZ)^XyPWQP~y+plyL@B$D!5=f#Me3_8hNWSGT5a?VO|1;*x^W zMdfUnh74L6CSRDL(`PBG#y9BqRBqSFPE=w^^$~qqK>V1bi7c5@LM#ad@#6YfSl8J@ z^=DcXhdw)vMTfurb@|^DzlusF$V){=`Q8eZV%yG#uaDw^hqIb9_hudZTAI2eX7}9P zYcJK@sJh1*Gl&hBG%4OOF^bK~U%Wh5o~mtV`}*m(c<5TC1b^GTlUq3XK|W5O|GHSV z<>BaBlH59Ww=9qNbFOPYwaz993)Ql&f|^Zrn>TKjeRUV3b}W_u(iJ>`SP@4xxpP?A zFpe^?X?PUwgMBP;FWc{k1L;kK@vb8kYj14E!#8w*YacdorP-O~8S+V?drvhtHy(e` z9z3yDNnUq{a+Pk|DtG9?27n7=5ukT6mxjM^Lu!L91r}P z=kRgSvnHe|Uw`8JCFv)33=Ev+_H-s3-Xtvj6_3SoBL0btkDgnyc%JgnO~AA5hw3+O zU0=U`-X-a>vL$8dH%vDN$rwsvI%IDgmoTcf-vhbY1oM|cSrJ}I? z{_+L0igV(X2aY#g+;I*Is&L=hA2;c`qz;9A1})imvi5Mz-f^p?a;Sv!oNID@sKXVp$}T_HGZ~DwtH>M zhFz5@XdqwL1c!KEbnhoo>*VKa7n!0yQ5=E(P z$q3@1x*aaX<`19${l*IxzoZwAiy+8<%+%8!r~D4h4wc@BeGuLm?mU8g>_jGocLq!r z9ZfmpDT>{v+4ta=^SIZp?;wU6>NfZtk*+Mv8z$P;vZlWNz~)_WuTSk>#JMs0olUx? zi}?6>QDV6^lpJ3*OP-ouS*cY*FjevMdm2~mmE&`CnZr`o`BU9Qw~ur=h_n#IFTUjx z$|Fls6+h6WP37nJSybSs*enTq`eMOM*@0)2^*IvUNu)afCupJ>N9@R~QHn>iC_EpFy08n*w7~WBzM~c8o0QL(^2PDxDRQy}ThMze z+bg#!?C9!+RgvkC(#a_%DWOuI53*eErny-LDRZ9bihdl|nYnK3=G~h%6Q_3K`4ch6 zy;v{W&4x@MX2gB^hQ(VlYYV?>njc8VpJjMW@@ zbN{d*GwuO?LpbKh_14aP*F&~CSQ#()8rIQS^U^{yqAM$;+qBd@yJ8NG%ov(g<{)UvAeDczb)x7o-; zg{71+Ynovs9TOiB8=1ixU2!)@mUdyc?y1D>Azo_i&tTz1h6ou6RCP7@4QD7N2aKgO z#^$&I;0DHcAOFa`{qo^WLut^H?uQ`Lg6=0ANiZrWn& zBzHwFOLhd{rL-p4J%T6_HWqEosw>13&wSl^>&mX^X=j1)z94YyDIT2^4 zDpoFCR=T7-uRNh*UKP7OzcwS+;%M0sUDN7@%KEh%>e(ab*J2sAw!nRs;a(wFplCk2 z|JcqpC{mW;cFwTP%ealk;pT>I-T+L&{W)?8pYk)R0}N`0^m)YKJ_hwQqxu#;2i`Sy z;dE|W`qr4WsVvzP@Vmt|QcrJb+~xL%`o+ZY5OTs(&q)8rf;49G|m@-!eL0VzS0Kp+}mY3pcE^&VPn+26ar zVXz7C>u<0&0fGGuLwI1owEhO0-a~6G2WIyFz3AiLxDS`=0Z1$m^y|< zThVUTVVFn^-e|tUhz$#dS&7$d-L_}bMlpG42l%V$_vf}Gwq27ZHhZ)MiE+QY5~_-h(PT`l$4n{Hm@rs$@?292rqeC+1`j;VfLv#@xDaWqZ>F#io$X z?JIV!*uA1r_N?;@Y<3Y_v#&DG_)NAS1BtmK@mr)g=1bwCgRgO4Y-53ip5KvvO$uEV zRlTWhcg+UY$PiDifbm!}euK9nF=B3EoIHNfwxeHm{rJNLrG(QE^T6?bGuWk1Hug)qK=N6r)WoybmSp1kC&@-t@8(BNp$Z8}?UJY*#*IN^?>w^23Zrq1Z{& z>MOXAol)W~x zhX+$$jmCZfU&zE<$R4V!aA7(4XgRHK(jU;Z=o<9Rm0~X9FjcZRt1KmfTw4_@PY5C)1P zG2G^E_)4*zc-3J!svzEUe2iYqsT)7d zTAZ}3Kqh8!6Rj#H74c#zo&-O@zI*mksBOdjwuE+mr}=i zz>l2%N+KXXbGQ+HiDK?vjM~nVM@p&&!lRnL*!(xB2`7pTeTuo)ori8;QjQdn8`tB^sh7L6PaxIwJ^|ayf zj<01^bc4%|3A^2z2_=<8T&BXK!qTS7iE1=FRt&(?UttO>uAkDz99Fhpp-yV19vdYm zfka7$kY(c7YgBsj{G2)RVLnfBKiseLs~=mN7isq?Ph6o6#7FM+20EmVHSrVgIxhe9 z$OO`dSon>1R!WB74tc5A)y2)$G+cw{z5;kdAQaGLL6eK-e>Cx*$js~&V9*x_^-n(Z z#)0IOIFsoMV>2Y=u>ib6al(-(0-7B(E8zloxln`Rg~e#Ra5S1AOhHkCKhPLhHwKLm zR)Fi%kq26Ug5be#1j>QYG31L5qf9g(;5Zb8<_r9g2A)1UL68hX2aysDg&cX^|1fZw)^nz$KN-!0$Vwn2y5wOe$ z5FwwSARmyaBw!MdY3F-v<-oqk+$jtpc zEJp6;zVG4QXpDKud$^Bqt@#HzC1?yY6W+t9;C+!AIfFMGOwcHRRdCOWLs}Gr;sLfp zNg#PHcw{_S-3cu;eI3CkAJ#a+TOBQU%Lxrfjwle|Txcp30&6nBrz&`F;E$$5?+>q> zf`Mux%0p=|n+-f2fxjB~PXPZ0!Pf_%4>Ued=-HnFGC2Ip{s|zfH|&~%rlK&QoDKYv zfqMpU`!nq%lT^H%87Lp-oK1Wkp?8KjngG&kP%7-_Wlw;95wso(V_~HW8VgO0M*lA) z`=Emg$-wjD<;esO z=Ya1fz|2e-^S)7mRUP5#3!@B>o2NSi=5k?ghKaHd?2U&nuLDoh3na{j*);ejgY-T0 zdOXNS3XkVk@v@|wsPe0NDDqwb8=X6NCkvB zb4(n03?K15(bPjb2l(?EXuzWUJZ~>=(3iK-bklAX*oT*o_b$&T6K1t21xB$zi}!yv zKuKU5o-&W){lj~_z~liH(B%25OcWAKcIMZ9h-O|xEyOM_mBu8C>OGG*;LFqNS=nRv z09ca)Ef*Re54%7ET8&Z(&|?UjGn)czgcu?W!_ms6YE8$K;Dxpw6F%SfI+Ez@F2qfwR-&3*KMZ zAO~;t580oWkYCAro0pEC<l@!4~pGGMkRv-D&cUWPoN#p}_NpLuQmZ(enT zHGjTS=77Y!XEOnt1GeF1OM{-bTq4A95X9~bs1~4@Fx3nQel$Zq znZtd@3&pC1>OSfz>KW=e>O^&lTC2`hf1)l|uTrm8?^mBtpH*L0UsYcZ9U1CAH$1*H z{zam_);eco;j@Y@Rm0!C18UCTWsO>(?)@h!89=4@e@^Ayop+zT+kL0k9n^8N%zB8tTLE|Zzxwc8Snx+2Sda0ZHX{_-#};I#A@{`W_*CScq{&D`_Ha!`_Nnkk z3}m0C$<09SrfQD&hdENf$s3B+zsHyXrZ?&fo+&|scgK*xYzp*cfE|d;lVcNNpyejR zUp}(6=^6h|97p^&j#99hBXB%vvYi0VVNVSzLY5$Y0qDik&8)t7nAlb|3c$E9kWbRU5`*L=w1R>=mlOVRNy7778W6E!DPsW)=<4~ zfy}W3PMRLZ%b=z&hO9FWH9^MG!F@0VtT`Q;9h`yTaOJ<#p`r&_`y#=cKYH^u{5zMo QbD@TRUt{wEm_CC409PfZ@Bjb+ literal 0 HcmV?d00001 diff --git a/tests/fixtures/fonts/OFL-NotoSansCJK.txt b/tests/fixtures/fonts/OFL-NotoSansCJK.txt new file mode 100644 index 00000000..d952d62c --- /dev/null +++ b/tests/fixtures/fonts/OFL-NotoSansCJK.txt @@ -0,0 +1,92 @@ +This Font Software is licensed under the SIL Open Font License, +Version 1.1. + +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font +creation efforts of academic and linguistic communities, and to +provide a free and open framework in which fonts may be shared and +improved in partnership with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply to +any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software +components as distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, +deleting, or substituting -- in part or in whole -- any of the +components of the Original Version, by changing formats or by porting +the Font Software to a new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, +modify, redistribute, and sell modified and unmodified copies of the +Font Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, in +Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the +corresponding Copyright Holder. This restriction only applies to the +primary font name as presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created using +the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/tests/fixtures/fonts/README.md b/tests/fixtures/fonts/README.md index d6b620dc..ac2f0a91 100644 --- a/tests/fixtures/fonts/README.md +++ b/tests/fixtures/fonts/README.md @@ -6,3 +6,12 @@ under the Apache License, Version 2.0 (https://www.apache.org/licenses/LICENSE-2.0). Used unchanged as a fixture for the TrueType reader, subsetting, and PDF font embedding; it covers Latin, Greek, and Cyrillic. + +`NotoSansCJKjp-Subset.otf`: a subset of Noto Sans CJK JP Regular +(Copyright 2014-2021 Adobe, with Reserved Font Name 'Source'), from +github.com/notofonts/noto-cjk, licensed under the SIL Open Font License +1.1 (`OFL-NotoSansCJK.txt`). Cut to 44 glyphs with fontTools' +`pyftsubset` (the text `日本語の文章です。ひらがな、カタカナ、漢字を含みます。「テスト」吾輩は猫であるABCabc123`, +no layout features, no hinting); it stays CID-keyed CFF with five font +DICTs and global subroutines. The fixture for CFF subsetting and PDF +embedding of the fonts CJK systems ship. diff --git a/tests/fixtures/japanese-input/ansicpg932.rtf b/tests/fixtures/japanese-input/ansicpg932.rtf new file mode 100644 index 00000000..f8d3e316 --- /dev/null +++ b/tests/fixtures/japanese-input/ansicpg932.rtf @@ -0,0 +1 @@ +{\rtf1\ansi\ansicpg932\deff0{\fonttbl{\f0\fnil\fcharset128 MS Mincho;}}\f0 \'93\'fa\'96\'7b\'8c\'ea\'82\'cc\'95\'b6\'8f\'cd\'82\'c5\'82\'b7\'81\'42\'82\'d0\'82\'e7\'82\'aa\'82\'c8\'81\'41\'83\'4a\'83\'5e\'83\'4a\'83\'69\'81\'41\'8a\'bf\'8e\'9a\'82\'f0\'8a\'dc\'82\'dd\'82\'dc\'82\'b7\'81\'42\'94\'bc\'8a\'70\'b6\'c0\'b6\'c5\'82\'c6English\'81\'49\par} \ No newline at end of file diff --git a/tests/fixtures/japanese-input/ansicpg932.rtf.expected b/tests/fixtures/japanese-input/ansicpg932.rtf.expected new file mode 100644 index 00000000..440d6fe6 --- /dev/null +++ b/tests/fixtures/japanese-input/ansicpg932.rtf.expected @@ -0,0 +1 @@ +日本語の文章です。ひらがな、カタカナ、漢字を含みます。半角ｶﾀｶﾅとEnglish！ \ No newline at end of file diff --git a/tests/fixtures/japanese-input/euc-jp.csv b/tests/fixtures/japanese-input/euc-jp.csv new file mode 100644 index 00000000..e9bd537b --- /dev/null +++ b/tests/fixtures/japanese-input/euc-jp.csv @@ -0,0 +1,3 @@ +̾, +, +,300 diff --git a/tests/fixtures/japanese-input/euc-jp.csv.expected b/tests/fixtures/japanese-input/euc-jp.csv.expected new file mode 100644 index 00000000..fd942947 --- /dev/null +++ b/tests/fixtures/japanese-input/euc-jp.csv.expected @@ -0,0 +1,3 @@ +名前,値段 +寿司,１２００円 +ｶﾀｶﾅ,300 diff --git a/tests/fixtures/japanese-input/euc-jp.txt b/tests/fixtures/japanese-input/euc-jp.txt new file mode 100644 index 00000000..4fbd81ed --- /dev/null +++ b/tests/fixtures/japanese-input/euc-jp.txt @@ -0,0 +1 @@ +ܸʸϤǤҤ餬ʡʡޤߤޤȾюŤEnglish diff --git a/tests/fixtures/japanese-input/euc-jp.txt.expected b/tests/fixtures/japanese-input/euc-jp.txt.expected new file mode 100644 index 00000000..440d6fe6 --- /dev/null +++ b/tests/fixtures/japanese-input/euc-jp.txt.expected @@ -0,0 +1 @@ +日本語の文章です。ひらがな、カタカナ、漢字を含みます。半角ｶﾀｶﾅとEnglish！ \ No newline at end of file diff --git a/tests/fixtures/japanese-input/fcharset128.rtf b/tests/fixtures/japanese-input/fcharset128.rtf new file mode 100644 index 00000000..54e84530 --- /dev/null +++ b/tests/fixtures/japanese-input/fcharset128.rtf @@ -0,0 +1 @@ +{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\froman\fcharset0 Times;}{\f1\fnil\fcharset128 MS Mincho;}}\f0 English then {\f1 \'93\'fa\'96\'7b\'8c\'ea\'82\'cc\'95\'b6\'8f\'cd\'82\'c5\'82\'b7\'81\'42\'82\'d0\'82\'e7\'82\'aa\'82\'c8\'81\'41\'83\'4a\'83\'5e\'83\'4a\'83\'69\'81\'41\'8a\'bf\'8e\'9a\'82\'f0\'8a\'dc\'82\'dd\'82\'dc\'82\'b7\'81\'42\'94\'bc\'8a\'70\'b6\'c0\'b6\'c5\'82\'c6English\'81\'49}\par} \ No newline at end of file diff --git a/tests/fixtures/japanese-input/fcharset128.rtf.expected b/tests/fixtures/japanese-input/fcharset128.rtf.expected new file mode 100644 index 00000000..100ee72c --- /dev/null +++ b/tests/fixtures/japanese-input/fcharset128.rtf.expected @@ -0,0 +1 @@ +English then 日本語の文章です。ひらがな、カタカナ、漢字を含みます。半角ｶﾀｶﾅとEnglish！ \ No newline at end of file diff --git a/tests/fixtures/japanese-input/make.py b/tests/fixtures/japanese-input/make.py new file mode 100644 index 00000000..ccb52ff8 --- /dev/null +++ b/tests/fixtures/japanese-input/make.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Writes Japanese inputs in the legacy encodings and what each reads as. + +Plain text in Shift_JIS and EUC-JP, HTML declaring Shift_JIS, CSV in +Shift_JIS (as Excel saves a Japanese CSV) and EUC-JP, XML declaring +Shift_JIS, and RTF as Word writes Japanese: Shift_JIS byte pairs under +\\ansicpg932, and in a \\fcharset128 font inside a Western document. +Python's own codecs make the bytes. Run from the repository root: + + python3 tests/fixtures/japanese-input/make.py +""" +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) +TEXT = "日本語の文章です。ひらがな、カタカナ、漢字を含みます。半角ｶﾀｶﾅとEnglish！" +ROWS = [["名前", "値段"], ["寿司", "１２００円"], ["ｶﾀｶﾅ", "300"]] + + +def write(name, data, expected): + open(os.path.join(HERE, name), "wb").write(data) + open(os.path.join(HERE, name + ".expected"), "w", encoding="utf-8").write(expected) + + +def rtf_bytes(text): + out = "" + for ch in text: + encoded = ch.encode("cp932") + if len(encoded) == 1 and encoded[0] < 0x80: + out += ch + else: + out += "".join(f"\\'{byte:02x}" for byte in encoded) + return out + + +write("shift-jis.txt", (TEXT + "\n").encode("cp932"), TEXT) +write("euc-jp.txt", (TEXT + "\n").encode("euc_jp"), TEXT) +html = f'

    {TEXT}

    \n' +write("shift-jis.html", html.encode("cp932"), TEXT) +csv = "".join(",".join(row) + "\n" for row in ROWS) +write("shift-jis.csv", csv.encode("cp932"), csv) +write("euc-jp.csv", csv.encode("euc_jp"), csv) +xml = '\n寿司１２００円\n' +write("shift-jis.xml", xml.encode("cp932"), '{"item":{"name":"寿司","price":"１２００円"}}') +rtf = "{\\rtf1\\ansi\\ansicpg932\\deff0{\\fonttbl{\\f0\\fnil\\fcharset128 MS Mincho;}}\\f0 " + rtf_bytes(TEXT) + "\\par}" +write("ansicpg932.rtf", rtf.encode("latin-1"), TEXT) +rtf = ("{\\rtf1\\ansi\\ansicpg1252\\deff0{\\fonttbl{\\f0\\froman\\fcharset0 Times;}{\\f1\\fnil\\fcharset128 MS Mincho;}}" + "\\f0 English then {\\f1 " + rtf_bytes(TEXT) + "}\\par}") +write("fcharset128.rtf", rtf.encode("latin-1"), "English then " + TEXT) diff --git a/tests/fixtures/japanese-input/shift-jis.csv b/tests/fixtures/japanese-input/shift-jis.csv new file mode 100644 index 00000000..59f218f0 --- /dev/null +++ b/tests/fixtures/japanese-input/shift-jis.csv @@ -0,0 +1,3 @@ +O,li +i,PQOO~ +,300 diff --git a/tests/fixtures/japanese-input/shift-jis.csv.expected b/tests/fixtures/japanese-input/shift-jis.csv.expected new file mode 100644 index 00000000..fd942947 --- /dev/null +++ b/tests/fixtures/japanese-input/shift-jis.csv.expected @@ -0,0 +1,3 @@ +名前,値段 +寿司,１２００円 +ｶﾀｶﾅ,300 diff --git a/tests/fixtures/japanese-input/shift-jis.html b/tests/fixtures/japanese-input/shift-jis.html new file mode 100644 index 00000000..c4ef8113 --- /dev/null +++ b/tests/fixtures/japanese-input/shift-jis.html @@ -0,0 +1 @@ +

    {͂̕łBЂ炪ȁAJ^JiA܂݂܂BpłEnglishI

    diff --git a/tests/fixtures/japanese-input/shift-jis.html.expected b/tests/fixtures/japanese-input/shift-jis.html.expected new file mode 100644 index 00000000..440d6fe6 --- /dev/null +++ b/tests/fixtures/japanese-input/shift-jis.html.expected @@ -0,0 +1 @@ +日本語の文章です。ひらがな、カタカナ、漢字を含みます。半角ｶﾀｶﾅとEnglish！ \ No newline at end of file diff --git a/tests/fixtures/japanese-input/shift-jis.txt b/tests/fixtures/japanese-input/shift-jis.txt new file mode 100644 index 00000000..03149fa0 --- /dev/null +++ b/tests/fixtures/japanese-input/shift-jis.txt @@ -0,0 +1 @@ +{͂̕łBЂ炪ȁAJ^JiA܂݂܂BpłEnglishI diff --git a/tests/fixtures/japanese-input/shift-jis.txt.expected b/tests/fixtures/japanese-input/shift-jis.txt.expected new file mode 100644 index 00000000..440d6fe6 --- /dev/null +++ b/tests/fixtures/japanese-input/shift-jis.txt.expected @@ -0,0 +1 @@ +日本語の文章です。ひらがな、カタカナ、漢字を含みます。半角ｶﾀｶﾅとEnglish！ \ No newline at end of file diff --git a/tests/fixtures/japanese-input/shift-jis.xml b/tests/fixtures/japanese-input/shift-jis.xml new file mode 100644 index 00000000..c98d4aa4 --- /dev/null +++ b/tests/fixtures/japanese-input/shift-jis.xml @@ -0,0 +1,2 @@ + +iPQOO~ diff --git a/tests/fixtures/japanese-input/shift-jis.xml.expected b/tests/fixtures/japanese-input/shift-jis.xml.expected new file mode 100644 index 00000000..e780ad5f --- /dev/null +++ b/tests/fixtures/japanese-input/shift-jis.xml.expected @@ -0,0 +1 @@ +{"item":{"name":"寿司","price":"１２００円"}} \ No newline at end of file diff --git a/tests/japanese_input.rs b/tests/japanese_input.rs new file mode 100644 index 00000000..efd5915f --- /dev/null +++ b/tests/japanese_input.rs @@ -0,0 +1,64 @@ +//! Japanese in the legacy encodings: every input in +//! tests/fixtures/japanese-input (from `make.py` there) converts, and its +//! Japanese reads as written. Text, HTML, and RTF go to text; CSV to CSV +//! (as UTF-8); XML to JSON. + +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +fn dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/japanese-input") +} + +fn convert(input: &PathBuf, to: &str) -> String { + let output = Command::new(env!("CARGO_BIN_EXE_sublime")) + .args(["-q", "convert"]) + .arg(input) + .args(["--to", to]) + .output() + .expect("runs"); + assert!( + output + .status + .code() + .is_some_and(|code| code == 0 || code == 2), + "{}: {}", + input.display(), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("UTF-8 out") +} + +fn squeezed(text: &str) -> String { + text.chars().filter(|ch| !ch.is_whitespace()).collect() +} + +#[test] +fn every_japanese_input_reads_as_written() { + let mut paths: Vec = fs::read_dir(dir()) + .expect("fixture dir") + .map(|entry| entry.expect("entry").path()) + .filter(|path| { + let extension = path.extension().and_then(|extension| extension.to_str()); + matches!(extension, Some("txt" | "html" | "csv" | "xml" | "rtf")) + }) + .collect(); + paths.sort(); + assert_eq!(paths.len(), 8, "the eight inputs make.py writes"); + for path in paths { + let to = match path.extension().and_then(|extension| extension.to_str()) { + Some("csv") => "tsv", + Some("xml") => "json", + Some("txt") => "markdown", + _ => "text", + }; + let mut read = convert(&path, to); + if to == "tsv" { + read = read.replace('\t', ","); + } + let expected_path = PathBuf::from(format!("{}.expected", path.display())); + let expected = fs::read_to_string(expected_path).expect("expected beside the input"); + assert_eq!(squeezed(&read), squeezed(&expected), "{}", path.display()); + } +} diff --git a/tests/to_pdf.rs b/tests/to_pdf.rs index 8f841ee5..e776b30c 100644 --- a/tests/to_pdf.rs +++ b/tests/to_pdf.rs @@ -152,6 +152,36 @@ fn a_given_font_is_subset_and_embedded() { ); } +/// A CID-keyed CFF font (as every CJK font is: Noto Sans CJK here) given +/// for the body is subset and embedded as a CIDFontType0 with its CFF +/// program, and Japanese set in it reads back as written. +#[test] +fn a_cid_keyed_cff_font_is_subset_and_embedded() { + let font = fixture("tests/fixtures/fonts/NotoSansCJKjp-Subset.otf"); + let options = ConvertOptions { + font: Some(std::sync::Arc::new(font)), + ..ConvertOptions::default() + }; + let text = "日本語の文章です。ひらがな、カタカナ、漢字を含みます。\n"; + let pdf = convert_with(&MARKDOWN_TO_PDF, text.as_bytes(), options); + let raw = String::from_utf8_lossy(&pdf); + assert!(raw.contains("/Subtype /CIDFontType0 ")); + assert!(raw.contains("/Subtype /CIDFontType0C")); + assert!(raw.contains("/FontFile3")); + assert!(raw.contains("+NotoSansCJKJP")); + let mut out = Vec::new(); + write_pdf_text(&pdf, None, &mut out).expect("reads"); + let read: String = String::from_utf8(out) + .unwrap() + .chars() + .filter(|ch| !ch.is_whitespace()) + .collect(); + assert_eq!( + read, + "日本語の文章です。ひらがな、カタカナ、漢字を含みます。" + ); +} + /// A Pages document's pictures are set on the page at the size the /// document shows each (one shown at twice the size takes twice the /// room); a Markdown image by path, from the input's folder, and by a From 1c9e844049606338fe2d3fa32767df9e5a69ed4c Mon Sep 17 00:00:00 2001 From: Chimi6 <43803253+Chimi6@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:27:46 -0600 Subject: [PATCH 2/2] docs: language and script support --- DOCS/CHANGELOG.md | 1 + DOCS/FORMATS.md | 2 +- DOCS/LANGUAGES.md | 35 +++++++++++++++++++++++++++++++++++ README.md | 3 ++- src/cli/commands/paths.rs | 2 +- 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 DOCS/LANGUAGES.md diff --git a/DOCS/CHANGELOG.md b/DOCS/CHANGELOG.md index daa1baef..416c0308 100644 --- a/DOCS/CHANGELOG.md +++ b/DOCS/CHANGELOG.md @@ -11,6 +11,7 @@ section under a version heading. - Japanese in the legacy encodings reads everywhere. Plain text, Markdown, and HTML in Shift_JIS, EUC-JP, or ISO-2022-JP (an HTML `charset` or XML `encoding` is followed; otherwise the encoding is detected), CSV and TSV in Shift_JIS (as Excel saves a Japanese CSV) or EUC-JP, decoded as they stream, and XML declaring Shift_JIS. Detection takes bytes for Japanese only when they read as Japanese does (common kana and kanji), so a Western file is still refused with an error rather than turned into kanji. Word's RTF with Japanese as Shift_JIS byte pairs (`\ansicpg932`, `\fcharset128`) reads in full; it read as replacement characters before. - PDFs embed the fonts CJK systems ship: CID-keyed CFF OpenType (Hiragino on the Mac, Noto Sans CJK and Source Han Sans on Linux), subset to the glyphs and subroutines used (a Japanese page with Noto Sans CJK is 264 KB from a 16 MB font). Japanese text prefers a Japanese font (Hiragino, Noto Sans CJK, Yu Gothic, Meiryo, MS Gothic) over the broad fallbacks such as Arial Unicode. `--font` takes a CID-keyed CFF font too. - Japanese in PDFs breaks lines between characters with the kinsoku rules: no line starts with closing brackets, punctuation, small kana, or the long-vowel mark, and none ends with an opening bracket. +- `DOCS/LANGUAGES.md`: which scripts each path handles (text and documents, legacy encodings, PDF to text, text to PDF) and what is not yet supported, linked from the README and `FORMATS.md`. ## [1.0.1] - 2026-10-30 diff --git a/DOCS/FORMATS.md b/DOCS/FORMATS.md index 353aff05..f3b49836 100644 --- a/DOCS/FORMATS.md +++ b/DOCS/FORMATS.md @@ -2,7 +2,7 @@ Generated by `sublime paths --markdown`. Do not edit by hand. CI fails if this file is stale. -Every converter and every conversion path, with what each loses, are in [CONVERSIONS.md](CONVERSIONS.md). +Every converter and every conversion path, with what each loses, are in [CONVERSIONS.md](CONVERSIONS.md); which languages and scripts each handles is in [LANGUAGES.md](LANGUAGES.md). ## Map diff --git a/DOCS/LANGUAGES.md b/DOCS/LANGUAGES.md new file mode 100644 index 00000000..85cc456b --- /dev/null +++ b/DOCS/LANGUAGES.md @@ -0,0 +1,35 @@ +# Language support + +Text is Unicode throughout, so any language in a UTF-8 or UTF-16 file, or +in a format that stores Unicode (Word, Pages, Numbers, Excel, ODF, EPUB, +HTML, XML, JSON, YAML, TOML), converts as written. What differs by +language is reading older encodings, reading text back out of a PDF, and +setting it into a new PDF. + +| Script | Text and documents | Legacy encodings read | PDF to text | Text to PDF | +|---|:-:|---|:-:|---| +| Latin, Greek, Cyrillic | ✓ | RTF code pages (437, 850, 1250-1258, Mac Roman) | ✓ | ✓ | +| Japanese | ✓ | Shift_JIS, EUC-JP, ISO-2022-JP in text, Markdown, HTML, XML, CSV, TSV; RTF code page 932 | ✓ incl. vertical writing and fonts with no ToUnicode map | ✓ Japanese line breaking (kinsoku), horizontal only | +| Chinese | ✓ | RTF through `\uN` only | ✓ when the PDF carries a ToUnicode map or a Unicode CMap | ✓ may be set in a Japanese font | +| Korean | ✓ | RTF through `\uN` only | ✓ when the PDF carries a ToUnicode map or a Unicode CMap | ✓ | +| Hebrew, Arabic | ✓ | RTF code pages 1255, 1256 | in drawing order | left to right, isolated letter forms | +| Hindi and other Indic, Thai | ✓ | RTF code page 874 (Thai) | ✓ | glyphs unshaped (no conjuncts); Thai breaks only where a line overflows | + +✓ means read back as written in the tests. Text to PDF uses a font +installed on the machine that has the characters (or `--font`); a +character no installed font has is `?` and reported. + +## Not yet + +- Chinese and Korean legacy encodings (GBK, GB18030, Big5, EUC-KR) in + text and CSV files: rejected with an error, not misread. +- Chinese and Korean PDFs whose fonts have no ToUnicode map and a + non-Unicode CMap (the Adobe-GB1, -CNS1, -Korea1 tables). +- Western code page detection in text and CSV files (a Windows-1252 CSV + is rejected; save it as UTF-8). +- Bidirectional reordering and complex-script shaping in PDF output. +- Vertical Japanese and bold Japanese in PDF output (bold is set in the + regular weight). + +Per-format detail is in [formats/](formats/): `pdf.md`, `csv.md`, +`html.md`, `rtf.md`. diff --git a/README.md b/README.md index 8418c186..8ee2d367 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Universal efficient file conversion. One binary, zero runtime dependencies. Apple Pages and Numbers, Word, Excel, Markdown, HTML, RTF, PDF, plain text, CSV, JSON, and images today; the formats and a map are in `DOCS/FORMATS.md`, and every path with its fidelity in -`DOCS/CONVERSIONS.md`. +`DOCS/CONVERSIONS.md`, and language and script support in +`DOCS/LANGUAGES.md`. sublime convert input.csv output.json sublime check csv json diff --git a/src/cli/commands/paths.rs b/src/cli/commands/paths.rs index 52f20790..1577e21a 100644 --- a/src/cli/commands/paths.rs +++ b/src/cli/commands/paths.rs @@ -111,7 +111,7 @@ pub fn render_markdown() -> String { let mut text = String::new(); text.push_str("# Formats\n\n"); text.push_str("Generated by `sublime paths --markdown`. Do not edit by hand. CI fails if this file is stale.\n\n"); - text.push_str("Every converter and every conversion path, with what each loses, are in [CONVERSIONS.md](CONVERSIONS.md).\n\n"); + text.push_str("Every converter and every conversion path, with what each loses, are in [CONVERSIONS.md](CONVERSIONS.md); which languages and scripts each handles is in [LANGUAGES.md](LANGUAGES.md).\n\n"); text.push_str("## Map\n\n"); text.push_str(&format!("One graph per category and one for the crossings between them. A line joins two formats with a direct converter between them, in either direction. The grid under each graph gives every pair (rows from, columns to): {} lossless, {} conditional, {} lossy, with the number of steps (1 is a direct converter), blank for no path. [CONVERSIONS.md](CONVERSIONS.md) has each path and what it loses.\n\n", kind_mark(FidelityKind::Lossless), kind_mark(FidelityKind::Conditional), kind_mark(FidelityKind::Lossy)));