diff --git a/README.md b/README.md index 8c68de3..ca2a062 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ const result = await extractAttachment({ // result.truncated === false ``` -The heavy parsers (`unpdf`, `mammoth`, `exceljs`, …) are lazy-loaded per handler, so importing +The heavy parsers (`unpdf`, `exceljs`, `saxes`, …) are lazy-loaded per handler, so importing `extractAttachment` costs nothing until you actually call it on a matching attachment. It's also available on its own subpath — `import { extractAttachment } from 'agentextract/attachment'` — if you want to reach it without touching the body-extraction entry point. @@ -90,15 +90,41 @@ guards reduce blast radius; they are **not** a sandbox. or ZIP64 metadata is treated as over-budget (fail-closed), not trusted. - **Output** — extracted text is capped at `MAX_OUTPUT_CHARS` (250k), or lower via `maxOutputChars`. Cutting sets `truncated` on the result, so a partial extraction is never mistaken for a complete - one. The PDF and `.xlsx` handlers apply the cap **incrementally** as they build — and stop reading - the document once they reach it — so neither ever materializes in full. The `.docx` (mammoth) and - HTML (html-to-text) handlers return a complete string that is then trimmed, so for those the cap is + one. The PDF, `.docx` and `.xlsx` handlers apply the cap **incrementally** as they build — and stop + reading the document once they reach it — so none of them ever materializes in full. Only the HTML + handler (html-to-text) returns a complete string that is then trimmed, so for that one the cap is **post-materialization** and peak memory follows the whole document. - **Timeout** — `HANDLER_TIMEOUT_MS` (10 s) stops *awaiting* a slow async parse. It cannot cancel synchronous CPU already running inside a parser, so handlers that yield between units of work (PDF - per page, `.xlsx` per row) also check the deadline themselves and stop; the others cannot. + per page, `.docx` per inflate chunk, `.xlsx` per row) also check the deadline themselves and return + what they have with `truncated` set. The HTML, `.doc` and text handlers cannot. - **PDF** — page count and accumulated output are bounded (`MAX_PDF_PAGES`, `MAX_OUTPUT_CHARS`), but pdf.js's internal per-page decompression is **not** bounded in-library (no hook exists). +- **`.docx`** — `word/document.xml` is located in the archive's central directory, inflated on its + own, and read with a streaming SAX parser (`saxes`) rather than loaded into a DOM. Peak memory + therefore tracks one inflate chunk plus the text kept so far, independent of document size. + Measured at a 1024 MB heap on a 45 MB `document.xml` inside a 3.65 MB archive, against the previous + DOM-based reader: + + | concurrency | before | after | + |---|---|---| + | 1 | 812 ms / 607 MB | **68 ms / 175 MB** | + | 2 | 1552 ms / 970 MB | **59 ms / 164 MB** | + | 4 | 3512 ms / 1217 MB | **58 ms / 168 MB** | + + The old reader built 43.4M characters and kept 250k. The sharpest case is not the large archive: a + 0.41 MB attachment holding one 43 MB `` peaked the old reader at 1270 MB, against 199 MB here. + Two residual bounds, stated rather than glossed: peak is not independent of *input* size (the API + takes a `Buffer`), and `saxes` buffers one text node whole, so a single enormous run still costs + about twice its own size. Malformed XML is also stricter than before — a document the old reader + silently half-read now comes back either `truncated` or `failed`. +- **`.docx` scope** — text comes from `word/document.xml` only. **Footnote, endnote and comment + bodies are not extracted** (they are separate zip parts), and neither are headers or footers. + **Table structure is not preserved**: each cell's paragraphs are emitted in reading order with the + same blank-line separator as body paragraphs, so a 2×2 table is indistinguishable from four + consecutive paragraphs. List bullets and numbers are dropped; the item text remains. All of this + matches the previous reader exactly — it is a documented limit, not a regression — but a consumer + reading an invoice or a contract should know the column a figure sat in is gone. - **`.xlsx`** — read row-by-row through `exceljs`'s streaming reader rather than loaded whole, so peak memory tracks the shared-string table plus one row instead of a live object per cell (measured: 294 MB → 171 MB, and 3.7x faster, on a 5 MB / 38 MB-uncompressed workbook). It is not diff --git a/attachment.ts b/attachment.ts index e24e56f..07242fe 100644 --- a/attachment.ts +++ b/attachment.ts @@ -168,7 +168,7 @@ const decodeText = (content: Buffer, hint?: string): string => { // HANDLER REGISTRY // Handlers lazy-load their parsers: a Lambda that only ever sees text never pays to load pdf.js -// or mammoth. +// or saxes. // Text — txt, csv, calendar, vcard, json, xml, yaml const textHandler: Handler = { @@ -284,19 +284,69 @@ const pdfHandler: Handler = { }, } -// DOCX — modern OOXML Word +// DOCX — modern OOXML Word. Streams word/document.xml; see DOCX STREAMING READER for the machinery, +// and for why the output contract is mammoth's, reproduced rather than invented. const docxHandler: Handler = { kind: 'docx', contentTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'], extensions: ['.docx'], - extract: async ({ content }) => { - const { default: mammoth } = await import('mammoth') - const { value } = await mammoth.extractRawText({ buffer: content }) - return { text: value } + extract: async ({ content, maxOutputChars, deadline }) => { + // Fail closed, as the xlsx handler does on !rewritten. Both bails are unreachable rather + // than defensive: an archive whose central directory can't be walked never gets past + // checkDecompressionBudget to reach this (Invariant 2 makes budget-ok imply zipEntries-ok + // over the same records), and routing pins the second — ooxmlKind only answers 'docx' for an + // archive whose directory NAMES word/document.xml. Costs nothing, and keeps neither of those + // proofs load-bearing. + const entries = zipEntries(content) + if (!entries) throw new Error('docx central directory could not be read for streaming') + const part = entries.find((entry) => entry.name === DOCX_MAIN_PART) + if (!part) throw new Error(`docx archive has no ${DOCX_MAIN_PART} entry`) + + const reader = await createDocxReader() + // Decode ACROSS inflate chunks, not per chunk: a 16 KB boundary lands mid-sequence in any + // document with a non-ASCII character, and chunk.toString('utf8') would turn that one + // character into two U+FFFD. StringDecoder carries the partial bytes forward. (saxes handles + // a surrogate pair split across write() calls itself; this is the layer below that.) + const { StringDecoder } = await import('node:string_decoder') + const decoder = new StringDecoder('utf8') + + let truncated = false + try { + for await (const chunk of docxMainPartChunks(part)) { + // Both guards here, ahead of the work, at one inflate chunk of granularity. Finer + // than the pdf per-page and xlsx per-row checks, and the only place a stop is + // possible: saxes has no abort, so the way to stop parsing is to stop feeding it. + // Per CHUNK rather than per emitted character, for the same reason the xlsx deadline + // sits above its empty-row skip — tens of MB of w:pPr/w:rPr markup produces no text + // at all, so a cap-only check would never fire on precisely the cheapest loop to + // spin. Breaking a `for await` destroys the inflate stream, so the rest of the + // document is never decompressed either. + if (Date.now() > deadline || reader.chars() > maxOutputChars) { + truncated = true + break + } + reader.write(decoder.write(chunk)) + } + // Only a read that ran to the end may assert the document ended cleanly. + if (!truncated) reader.end(decoder.end()) + } catch (error) { + // saxes is conformant where mammoth's DOM parser recovered, so a document the old reader + // read to the end can stop short here. Text already extracted is still text, and the + // contract has a word for "the document continues past this point" — so keep it and say + // so. Nothing read means nothing to label: that stays a failure, which a caller can see + // and retry. What this deliberately does NOT do is install a saxes error handler and + // parse on; measured, that emits close-tag text as content and descends into elements + // mammoth drops — silent wrong output, the one outcome this file fails over everywhere else. + if (reader.text().trim().length === 0) throw error + truncated = true + } + + // Emptiness is left to the entry point: an empty document is exactly '\n\n', which trims to ''. + return { text: reader.text(), truncated } }, } -// DOC — legacy Word 97–2003. mammoth only reads the modern .docx zip, so the OLE binary needs +// DOC — legacy Word 97–2003. The .docx reader above is an OOXML zip reader, so the OLE binary needs // word-extractor instead. const docHandler: Handler = { kind: 'doc', @@ -506,17 +556,32 @@ const ooxmlKind = (content: Buffer): HandlerKind | undefined => { // declared size is attacker-controlled. So measure it: stream-inflate each entry (peak stays ~one // zlib chunk), count REAL bytes, abort once the total crosses the cap. Unmeasurable = fail closed. // -// WHICH READER THIS BINDS — the two formats no longer share one: +// WHICH READER THIS BINDS — since the .docx handler stopped using mammoth, one answer for both +// formats: OUR OWN zipEntries walk. Neither handler hands the original bytes to a third-party zip +// reader any more, so there is no foreign directory choice left to second-guess. // -// .docx -> mammoth -> jszip reads the CENTRAL DIRECTORY, so measuring it measures what jszip will -// inflate. The invariants below pin both walks to the same records; their reasoning is this case. +// .docx -> zipEntries locates word/document.xml and the handler inflates exactly that region of +// exactly this buffer. Nothing else in the archive is decompressed at read time. // // .xlsx -> exceljs -> unzipper dispatches on LOCAL file-header signatures front-to-back -// (unzipper/lib/parse.js:51), only skipping PAST directory records — so the invariants do not -// bind it, and a local entry the directory omits would still be inflated unmeasured. -// reorderForStreaming binds it instead: it re-emits one local header per measured entry and -// nothing else, so both walks see the same set by construction. Hence the xlsx handler FAILS -// when the rewrite can't be produced; the original bytes would drop the budget on the floor. +// (unzipper/lib/parse.js:51), only skipping PAST directory records — so a central-directory +// measurement would not bind it at all. reorderForStreaming binds it instead: it re-emits one +// local header per measured entry and nothing else, so both walks see the same set by +// construction. Hence the xlsx handler FAILS when the rewrite can't be produced; the original +// bytes would drop the budget on the floor. +// +// So the two invariants below no longer exist to keep a foreign reader honest. Invariant 2 has +// instead become the proof that this measurement and zipEntries can never disagree — now +// load-bearing for BOTH formats. Invariant 1 is a structural check that no longer prevents anything. +// See each. +// +// STALE, AND KNOWN TO BE: this measures EVERY entry, while the .docx handler now inflates one. On +// real documents word/document.xml is 2–28% of the archive (a Word template measured 66 KB of 3.1 MB), +// so a document with ≥50 MB of compressible non-document parts — EMF/WMF vector art, embedded OLE +// objects — is declined over bytes that would never be read. Narrow: ordinary media is PNG/JPEG, +// already compressed, so 50 MB of it needs >10 MB of archive and MAX_INPUT_BYTES turns it away +// first. Left alone deliberately — loosening a zip-bomb gate is a permissiveness change that belongs +// with the input caps, not smuggled in behind a parser swap. const EOCD_MAGIC = Buffer.from([0x50, 0x4b, 0x05, 0x06]) // End-of-Central-Directory const CD_SIG = 0x02014b50 // Central-Directory file header @@ -555,11 +620,12 @@ const inflateCounting = (comp: Buffer, runningTotal: number, cap: number): Promi inflate.end(comp) }) -// Pick the EOCD the way jszip (inside mammoth, and so the .docx reader) does: the LAST signature in -// the buffer, no comment-length check. Matching its choice is the point — measuring a different -// directory than the parser reads is a bomb-bypass. An invariant here would diverge: jszip -// follows a second EOCD planted after the real one, so rejecting that leaves the parser inflating a -// directory we never measured. Bonus: last-match doesn't false-skip zips with bytes after the EOCD. +// The LAST signature in the buffer, no comment-length check. Chosen originally to match jszip, which +// read .docx through mammoth: measuring a different directory than the parser reads is a bomb-bypass, +// and jszip follows a second EOCD planted after the real one. jszip is out of the read path now, so +// the choice is purely internal — this function is shared by BOTH walks (zipEntries and the budget), +// so any deterministic pick is self-consistent. Kept as last-match because it still has to agree with +// unzipper on the .xlsx path, and because it doesn't false-skip zips with bytes after the EOCD. const findEocd = (buf: Buffer): number => { const eocd = buf.lastIndexOf(EOCD_MAGIC) return eocd >= 0 && eocd + 22 <= buf.length ? eocd : -1 // need room for the 22-byte fixed record @@ -580,10 +646,12 @@ interface ZipEntry { // two callers do with that differs, because they ask for different things: // ooxmlKind — identification, so it degrades to a raw byte scan. // reorderForStreaming — the entry set unzipper will be given, so it refuses and its caller fails. +// docxHandler — the entry it will inflate, so it refuses and the handler fails. // -// That second caller makes this walk load-bearing for the .xlsx zip-bomb guard, not merely a lenient -// identifier: the archive it emits is what makes the budget's measurement bind (see DECOMPRESSION -// BUDGET). Still deliberately NOT merged with checkDecompressionBudget — that one is the measurement +// Those last two make this walk load-bearing for the zip-bomb guard on BOTH formats, not merely a +// lenient identifier: on .xlsx the archive it emits is what makes the budget's measurement bind, and +// on .docx it is what picks the one region the handler decompresses (see DECOMPRESSION BUDGET). +// Still deliberately NOT merged with checkDecompressionBudget — that one is the measurement // itself, and its two invariants must reject archives this one accepts. Merging would force a single // contract onto both. What keeps the split safe is the one-way relation pinned at Invariant 2: this // walk can never be the more permissive of the two. @@ -623,10 +691,10 @@ const zipEntries = (buf: Buffer): ZipEntry[] | undefined => { const zipEntryNames = (buf: Buffer): string[] | undefined => zipEntries(buf)?.map((entry) => entry.name) // Measure a zip's ACTUAL decompressed size, capped, from each entry's real (structural, not -// self-declared) compressed region. The two invariants below pin us to the records jszip will read -// — the .docx path; the .xlsx path is bound differently, see the section header. Both assert the -// FILE is self-consistent rather than mirroring jszip, so neither rots if it changes. Every real -// archive satisfies them (73 measured, 0 failures). +// self-declared) compressed region. The two invariants below were written to pin us to the records +// jszip would read on the old .docx path; jszip is gone, and what they now do is stated at each. +// Both assert the FILE is self-consistent rather than mirroring any particular reader, which is why +// neither rotted when the reader changed. Every real archive satisfies them (73 measured, 0 failures). const checkDecompressionBudget = async (buf: Buffer, cap: number): Promise => { const eocd = findEocd(buf) if (eocd < 0) return corrupt('malformed zip: no end-of-central-directory record') @@ -640,9 +708,13 @@ const checkDecompressionBudget = async (buf: Buffer, cap: number): Promise raw text) +// +// Reads word/document.xml straight out of the archive with a SAX parser, so the largest live object +// is one inflate chunk plus the text kept so far — never a DOM. mammoth, which this replaces, built +// an xmldom tree AND a document model on top of it, then let the central cap throw almost all of it +// away. Measured through dist/ at a 1024 MB heap on a 45 MB document.xml inside a 3.65 MB archive — +// every gate cleared, nothing skipped: +// +// mammoth this +// concurrency 1 812 ms / 607 MB 68 ms / 175 MB +// concurrency 2 1552 ms / 970 MB 59 ms / 164 MB +// concurrency 4 3512 ms / 1217 MB 58 ms / 168 MB +// +// mammoth built 43,420,044 characters every time and kept 250,000 — 0.6%. Its peak tracks the +// document and multiplies by concurrency; this one is flat, because the cap now stops the READ +// rather than trimming the result. The sharpest case isn't even the big archive: a 0.41 MB +// attachment holding one 43 MB w:t peaked mammoth at 1270 MB, versus 199 MB here. +// +// Same failure mode as the .xlsx reader, one format over, and reachable from half a megabyte. +// +// THE OUTPUT CONTRACT is mammoth's extractRawText, reproduced deliberately rather than invented +// (raw-text.js is 13 lines): w:t text verbatim, w:tab -> '\t', a paragraph's children then TWO +// newlines, everything else its children only. Nothing is trimmed, so a document ends with '\n\n'. +// +// THE SHAPE is a WHITELIST, mirroring body-reader.js:46-57 — an element mammoth has no handler for +// has its ENTIRE SUBTREE dropped rather than recursed into, and bare text outside a w:t is dropped +// too. So the tables below are what we KEEP; anything absent disappears with its children. That is +// the load-bearing decision: "emit every w:t, newline on " over-extracts on any document with +// a text box or a field. It also DELETES work rather than adding it — +// - mammoth's 20-entry ignore list (w:pPr, w:rPr, w:sectPr, w:proofErr, w:tblGrid, ...) has no +// counterpart here. It exists only to suppress a warning; for raw text "deliberately ignored" +// and "unrecognised" produce the identical empty result, and the default already IS that result. +// - every emit-nothing leaf mammoth spells out (w:br, w:cr, w:fldChar, w:instrText, +// w:footnoteReference, w:bookmarkStart) is likewise just the default. +// - w:sdt keeping only w:sdtContent, and mc:AlternateContent keeping only mc:Fallback, fall out +// for free: w:sdtPr and mc:Choice simply aren't on the list. +// +// Fidelity is pinned by tests/docx-fidelity.test.ts against mammoth itself over a real Word corpus. +// The deliberate divergences are listed at DOCX_CONTAINERS. + +const DOCX_MAIN_PART = 'word/document.xml' + +// Namespace URI -> the prefix mammoth's element names carry (office-xml-reader.js:10-36). Matching a +// literal `w:` prefix instead would be a shippable bug three times over: the ISO-strict format binds +// w to a different URI (mammoth's own corpus carries strict-format.docx), a producer may bind +// wordprocessingml as the DEFAULT namespace — giving //

with no prefix at all — +// and a producer may pick any prefix it likes. Only these four URIs are ever matched; mammoth's +// other eleven mappings feed its HTML converter and its ignore list, neither of which survives here. +const OOXML_PREFIXES: Record = { + 'http://schemas.openxmlformats.org/wordprocessingml/2006/main': 'w', // transitional + 'http://purl.oclc.org/ooxml/wordprocessingml/main': 'w', // ISO strict + 'http://schemas.openxmlformats.org/markup-compatibility/2006': 'mc', + 'urn:schemas-microsoft-com:vml': 'v', +} + +// Any prefix that reaches saxes undeclared resolves HERE, and this URI is deliberately absent from +// OOXML_PREFIXES — so the element names `{urn:agentextract:unbound}local`, misses the whitelist, and +// its subtree is dropped. Which is precisely what mammoth does with an unmapped namespace +// (xml/reader.js:53-66 emits the same `{uri}local` shape, and no handler matches it). +// +// It exists because saxes fails the whole parse on an unbound prefix — for ATTRIBUTES too +// (saxes.js:1920-1925) — where xmldom shrugged. That is not a rare shape: Word 2013 and later stamp +// w15:paraId on EVERY w:p, and w16cid:durableId alongside it, so a document that carries those +// without their xmlns (a fragment assembled by a templating tool, a repaired file) used to lose +// everything after its first paragraph. An allowlist cannot close that — there is always another +// prefix — so this resolves the open class instead, and the map below shrinks to the only entries +// that do something an allowlist has to do. +const UNBOUND_NAMESPACE = 'urn:agentextract:unbound' + +// The three prefixes where GUESSING is better than dropping, because these are the only URIs +// OOXML_PREFIXES maps: an undeclared `w:` resolved to the sentinel above would take the whole +// document body with it. Every other prefix a real document leaves undeclared — o:, w10:, wne:, r:, +// wp:, a:, pic:, wps:, w14: — resolves to a URI this reader does not map either way, so guessing it +// and sentinelling it are the same outcome, and the sentinel needs no maintenance. +// An in-document xmlns still shadows these (saxes.resolve checks scope first, then this map, then +// the sentinel), so a strict-format file resolves to the strict URI and routes correctly. +const OOXML_ASSUMED_PREFIXES: Record = { + w: 'http://schemas.openxmlformats.org/wordprocessingml/2006/main', + mc: 'http://schemas.openxmlformats.org/markup-compatibility/2006', + v: 'urn:schemas-microsoft-com:vml', +} + +// Recurse into these; emit nothing of their own. Absent from the list = subtree dropped. +const DOCX_CONTAINERS = new Set([ + 'w:document', + 'w:body', + 'w:r', + 'w:hyperlink', + 'w:ins', + 'w:smartTag', + 'w:tbl', + 'w:tr', + 'w:tc', + 'w:sdt', // w:sdtPr isn't here, which is the whole of mammoth's firstOrEmpty("w:sdtContent") + 'w:sdtContent', + 'mc:AlternateContent', + 'mc:Fallback', // likewise mc:Choice: dropped by omission + 'w:object', + 'w:drawing', + 'w:txbxContent', + 'v:group', + 'v:rect', + 'v:roundrect', + 'v:shape', + 'v:textbox', +]) +// Deliberately NOT here, each matching mammoth: +// wp:inline / wp:anchor — mammoth's readDrawingElement digs for a:blip images and never reads +// arbitrary children, so the DrawingML text-box path (wps:txbx) yields nothing. Text boxes reach +// us through the VML fallback above, which is the branch mc:Fallback selects anyway. +// v:imagedata, v:shapetype, v:shadow — images and shape defs; no text either way. +// w:fldSimple, w:delText, w:ruby — mammoth DROPS these with their children. w:delText is right +// (deleted text does not belong in an extract); w:fldSimple and w:ruby are real text loss in +// mammoth, replicated on purpose so this change stays a pure port. Recovering them is one array +// entry each, deliberately left to its own change once the corpus says how often they matter. +// +// Deviations from mammoth, all four in tests/docx-fidelity.test.ts's ACCEPTED_DIVERGENCES if they +// ever show up on a real document: a w:sdt carrying wordml:checkbox has its first text character +// REPLACED by a checkbox node in mammoth (we keep the character — strictly better, and costs nothing +// here); w:sym needs mammoth's dingbat-to-unicode table to map (we drop it, and it is the only +// element that would make this reader read an attribute at all); a w:t holding a comment or a nested +// element makes mammoth's text() throw "Not implemented" (we extract); and the main part is read at +// its conventional path rather than resolved through _rels/.rels, which routing already requires. + +// Emit a literal, then drop any children — mammoth's handlers for these ignore children entirely. +const DOCX_LITERALS: Record = { + 'w:tab': '\t', + 'w:noBreakHyphen': '\u2011', + 'w:softHyphen': '\u00ad', +} + +// xmldom normalised these to '\n' before mammoth ever saw the markup (dom-parser.js:34-38). saxes +// does the XML-standard \r\n and \r itself — correctly, across chunk boundaries — but leaves NEL and +// LINE SEPARATOR alone. Both are single characters, so matching xmldom on the way out needs no +// cross-chunk state, unlike normalising the raw markup would. +const XML_EXTRA_SEPARATORS = /[\u0085\u2028]/g + +// The two markers that change what an ANCESTOR emits, so they have to be spotted even though +// they sit inside an already-dropped w:pPr / w:trPr subtree. ECMA-376 17.13.5.15 (deleted paragraph +// mark: the mark is gone, so the paragraph merges into the next one) and 17.13.5.12 (deleted table +// row: the whole row goes). +const PARAGRAPH_DELETED = ['w:p', 'w:pPr', 'w:rPr', 'w:del'] +const ROW_DELETED = ['w:tr', 'w:trPr', 'w:del'] + +const endsWith = (stack: string[], path: string[]): boolean => + path.every((name, i) => stack[stack.length - path.length + i] === name) + +// A paragraph's own text, and the text hoisted out of any w:pict inside it. mammoth returns a +// (value, extra) pair from every element it reads: w:pict moves its whole result into `extra` +// (body-reader.js:439 .toExtra()), extras bubble up through every container, and only w:p reinserts +// them — as a SIBLING AFTER the paragraph (:296 .insertExtra()). So VML text-box text lands after +// the paragraph's '\n\n' rather than glued into the middle of it, and an extra that never reaches a +// w:p is silently lost. Two string fields per open w:p / w:pict reproduce all of that. +interface DocxFrame { + value: string + extra: string +} + +interface DocxReader { + write: (chunk: string) => void // one decoded slice of document.xml; THROWS on malformed XML + chars: () => number // characters emitted so far — what the handler's cap check reads + end: (tail: string) => void // flush, close, and run the end-of-document checks + text: () => string // the text, complete or partial +} + +const createDocxReader = async (): Promise => { + // Lazy like every other parser here: a Lambda that only ever sees text never loads saxes. + const { SaxesParser } = await import('saxes') + + const stack: string[] = [] // qualified names of every open element, innermost last + const frames: DocxFrame[] = [{ value: '', extra: '' }] // index 0 is the document; extras reaching it are lost + const deleted: boolean[] = [false] // one flag per open w:p; index 0 pairs with the root frame + let top = frames[frames.length - 1] + let skip = -1 // stack index where the dropped subtree began, or -1 when we're reading + let rowDeleted = false // a w:trPr said its row is deleted; act on it once that w:trPr closes + let sawBody = false + let chars = 0 + + const emit = (text: string): void => { + top.value += text + chars += text.length + } + + const parser = new SaxesParser({ + xmlns: true, + additionalNamespaces: OOXML_ASSUMED_PREFIXES, + // Consulted only after scope and the map above (saxes.js:1845-1862), so this catches exactly + // the prefixes nothing else claimed, and turns "unbound prefix" from a fatal parse error into + // an unmapped namespace — which this reader already knows how to drop. + resolvePrefix: () => UNBOUND_NAMESPACE, + }) + + parser.on('opentag', (tag) => { + // A direct port of mammoth's convertName (xml/reader.js:53-66): mapped URI -> `w:t`, + // unmapped -> `{uri}local`, no namespace -> the bare local name. Only the first form can + // match a table, so the other two exist to guarantee a miss — but they keep the stack + // readable in a debugger, and they are what makes an unmapped namespace fail closed. + const uri = tag.uri ?? '' + const prefix = OOXML_PREFIXES[uri] + const name = uri === '' ? tag.local : prefix === undefined ? `{${uri}}${tag.local}` : `${prefix}:${tag.local}` + stack.push(name) + + if (skip >= 0) { + // Inside a dropped subtree nothing is emitted, and only the two ancestor-affecting + // markers are still looked for. The `skip ===` tests prove the marker really sits in the + // properties element of a LIVE w:p / w:tr — i.e. that the drop began at that w:pPr / + // w:trPr — rather than somewhere deeper in unrelated debris. + if (name === 'w:del') { + if (skip === stack.length - 3 && endsWith(stack, PARAGRAPH_DELETED)) deleted[deleted.length - 1] = true + else if (skip === stack.length - 2 && endsWith(stack, ROW_DELETED)) rowDeleted = true + } + return + } + + // w:t is the ONE text-bearing element; the text handler recognizes it off the stack top, so + // it needs no flag of its own. Deliberately before the container test, so its children — + // illegal anyway — fall through to the drop below exactly as they do in mammoth. + if (name === 'w:t') return + + if (name === 'w:p' || name === 'w:pict') { + top = { value: '', extra: '' } + frames.push(top) + if (name === 'w:p') deleted.push(false) + return + } + if (DOCX_CONTAINERS.has(name)) { + if (name === 'w:body') sawBody = true + return + } + const literal = DOCX_LITERALS[name] + if (literal !== undefined) emit(literal) + // The whitelist default: this element and everything under it is gone. + skip = stack.length - 1 + }) + + parser.on('text', (text) => { + if (skip < 0 && stack[stack.length - 1] === 'w:t') emit(text.replace(XML_EXTRA_SEPARATORS, '\n')) + }) + + parser.on('closetag', () => { + const name = stack.pop() + + if (skip >= 0) { + if (stack.length !== skip) return // still inside the dropped subtree + skip = -1 + // A deleted row's properties have just closed; drop the rest of the row with them. + if (rowDeleted && name === 'w:trPr') { + rowDeleted = false + skip = stack.length - 1 + } + return + } + + if (name !== 'w:p' && name !== 'w:pict') return + const frame = top + frames.pop() + top = frames[frames.length - 1] + if (name === 'w:pict') { + // Hoist: the picture's own text becomes the parent's `extra`, behind any extra it + // already carried. It reaches the output only if some ancestor w:p reinserts it. + top.extra += frame.extra + frame.value + } else if (deleted.pop()) { + // The paragraph mark was deleted, so there is no paragraph break here: this text runs + // straight into the next paragraph. mammoth re-reads the stashed children in the next + // paragraph's context and lands on the same string; suppressing the tail is the + // streaming form of that, and is equivalent because this reader is context-free. + top.value += frame.value + top.extra += frame.extra + } else { + top.value += `${frame.value}\n\n${frame.extra}` + chars += 2 + } + }) + + return { + write: (chunk) => void parser.write(chunk), + chars: () => chars, + end: (tail) => { + parser.write(tail).close() // close() is the well-formedness check: it throws on an unclosed element + // mammoth throws "Could not find the body element: are you sure this is a docx file?" + // here — for a foreign root AND for a w:document with no w:body, both verified. Keeping + // it means a zip whose main part is not WordprocessingML stays a labeled failure instead + // of becoming an 'extracted' with no text. + if (!sawBody) throw new Error('docx main part has no w:body element') + }, + // Frames above the root are only still open on the truncated path; concatenating every + // frame's `value` in order is exactly the partial text in document order, so a document that + // is one enormous paragraph still returns what was read instead of ''. On the complete path + // there is only the root frame and this is just frames[0].value. Pending `extra` is dropped + // either way — mammoth's own behaviour for an extra that never reaches a w:p. + text: () => frames.reduce((text, frame) => text + frame.value, ''), + } +} + +// One shape for both storage methods, so the handler's loop has a single form. Method 0 is its own +// single chunk (bounded by MAX_INPUT_BYTES, and no real producer stores document.xml); method 8 +// yields a chunk at a time under backpressure, which is what keeps peak memory flat. +const docxMainPartChunks = (part: ZipEntry): AsyncIterable | Iterable => { + if (part.method === 0) return [part.data] + const inflate = zlib.createInflateRaw() + inflate.end(part.data) // pushes the compressed bytes; the readable side inflates only on demand + return inflate +} + ///////////////////////////////////////////////////////////// // ROUTING (continued) — claim verification + the final decision @@ -1035,8 +1414,8 @@ export const extractAttachment = async ( return { status: 'skipped', reason: type ? `unsupported type ${type}` : 'unrecognized attachment' } } - // Zip bombs: measure the real decompressed size before exceljs/mammoth touch the bytes. The - // preflight also decides which refusal this is — over budget skips, malformed fails. + // Zip bombs: measure the real decompressed size before either OOXML handler inflates anything. + // The preflight also decides which refusal this is — over budget skips, malformed fails. // (pdf/doc/text aren't zips.) if (kind === 'docx' || kind === 'xlsx') { const check = await checkDecompressionBudget(input.content, MAX_UNCOMPRESSED_BYTES) @@ -1060,9 +1439,10 @@ export const extractAttachment = async ( ) // Central cap, so a pathological document can't dump megabytes into S3 and the search index. // Handlers that build incrementally overshoot by one unit, so for them this is the final - // precise trim; docx and html return a full string, so for those it's POST-materialization — - // peak memory follows the whole document, and hard containment is the host memory limit - // (see README). Don't split a surrogate pair at the boundary: a lone half serializes as U+FFFD. + // precise trim; html alone still returns a full string, so for that one it's + // POST-materialization — peak memory follows the whole document, and hard containment is the + // host memory limit (see README). Don't split a surrogate pair at the boundary: a lone half + // serializes as U+FFFD. const overCap = output.text.length > maxOutputChars const capEnd = overCap && output.text.charCodeAt(maxOutputChars - 1) >= 0xd800 && output.text.charCodeAt(maxOutputChars - 1) <= 0xdbff diff --git a/package-lock.json b/package-lock.json index 2ad0f6f..f61f039 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "html-to-text": "^10.0.0", "iconv-lite": "^0.7.2", "jschardet": "^3.1.4", - "mammoth": "^1.12.0", + "saxes": "^5.0.1", "unpdf": "^1.6.2", "word-extractor": "^1.0.4" }, @@ -21,6 +21,7 @@ "@types/html-to-text": "^9.0.4", "@types/node": "^22.20.0", "@types/word-extractor": "^1.0.6", + "mammoth": "^1.12.0", "typescript": "^6.0.3", "vitest": "^4.1.2" }, @@ -606,6 +607,7 @@ "version": "0.8.13", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", "integrity": "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==", + "dev": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -668,6 +670,7 @@ "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, "license": "MIT", "dependencies": { "sprintf-js": "~1.0.2" @@ -966,6 +969,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz", "integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==", + "dev": true, "license": "BSD-2-Clause" }, "node_modules/dom-serializer": { @@ -1027,6 +1031,7 @@ "version": "0.1.12", "resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz", "integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==", + "dev": true, "license": "BSD", "dependencies": { "underscore": "^1.13.1" @@ -1749,6 +1754,7 @@ "version": "0.4.2", "resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz", "integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "duck": "^0.1.12", @@ -1770,6 +1776,7 @@ "version": "1.12.0", "resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz", "integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "@xmldom/xmldom": "^0.8.6", @@ -1878,6 +1885,7 @@ "version": "0.2.4", "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", "integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==", + "dev": true, "license": "BSD-2-Clause" }, "node_modules/pako": { @@ -2140,6 +2148,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/stackback": { @@ -2283,6 +2292,7 @@ "version": "1.13.8", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.8.tgz", "integrity": "sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==", + "dev": true, "license": "MIT" }, "node_modules/undici-types": { @@ -2545,6 +2555,7 @@ "version": "10.1.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz", "integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==", + "dev": true, "license": "MIT", "engines": { "node": ">=4.0" diff --git a/package.json b/package.json index 796935a..1c68843 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "@types/html-to-text": "^9.0.4", "@types/node": "^22.20.0", "@types/word-extractor": "^1.0.6", + "mammoth": "^1.12.0", "typescript": "^6.0.3", "vitest": "^4.1.2" }, @@ -69,7 +70,7 @@ "html-to-text": "^10.0.0", "iconv-lite": "^0.7.2", "jschardet": "^3.1.4", - "mammoth": "^1.12.0", + "saxes": "^5.0.1", "unpdf": "^1.6.2", "word-extractor": "^1.0.4" } diff --git a/tests/__snapshots__/docx-fidelity.test.ts.snap b/tests/__snapshots__/docx-fidelity.test.ts.snap new file mode 100644 index 0000000..adcb67e --- /dev/null +++ b/tests/__snapshots__/docx-fidelity.test.ts.snap @@ -0,0 +1,115 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`docx fidelity — our reader against mammoth > comments.docx 1`] = ` +"Ouch. + +" +`; + +exports[`docx fidelity — our reader against mammoth > embedded-style-map.docx 1`] = ` +"Walking on imported air + +" +`; + +exports[`docx fidelity — our reader against mammoth > empty.docx 1`] = `null`; + +exports[`docx fidelity — our reader against mammoth > endnotes.docx 1`] = ` +"Ouch. + +" +`; + +exports[`docx fidelity — our reader against mammoth > external-picture.docx 1`] = `null`; + +exports[`docx fidelity — our reader against mammoth > fixtures/empty.docx 1`] = `null`; + +exports[`docx fidelity — our reader against mammoth > fixtures/image.docx 1`] = ` +"Report + +" +`; + +exports[`docx fidelity — our reader against mammoth > fixtures/sample.docx 1`] = ` +"First paragraph of the sample document. + + + +Second paragraph with more words here. + +" +`; + +exports[`docx fidelity — our reader against mammoth > footnote-hyperlink.docx 1`] = `null`; + +exports[`docx fidelity — our reader against mammoth > footnotes.docx 1`] = ` +"Ouch. + +" +`; + +exports[`docx fidelity — our reader against mammoth > simple-list.docx 1`] = ` +"Apple + +Banana + +" +`; + +exports[`docx fidelity — our reader against mammoth > single-paragraph.docx 1`] = ` +"Walking on imported air + +" +`; + +exports[`docx fidelity — our reader against mammoth > strict-format.docx 1`] = ` +"Test + +" +`; + +exports[`docx fidelity — our reader against mammoth > strikethrough.docx 1`] = ` +"Today's Special: Salmon Sold out + +" +`; + +exports[`docx fidelity — our reader against mammoth > tables.docx 1`] = ` +"Above + +Top left + +Top right + +Bottom left + +Bottom right + +Below + +" +`; + +exports[`docx fidelity — our reader against mammoth > text-box.docx 1`] = ` +" + +Datum plane + +" +`; + +exports[`docx fidelity — our reader against mammoth > tiny-picture.docx 1`] = `null`; + +exports[`docx fidelity — our reader against mammoth > tiny-picture-target-base-relative.docx 1`] = `null`; + +exports[`docx fidelity — our reader against mammoth > underline.docx 1`] = ` +"The Sunset Tree + +" +`; + +exports[`docx fidelity — our reader against mammoth > utf8-bom.docx 1`] = ` +"This XML has a byte order mark. + +" +`; diff --git a/tests/docx-fidelity.test.ts b/tests/docx-fidelity.test.ts new file mode 100644 index 0000000..556cd19 --- /dev/null +++ b/tests/docx-fidelity.test.ts @@ -0,0 +1,96 @@ +import { readFileSync, readdirSync } from 'node:fs' +import { join } from 'node:path' + +import { describe, it, expect } from 'vitest' +import mammoth from 'mammoth' + +import { extractAttachment } from '../attachment' + +// --------------------------------------------------------------------------- +// .docx fidelity — our reader against mammoth, over a real Word corpus +// --------------------------------------------------------------------------- +// This library read .docx through mammoth.extractRawText until the streaming rewrite. That reader +// materialised an xmldom tree AND a document model before any cap could apply; the replacement walks +// word/document.xml with a SAX parser and stops at the cap. The output contract is unchanged, and +// this file is what says so. +// +// THE INTERLOCK. Each case pins ONE golden from TWO sides: the snapshot records what we produce, and +// mammoth is then asserted to produce the same string. Neither side can move alone — +// - a regression in our reader fails the snapshot; +// - `vitest -u`, which would launder that regression by rewriting the snapshot, then fails the +// mammoth assertion, because mammoth still produces the old value. +// The only way to change behaviour is to add an ACCEPTED_DIVERGENCES entry with a written reason, +// which is a reviewed, committed edit. That is the mechanical form of "any structural regressions +// understood and accepted". +// +// NO skipIf ON THE ORACLE. `import mammoth` at the top is deliberate: if mammoth is missing the +// suite fails loudly rather than degrading to a snapshot pinned only by the code that generated it, +// which is self-referential and reports green. mammoth is a devDependency, so it is always present; +// retiring the oracle should be a reviewed deletion of this file, not a condition that skips itself. + +// Vendored from mammoth 1.12.0 test/test-data — see docx-corpus/README.md for why they are copied +// rather than read out of node_modules, and what each one covers. +const CORPUS_DIR = join(process.cwd(), 'tests', 'fixtures', 'docx-corpus') +// The three fixtures the rest of the suite already uses. They are macOS textutil output, a dialect +// the vendored corpus (all Word) does not otherwise represent; elsewhere they are only asserted with +// `toContain`, which cannot catch a paragraph-break regression. +const FIXTURE_DIR = join(process.cwd(), 'tests', 'fixtures') +const LOCAL_FIXTURES = ['sample.docx', 'empty.docx', 'image.docx'] + +const DOCX_TYPE = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + +// Documents where our reader deliberately diverges from mammoth. Each entry must say WHY, and holds +// what mammoth produces — the snapshot holds ours. An empty table means the port is exact. +const ACCEPTED_DIVERGENCES: Record = {} + +// The entry point omits `extraction` entirely rather than ever returning '' (a ran-but-empty +// extraction is `{ status: 'extracted' }` with no text). That collapse is a library-wide contract +// applied identically to whatever reader is installed, so the oracle gets it too — otherwise +// empty.docx, whose text is exactly '\n\n', reads as a divergence in every run. +const asExtraction = (text: string): string | null => (text.trim().length === 0 ? null : text) + +const corpusFiles = () => readdirSync(CORPUS_DIR).filter((name) => name.endsWith('.docx')).sort() + +const cases = [ + ...corpusFiles().map((name) => [name, join(CORPUS_DIR, name)] as const), + ...LOCAL_FIXTURES.map((name) => [`fixtures/${name}`, join(FIXTURE_DIR, name)] as const), +] + +describe('docx fidelity — our reader against mammoth', () => { + it.each(cases)('%s', async (name, path) => { + const content = readFileSync(path) + + const result = await extractAttachment({ content, contentType: DOCX_TYPE }) + // No corpus document is anywhere near MAX_OUTPUT_CHARS, so anything but a complete + // extraction means a gate fired that should not have — worth failing on directly rather + // than letting it show up as a confusing snapshot diff. + expect(result.status).toBe('extracted') + expect(result.truncated).toBe(false) + + const ours = result.extraction ?? null + expect(ours).toMatchSnapshot() + + const { value } = await mammoth.extractRawText({ buffer: content }) + const theirs = asExtraction(value) + + const divergence = ACCEPTED_DIVERGENCES[name] + if (divergence) { + expect(theirs).toBe(divergence.mammoth) + return + } + // Paragraph array first: vitest diffs it element-wise, so a lost or hoisted paragraph reads + // as one changed line instead of a wall of escaped text. Then the exact string, which is + // what catches trailing breaks and whitespace the split would swallow. + expect((theirs ?? '').split('\n\n')).toEqual((ours ?? '').split('\n\n')) + expect(theirs).toBe(ours) + }) + + // A fixture added without a golden, or deleted while its golden lingers, are both silent holes: + // it.each simply enumerates fewer cases and the suite still passes. Counting them here is what + // makes the corpus itself part of the contract. + it('runs every corpus document, and only documents that exist', () => { + expect(corpusFiles()).toHaveLength(17) + expect(cases).toHaveLength(20) + for (const [, path] of cases) expect(readFileSync(path).length).toBeGreaterThan(0) + }) +}) diff --git a/tests/docx-hostile.test.ts b/tests/docx-hostile.test.ts new file mode 100644 index 0000000..7b15c97 --- /dev/null +++ b/tests/docx-hostile.test.ts @@ -0,0 +1,210 @@ +import { describe, it, expect } from 'vitest' +import JSZip from 'jszip' + +import { extractAttachment, HANDLER_TIMEOUT_MS } from '../attachment' + +// --------------------------------------------------------------------------- +// .docx on malformed and hostile archives +// --------------------------------------------------------------------------- +// THE ERROR POLICY, stated once. saxes is a conformant parser; the DOM parser it replaced recovered +// from some malformedness and threw on other. So the swap moves the line in BOTH directions, and +// this file is where that is pinned rather than discovered. The policy: +// +// text was read, then the parse broke -> `extracted` + `truncated: true`. A prefix is honest, and +// the contract already has a word for "the document +// continues past this point". +// nothing was read -> `failed`, which a caller can see and retry. +// +// Deliberately NOT a saxes error handler that reports and parses on. Measured, that recovery emits +// close-tag text as content and descends into elements mammoth drops — silent wrong output, which is +// the one outcome this library fails over everywhere else. +// +// Measured against mammoth on the eleven cases below: seven agree, two we are STRICTER on (a raw +// 0x0B, and the same character as a numeric reference — U+000B is not a legal XML character, and +// xmldom passed it through where saxes refuses), and two we are MORE AVAILABLE on (trailing junk +// after the root, and an unclosed root, both of which made mammoth throw away the whole document). +// Each case below names which. +// +// An undeclared namespace prefix is NOT in this file, deliberately. It used to be, asserting +// `truncated` — but the fixture put the offending element after ``, so it was measuring +// trailing junk under a misleading name and duplicating the case above it. Undeclared prefixes no +// longer break the parse at all (see the resolvePrefix block in docx-streaming.test.ts), which is +// what a test measuring the right thing would have said. + +const DOCX_TYPE = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' +const W = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' + +const zipWith = async (name: string, content: string | Buffer, compression: 'DEFLATE' | 'STORE' = 'DEFLATE') => { + const zip = new JSZip() + zip.file( + '[Content_Types].xml', + '' + ) + zip.file( + '_rels/.rels', + '' + ) + zip.file(name, content) + return Buffer.from(await zip.generateAsync({ type: 'nodebuffer', compression })) +} + +const docxOf = (documentXml: string) => zipWith('word/document.xml', documentXml) +const body = (inner: string) => `${inner}` +const para = (s: string) => `${s}` + +const extract = async (documentXml: string) => + extractAttachment({ content: await docxOf(documentXml), contentType: DOCX_TYPE }) + +const VT = String.fromCharCode(11) // U+000B — not a legal XML 1.0 character + +describe('docx — malformed XML', () => { + // The two we are STRICTER on. Both are the same illegal character by two spellings, and both + // sit in the FIRST text node — so nothing was read before the parse broke, and the policy says + // `failed`. The same character further into a document would keep everything before it. Pinned + // as a deliberate coverage loss, not an oversight: a document that is not well-formed XML now + // gets a labeled refusal instead of text the previous parser guessed at. + it.each([ + ['a raw control character in a w:t', body(para(`a${VT}b`))], + ['the same character as a numeric reference', body(para('a b'))], + ])('fails on %s, where the DOM parser passed it through', async (_name, documentXml) => { + const r = await extract(documentXml) + expect(r.status).toBe('failed') + expect(r.extraction).toBeUndefined() + }) + + // The two we are MORE AVAILABLE on: mammoth threw away the entire document for both of these, + // and the text before the fault is perfectly good. + it.each([ + ['trailing junk after the root element', `${body(para('one'))}`], + ['an unclosed root element', `${para('one')}`], + ])('keeps the text read before %s, and flags it truncated', async (_name, documentXml) => { + const r = await extract(documentXml) + expect(r.status).toBe('extracted') + expect(r.extraction).toBe('one\n\n') + expect(r.truncated).toBe(true) + }) + + // Same policy, a case where mammoth also recovered. The text matches it exactly; what differs is + // that we say the document was cut and it did not. + it('keeps the text and flags truncated on a mismatched close tag', async () => { + const r = await extract( + `${para('one')}` + ) + expect(r.status).toBe('extracted') + expect(r.truncated).toBe(true) + }) + + // The degradation the two `failed` cases above only show the worst end of. The same illegal + // character, moved off the first text node, costs everything after it and nothing before — + // which is what makes "stricter than mammoth" a bounded loss rather than a cliff. + it('keeps everything before an illegal character that appears partway through', async () => { + const r = await extract(body(`${para('first')}${para('second')}${para(`third${VT}broken`)}${para('fourth')}`)) + expect(r.status).toBe('extracted') + expect(r.truncated).toBe(true) + expect(r.extraction).toBe('first\n\nsecond\n\n') + }) + + // mammoth threw on both of these too, so this is agreement rather than a change. + it.each([ + ['an undefined entity', body(para('a b'))], + ['a duplicate attribute', `x`], + ])('fails on %s', async (_name, documentXml) => { + expect((await extract(documentXml)).status).toBe('failed') + }) + + // mammoth threw "Could not find the body element: are you sure this is a docx file?" for both. + // Without this check a zip whose main part is not WordprocessingML would come back `extracted` + // with no text — a silent nothing rather than a labeled refusal. + it.each([ + ['a w:document with no w:body', `${para('x')}`], + ['a foreign root element', '

hi

'], + ])('fails on %s rather than extracting nothing', async (_name, documentXml) => { + const r = await extract(documentXml) + expect(r.status).toBe('failed') + expect(r.reason).toMatch(/w:body/) + }) +}) + +describe('docx — hostile payloads', () => { + // saxes never parses entity declarations: its entity table is the five predefined ones on a null + // prototype, and nothing ever writes to it. So the expansion cannot happen — the reference is + // simply undefined and the parse fails. Asserted on TIME as well as status, because "bounded" + // is the actual property; a parser that expanded this would not return at all. + it('refuses a billion-laughs payload without expanding it', async () => { + const entities = + '' + + [1, 2, 3, 4].map((n) => ``).join('') + const started = Date.now() + const r = await extract( + `${para('&l4;')}` + ) + expect(Date.now() - started).toBeLessThan(HANDLER_TIMEOUT_MS) + expect(r.status).toBe('failed') + expect(r.extraction).toBeUndefined() + }) + + // 1000 seeded iterations. The property is not "produces good text" — it is that the degraded + // state is always REACHABLE and always LABELED, which is what makes shipping without a fallback + // reader safe. Seeded so a failure is reproducible; the long unseeded runs live in eval/. + it('never throws, always labels, and always returns in time on mutated bytes', async () => { + let seed = 0x5eed + const rnd = () => (seed = (seed * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff + + // Two archives, because they fuzz different layers. The deflated one mostly exercises the + // zip walk and the decompression preflight; the stored one puts the mutated bytes straight + // in front of saxes, which is where the interesting failures are. + const deflated = await docxOf(body([...Array(20)].map((_, i) => para(`paragraph ${i}`)).join(''))) + const stored = await zipWith( + 'word/document.xml', + body([...Array(20)].map((_, i) => para(`paragraph ${i}`)).join('')), + 'STORE' + ) + + for (let i = 0; i < 1000; i++) { + const source = i % 2 === 0 ? deflated : stored + const mutated = Buffer.from(source) + for (let f = 0; f < 1 + ((rnd() * 4) | 0); f++) { + mutated[(rnd() * mutated.length) | 0] ^= 1 << ((rnd() * 8) | 0) + } + + const started = Date.now() + const r = await extractAttachment({ content: mutated, contentType: DOCX_TYPE }) + expect(Date.now() - started).toBeLessThan(HANDLER_TIMEOUT_MS) + expect(['extracted', 'skipped', 'failed']).toContain(r.status) + // The contract everywhere else in this library: text is present or the field is absent, + // never ''. A mutation must not be able to produce the third state. + expect(r.extraction).not.toBe('') + } + }) +}) + +describe('docx — archives the handler never sees', () => { + // Routing refuses this before the handler: ooxmlKind answers 'docx' only for an archive whose + // central directory NAMES word/document.xml, so a renamed main part is unroutable. mammoth would + // have followed _rels/.rels here — this is the one place we read the conventional path instead — + // but routing already declined such a package before this change, so it is a pinned agreement + // rather than a regression. If it ever needs fixing, the fix belongs in ooxmlKind. + it('declines an archive whose main part is not at the conventional path', async () => { + const content = await zipWith('word/document2.xml', body(para('unreachable'))) + const r = await extractAttachment({ content, contentType: DOCX_TYPE }) + expect(r.status).toBe('skipped') + expect(r.reason).toMatch(/unsupported type/) + }) + + // The decompression preflight inflates every entry to measure it, so a corrupt deflate stream is + // caught there — before the handler's own inflate can see it. The value of pinning it is that + // the handler's inflate-error path is proven unreachable-but-safe: if a future preflight change + // stops catching this, the reason string moves and this test says so. + it('fails a corrupt deflate stream at the preflight, not in the handler', async () => { + const good = await docxOf(body(para('x'))) + const corrupt = Buffer.from(good) + const at = corrupt.indexOf('word/document.xml') + 'word/document.xml'.length + 10 + corrupt[at] ^= 0xff + corrupt[at + 1] ^= 0xff + corrupt[at + 2] ^= 0xff + + const r = await extractAttachment({ content: corrupt, contentType: DOCX_TYPE }) + expect(r.status).toBe('failed') + expect(r.reason).toBe('malformed zip: unreadable compressed data') + }) +}) diff --git a/tests/docx-streaming.test.ts b/tests/docx-streaming.test.ts new file mode 100644 index 0000000..4e96f2c --- /dev/null +++ b/tests/docx-streaming.test.ts @@ -0,0 +1,430 @@ +import { describe, it, expect, vi } from 'vitest' +import JSZip from 'jszip' + +import { extractAttachment, HANDLER_TIMEOUT_MS, MAX_OUTPUT_CHARS } from '../attachment' + +// --------------------------------------------------------------------------- +// .docx streaming reader — element semantics, caps, and the deadline +// --------------------------------------------------------------------------- +// tests/docx-fidelity.test.ts proves the reader agrees with mammoth on a real Word corpus. This file +// covers what that corpus cannot: the corpus contains no w:tab, no w:br, no w:cr, no tracked +// changes, no fields, no w:sdt and no w:hyperlink at all — real Word documents simply don't carry +// every construct — and no corpus document is anywhere near the output cap or the deadline. +// +// THE CONTRACT BEING PINNED, in one place so the tests below can just reference it (mammoth's +// raw-text.js is 13 lines, and each of these was measured against it, not read off the spec): +// +// w:t -> its text, verbatim, never trimmed +// w:tab -> '\t'; w:noBreakHyphen -> U+2011; w:softHyphen -> U+00AD +// w:p -> its children, then TWO newlines. Including the last paragraph, so a document +// ends with '\n\n'. +// w:br, w:cr -> NOTHING. Two different mechanisms in mammoth (br is a document node with no raw +// mapping, cr isn't whitelisted at all) reaching the same result. +// anything else with no handler -> the WHOLE SUBTREE disappears, children included. This is the +// one that makes a naive "emit every w:t" reader wrong rather than merely different. +// bare text outside a w:t -> dropped. +// tables -> one paragraph per cell paragraph. No tabs, no row markers: a 2x2 table is +// byte-identical to four consecutive paragraphs. +// +// Everything here builds its own archive, because the point is to reach constructs no real document +// in the corpus contains. + +const DOCX_TYPE = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + +const NS = [ + 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"', + 'xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"', + 'xmlns:v="urn:schemas-microsoft-com:vml"', + 'xmlns:wps="http://schemas.microsoft.com/office/word/2010/wordprocessingShape"', + 'xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"', +].join(' ') + +// JSZip output satisfies both decompression-budget invariants, so a fixture built here reaches the +// handler rather than being turned away in front of it. STORE covers the stored-entry path. +const docxFrom = async (documentXml: string, compression: 'DEFLATE' | 'STORE' = 'DEFLATE') => { + const zip = new JSZip() + zip.file( + '[Content_Types].xml', + '' + ) + zip.file( + '_rels/.rels', + '' + ) + zip.file('word/document.xml', documentXml) + return Buffer.from(await zip.generateAsync({ type: 'nodebuffer', compression })) +} + +const buildDocx = (body: string, compression: 'DEFLATE' | 'STORE' = 'DEFLATE') => + docxFrom(`${body}`, compression) + +const extract = async (body: string, options?: { maxOutputChars?: number }) => + extractAttachment({ content: await buildDocx(body), contentType: DOCX_TYPE }, options) + +const para = (inner: string) => `${inner}` +const text = (s: string) => para(`${s}`) + +describe('docx — the whitelist drops unrecognised subtrees', () => { + // mc:Choice is the DrawingML branch and mc:Fallback the VML one; Word emits both for the same + // text box. Taking both would DOUBLE the text, and taking Choice instead of Fallback would take + // the branch mammoth discards. Corpus-covered by text-box.docx, kept here because a 20-document + // snapshot failure names the document, not the branch. + it('takes only the mc:Fallback branch of mc:AlternateContent', async () => { + const r = await extract( + `` + + `${text('CHOICE')}` + + `${text('FALLBACK')}` + + `` + ) + expect(r.extraction).toContain('FALLBACK') + expect(r.extraction).not.toContain('CHOICE') + }) + + // A w:pict's text is hoisted out and reinserted as a SIBLING of the enclosing paragraph, so it + // lands AFTER that paragraph's break rather than inside it. Emitting it inline would glue two + // words into one nonword token — 'HelloBoxed' — which is exactly the shape an LLM reads wrong. + it('emits w:pict content after the enclosing paragraph, not inside it', async () => { + const r = await extract( + `Hello${text('Boxed')}` + ) + expect(r.extraction).toBe('Hello\n\nBoxed\n\n') + }) + + // The other half, and the counter-intuitive one: an extra that never reaches a w:p is silently + // discarded. It looks like a bug worth "fixing", so it is pinned as the contract instead. + it('drops a w:pict that sits outside any paragraph', async () => { + const r = await extract( + `${text('Orphan')}${text('after')}` + ) + expect(r.extraction).toBe('after\n\n') + }) + + // Over-tightening the whitelist loses real body text silently. None of these containers appears + // anywhere in the fidelity corpus, so nothing else would catch it. + it.each([ + ['w:ins', `KEEP`], + ['w:smartTag', `KEEP`], + ['w:hyperlink', `KEEP`], + ['w:sdt', `KEEP`], + ['w:object', `KEEP`], + ])('recurses into %s', async (_name, body) => { + const r = await extract(body) + expect(r.extraction).toContain('KEEP') + expect(r.extraction).not.toContain('DROP') + }) + + // The load-bearing one. A reader that recursed into unknown elements — the obvious way to write + // this — would pass every other test in this file and still over-extract on real documents. + it('drops an unrecognised element WITH its children rather than recursing', async () => { + const r = await extract(`${text('kept')}${text('hidden')}`) + // Not just the text: no stray '\n\n' from the paragraph inside the dropped subtree either. + expect(r.extraction).toBe('kept\n\n') + }) + + // Character data lives only inside w:t. Whitespace between elements is common in + // pretty-printed markup, so a reader that accumulated all text would leak indentation. + it('drops bare character data outside a w:t', async () => { + const r = await extract('straykeptmore') + expect(r.extraction).toBe('kept\n\n') + }) + + // Field instructions are markup, not content. Leaking them puts ` HYPERLINK "http://..." ` into + // the body of an extract that an agent then reads as prose. + it('suppresses w:instrText across a complex field, keeping only the display run', async () => { + const r = await extract( + `` + + `` + + ` HYPERLINK "http://example.com" ` + + `` + + `link` + + `` + + `` + ) + expect(r.extraction).toBe('link\n\n') + }) + + // KNOWN mammoth DEFECT, MATCHED ON PURPOSE. w:fldSimple has no handler in mammoth and is not on + // its ignore list, so the whole subtree — including the cached result a reader sees on screen — + // is dropped. Cross-references, page numbers, DOCPROPERTY and merge fields all land here. This + // change is a pure port, so it keeps the defect; recovering it is one entry in DOCX_CONTAINERS + // and belongs in its own change, once the corpus says how many documents it affects. + it('drops a w:fldSimple subtree, matching mammoth rather than fixing it here', async () => { + const r = await extract( + `page 7` + ) + expect(r.extraction).toBe('page \n\n') + }) + + // Deleted text must not resurface in an extract; inserted text must. + it('drops tracked deletions and keeps tracked insertions', async () => { + const r = await extract( + `` + + `GONE` + + `ADDED` + + `` + ) + expect(r.extraction).toBe('ADDED\n\n') + }) + + // The subtlest paragraph rule. A deleted paragraph MARK means the break is gone, not the text: + // the content runs into the next paragraph. Emitting the usual '\n\n' would invent a break the + // author deleted, splitting one sentence into two. + it('merges a paragraph whose mark was deleted into the next one', async () => { + const r = await extract( + `first half ${text('second half')}` + ) + expect(r.extraction).toBe('first half second half\n\n') + }) + + // Same marker, one level out: a deleted table row takes the whole row with it. + it('drops a table row marked deleted, keeping the rows around it', async () => { + const r = await extract( + `` + + `${text('dropped cell')}` + + `${text('live cell')}` + + `` + ) + expect(r.extraction).toBe('live cell\n\n') + }) +}) + +describe('docx — leaf and character contract', () => { + // TWO newlines, and a trailing one on the last paragraph. A reader that joined paragraphs with + // '\n\n' instead of suffixing each would differ only in the final break — invisible in a + // `toContain` assertion, and different from mammoth on every document ever extracted. + it('suffixes every paragraph with two newlines, including the last', async () => { + expect((await extract(`${text('a')}${text('b')}`)).extraction).toBe('a\n\nb\n\n') + }) + + // An empty paragraph is exactly '\n\n', which the entry point then trims to nothing rather than + // storing ''. Both halves matter: the break is produced, and the result still omits `extraction`. + it('produces only breaks for an empty paragraph, and omits the extraction', async () => { + const r = await extract('') + expect(r.status).toBe('extracted') + expect(r.extraction).toBeUndefined() + }) + + // The "obvious improvement" that would silently change every document ever extracted. mammoth + // emits nothing for either — w:br is a document node with no raw-text mapping, w:cr is not + // whitelisted at all — so a line break inside a paragraph simply is not represented. + it('emits nothing for w:br or w:cr', async () => { + const r = await extract(para('ab')) + expect(r.extraction).toBe('ab\n\n') + }) + + it('maps w:tab, w:noBreakHyphen and w:softHyphen to their literals', async () => { + const r = await extract(para('abc')) + expect(r.extraction).toBe(`a\tb‑­c\n\n`) + }) + + // ACCEPTED DIVERGENCE. mammoth maps w:sym through the dingbat-to-unicode package; we drop it, + // because that table would be the only reason to keep any of mammoth's dependency tree and the + // only place this reader would read an attribute at all. Pinned so the choice is visible rather + // than incidental — if the corpus ever shows it matters, the fix is a small inline table. + it('drops w:sym rather than taking a dependency to map it', async () => { + const r = await extract(para('ab')) + expect(r.extraction).toBe('ab\n\n') + }) + + // Leading and trailing spaces inside a w:t are content. A "helpful" trim would silently join + // words across run boundaries, which is how formatted text is stored. + it('never trims whitespace inside or around a run', async () => { + const r = await extract(para(' spaced out ')) + expect(r.extraction).toBe(' spaced out \n\n') + }) + + // Entities decode; CDATA does NOT. The second is counter-intuitive: mammoth's DOM walk only + // handles element and text nodes, so a CDATA section vanishes. We reproduce it by NOT wiring + // saxes' cdata event — i.e. by absence, which is exactly the kind of thing a later reader + // "fixes" without realising it is a contract. + it('decodes XML entities but drops CDATA, as mammoth does', async () => { + expect((await extract(text('a & b <c>'))).extraction).toBe('a & b \n\n') + expect((await extract(para(''))).extraction).toBeUndefined() + }) + + // xmldom normalised all four to '\n' before mammoth ever saw the markup. saxes does the + // XML-standard \r\n and \r itself but leaves NEL (U+0085) and LINE SEPARATOR (U+2028) alone, so + // without an explicit normalisation this is the one test that fails. + it('normalises CRLF, CR, NEL and LINE SEPARATOR to a plain newline', async () => { + const CR = String.fromCharCode(13) + const r = await extract(text(`a${CR}\nb${CR}c…d
e`)) + expect(r.extraction).toBe('a\nb\nc\nd\ne\n\n') + }) + + // Catches decoding each inflate chunk independently. A chunk boundary lands mid-sequence in any + // document with non-ASCII text, and chunk.toString('utf8') turns that one character into two + // U+FFFD. Deliberately not tuned to the chunk size: a long run of 4-byte characters puts a + // boundary inside one of them whatever the chunking turns out to be. + it('decodes multi-byte UTF-8 across inflate chunk boundaries', async () => { + const emoji = '\u{1F600}'.repeat(20_000) // 80 KB of 4-byte sequences, several chunks deep + const r = await extract(text(emoji)) + expect(r.extraction).not.toContain('�') + expect(r.extraction).toBe(`${emoji}\n\n`) + }) + + // The entry point's surrogate guard only fires when the handler OVERSHOOTS the cap; a handler + // that stopped exactly on it would slip a lone half through, which serialises as U+FFFD. The + // existing cap tests go through text/plain and cannot reach this path. + it('never splits a surrogate pair when the cap lands mid-character', async () => { + const r = await extract(text('\u{1F600}'.repeat(4_000)), { maxOutputChars: 1_001 }) + expect(r.truncated).toBe(true) + expect(r.extraction).not.toContain('�') + expect([...(r.extraction ?? '')].every((c) => c === '\u{1F600}')).toBe(true) + }) +}) + +// Undeclared namespace prefixes. saxes fails the whole parse on one — for ATTRIBUTES as well as +// elements — where the DOM parser this replaced shrugged. Under the error policy that turns into +// `extracted` + `truncated`, i.e. everything after the offending element is silently gone. These are +// not exotic: Word 2013 and later stamp w15:paraId on EVERY w:p, with w16cid:durableId beside it, so +// a document carrying those without their xmlns loses everything after its first paragraph. +// +// An allowlist of known prefixes cannot close that — there is always another prefix, and the first +// version of this reader proved the point by naming wne: in a comment while omitting it from the +// map. resolvePrefix closes the open class instead: anything unclaimed resolves to a sentinel URI +// that OOXML_PREFIXES does not map, which is exactly the shape mammoth gives an unmapped namespace. +describe('docx — an undeclared prefix must not end the parse', () => { + const withAttr = (attr: string) => + `${text('before')}middle${text('after')}` + + // Attributes, which is the common case and the one that reads as impossible until you hit it: + // the element is perfectly well-formed WordprocessingML and only its attribute is unbound. + it.each([ + ['w15:paraId', 'w15:paraId="12AB34CD"'], + ['w16cid:durableId', 'w16cid:durableId="99"'], + ['a prefix no allowlist would think to carry', 'zz:whatever="1"'], + ])('reads the whole document despite an undeclared %s attribute', async (_name, attr) => { + const r = await extract(withAttr(attr)) + expect(r.extraction).toBe('before\n\nmiddle\n\nafter\n\n') + expect(r.truncated).toBe(false) + }) + + // Elements, where the prefix being unmapped must still mean "drop this subtree" — the fix must + // not turn an unknown namespace into a recursed-into container. + it('drops an undeclared-prefix element with its children, and keeps reading after it', async () => { + const r = await extract(`${text('before')}${text('hidden')}${text('after')}`) + expect(r.extraction).toBe('before\n\nafter\n\n') + expect(r.truncated).toBe(false) + }) + + // Why the assumed-prefix map still has to exist alongside the sentinel: `w` is one of the three + // prefixes OOXML_PREFIXES actually maps, so resolving an undeclared one to the sentinel would + // drop the entire document body. mammoth throws outright on this document; we recover it. + it('still recovers a document that does not declare w: at all', async () => { + const content = await docxFrom( + `${text('recovered')}` + ) + const r = await extractAttachment({ content, contentType: DOCX_TYPE }) + expect(r.extraction).toBe('recovered\n\n') + }) + + // ...and the assumed map must never win over a real declaration, or a strict-format document + // would be read against the transitional URI. + it('lets an in-document xmlns shadow the assumed prefix', async () => { + const content = await docxFrom( + `${text('strict')}` + ) + const r = await extractAttachment({ content, contentType: DOCX_TYPE }) + expect(r.extraction).toBe('strict\n\n') + }) +}) + +describe('docx — the cap and the deadline stop the read', () => { + // Long enough to cross the default cap, with a marker last so "did it stop reading" is + // observable from outside rather than inferred from a length. + const oversized = (marker = 'TRAILING-MARKER') => + `${Array.from({ length: 6_000 }, (_, i) => text(`paragraph ${i} ${'x'.repeat(50)}`)).join('')}${text(marker)}` + + it('reports truncated when the output cap stops it, and not when it does not', async () => { + const cut = await extract(oversized()) + expect(cut.status).toBe('extracted') + expect(cut.truncated).toBe(true) + expect(cut.extraction?.length).toBeLessThanOrEqual(MAX_OUTPUT_CHARS) + + const whole = await extract(text('short document')) + expect(whole.truncated).toBe(false) + expect(whole.extraction).toBe('short document\n\n') + }) + + // The claim is that the cap stops the READ, not that it trims the result. If it only trimmed, + // the marker would still have been parsed — and on a ceiling-sized document that is the whole + // difference between 68 ms and 812 ms, and between 175 MB and 607 MB. + it('stops parsing at the cap rather than reading on and trimming', async () => { + const started = Date.now() + const r = await extract(oversized()) + expect(r.extraction).not.toContain('TRAILING-MARKER') + expect(Date.now() - started).toBeLessThan(HANDLER_TIMEOUT_MS) + }) + + it('omits the extraction entirely, never returning an empty string, at a zero cap', async () => { + const r = await extract(text('some real text'), { maxOutputChars: 0 }) + expect(r.status).toBe('extracted') + expect(r.extraction).toBeUndefined() + expect(r.truncated).toBe(true) + }) + + // A single paragraph of many runs, deliberately: with the deadline checked per PARAGRAPH rather + // than per chunk, a one-paragraph document would spin unbudgeted, which is the docx analogue of + // the xlsx sheet-of-blank-rows case. + it('reports truncated when the deadline stops it', async () => { + const body = `${Array.from({ length: 4_000 }, (_, i) => `run ${i} `).join('')}` + const content = await buildDocx(body) + + // extractAttachment's FIRST Date.now() sets the deadline; every later call lands past it, so + // the read stops on its first check. Deliberately blunt — zlib and saxes may read the clock + // themselves, so a mock letting N chunks through would depend on how often they do. + const base = Date.now() + let calls = 0 + const clock = vi.spyOn(Date, 'now').mockImplementation(() => { + calls += 1 + return calls === 1 ? base : base + HANDLER_TIMEOUT_MS + 1 + }) + const stopped = await extractAttachment({ content, contentType: DOCX_TYPE }) + clock.mockRestore() + + expect(stopped.status).toBe('extracted') // a deadline stop is not a failure + expect(stopped.truncated).toBe(true) + + // Control: the same bytes on a real clock are complete and NOT flagged — without this the + // assertion above would also pass if `truncated` were hardcoded true. + const whole = await extractAttachment({ content, contentType: DOCX_TYPE }) + expect(whole.truncated).toBe(false) + expect(whole.extraction).toContain('run 3999') + }) + + // The memory claim in one assertion: a single text node far larger than the cap must not be + // accumulated whole before the cap applies. saxes buffers one text node, so this is also the + // bound on that claim — it is the shape that costs the most, and it still stops at the cap. + it('caps a document that is one enormous run', async () => { + const r = await extract(text('lorem ipsum dolor sit amet '.repeat(200_000))) // ~5.4M chars + expect(r.status).toBe('extracted') + expect(r.truncated).toBe(true) + expect(r.extraction?.length).toBeLessThanOrEqual(MAX_OUTPUT_CHARS) + expect(r.extraction?.startsWith('lorem ipsum')).toBe(true) + }) +}) + +describe('docx — archive-level reads', () => { + // Some producers store small parts uncompressed. Full fidelity is required on that path, and it + // is a different branch of the handler (no inflate stream at all). + it('reads a stored, uncompressed word/document.xml', async () => { + const content = await buildDocx(`${text('stored entry')}`, 'STORE') + const r = await extractAttachment({ content, contentType: DOCX_TYPE }) + expect(r.status).toBe('extracted') + expect(r.extraction).toBe('stored entry\n\n') + }) + + // A table flattens to one paragraph per cell paragraph: no tabs, no row markers, no column + // structure. Documented as a limit rather than a bug — it is what mammoth did, and what the + // README now states — but pinned here because it is the thing most likely to be "improved" + // without realising it changes every extracted invoice. + it('flattens a table to bare paragraphs, losing row and column structure', async () => { + const cell = (s: string) => `${text(s)}` + const r = await extract( + `${cell('r1c1')}${cell('r1c2')}${cell('r2c1')}${cell('r2c2')}` + ) + expect(r.extraction).toBe('r1c1\n\nr1c2\n\nr2c1\n\nr2c2\n\n') + // ...i.e. indistinguishable from four consecutive paragraphs: + expect(r.extraction).toBe((await extract(['r1c1', 'r1c2', 'r2c1', 'r2c2'].map(text).join(''))).extraction) + }) +}) diff --git a/tests/fixtures/docx-corpus/README.md b/tests/fixtures/docx-corpus/README.md new file mode 100644 index 0000000..fdb7026 --- /dev/null +++ b/tests/fixtures/docx-corpus/README.md @@ -0,0 +1,62 @@ +# docx fidelity corpus + +Seventeen real Word-authored `.docx`, vendored verbatim from **mammoth 1.12.0**, `test/test-data/`. + +## Why these files are here rather than read from `node_modules` + +`tests/docx-fidelity.test.ts` pins our `.docx` output against `mammoth.extractRawText` — the reader +this library used until the streaming rewrite. That suite needs a corpus that is: + +1. **Reviewable.** The acceptance criterion for the rewrite is "output diffed against mammoth across + a document corpus, with any structural regressions understood and accepted". That is only + checkable if the corpus and its expected output are in the diff. +2. **Stable.** mammoth declares no `files` field in its `package.json`, so `test/` ships by accident + rather than by contract. A future release that adds one would delete the regression suite + silently. mammoth is also a `devDependency` now, kept solely as the fidelity oracle. +3. **Extensible.** The corpus grows with redacted real-world documents; splitting it between + `node_modules` and here would be worse than one directory. + +## What they cover + +They are chosen for coverage of the WordprocessingML constructs that change the extracted text, not +for size. Notably: `strict-format.docx` binds the ISO-strict namespace URI (matching a literal `w:` +prefix would silently return nothing for it); `text-box.docx` carries a real `mc:AlternateContent` +with both a `mc:Choice` (DrawingML) and a `mc:Fallback` (VML `w:pict`) branch, which is the +double-count hazard and the paragraph-hoisting case at once; `tables.docx`, `footnotes.docx`, +`endnotes.docx` and `comments.docx` pin what is deliberately NOT extracted. + +Five of them — `embedded-style-map`, `strikethrough`, `underline`, `external-picture`, +`tiny-picture-target-base-relative` — add no construct the others don't already have. They are kept +anyway: the value of a real corpus is the cases nobody thought to enumerate, and they cost ~90 KB +and no measurable runtime. + +## Licence + +mammoth is BSD-2-Clause, which permits redistribution in source form provided the notice and +disclaimer are retained. They are, in full: + +> Copyright (c) 2013, Michael Williamson +> All rights reserved. +> +> Redistribution and use in source and binary forms, with or without +> modification, are permitted provided that the following conditions are met: +> +> 1. Redistributions of source code must retain the above copyright notice, this +> list of conditions and the following disclaimer. +> 2. Redistributions in binary form must reproduce the above copyright notice, +> this list of conditions and the following disclaimer in the documentation +> and/or other materials provided with the distribution. +> +> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +> ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +> WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +> DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +> ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +> (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +> LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +> ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +> (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +> SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Their contents are trivial sample prose ("Walking on imported air", "Ouch.") with no third-party +material. diff --git a/tests/fixtures/docx-corpus/comments.docx b/tests/fixtures/docx-corpus/comments.docx new file mode 100755 index 0000000..32cdae1 Binary files /dev/null and b/tests/fixtures/docx-corpus/comments.docx differ diff --git a/tests/fixtures/docx-corpus/embedded-style-map.docx b/tests/fixtures/docx-corpus/embedded-style-map.docx new file mode 100644 index 0000000..bb0a5e2 Binary files /dev/null and b/tests/fixtures/docx-corpus/embedded-style-map.docx differ diff --git a/tests/fixtures/docx-corpus/empty.docx b/tests/fixtures/docx-corpus/empty.docx new file mode 100644 index 0000000..e10b869 Binary files /dev/null and b/tests/fixtures/docx-corpus/empty.docx differ diff --git a/tests/fixtures/docx-corpus/endnotes.docx b/tests/fixtures/docx-corpus/endnotes.docx new file mode 100644 index 0000000..1ae4f93 Binary files /dev/null and b/tests/fixtures/docx-corpus/endnotes.docx differ diff --git a/tests/fixtures/docx-corpus/external-picture.docx b/tests/fixtures/docx-corpus/external-picture.docx new file mode 100644 index 0000000..3475626 Binary files /dev/null and b/tests/fixtures/docx-corpus/external-picture.docx differ diff --git a/tests/fixtures/docx-corpus/footnote-hyperlink.docx b/tests/fixtures/docx-corpus/footnote-hyperlink.docx new file mode 100755 index 0000000..208f3dc Binary files /dev/null and b/tests/fixtures/docx-corpus/footnote-hyperlink.docx differ diff --git a/tests/fixtures/docx-corpus/footnotes.docx b/tests/fixtures/docx-corpus/footnotes.docx new file mode 100755 index 0000000..3228e74 Binary files /dev/null and b/tests/fixtures/docx-corpus/footnotes.docx differ diff --git a/tests/fixtures/docx-corpus/simple-list.docx b/tests/fixtures/docx-corpus/simple-list.docx new file mode 100755 index 0000000..c2cd5ae Binary files /dev/null and b/tests/fixtures/docx-corpus/simple-list.docx differ diff --git a/tests/fixtures/docx-corpus/single-paragraph.docx b/tests/fixtures/docx-corpus/single-paragraph.docx new file mode 100755 index 0000000..09caf9a Binary files /dev/null and b/tests/fixtures/docx-corpus/single-paragraph.docx differ diff --git a/tests/fixtures/docx-corpus/strict-format.docx b/tests/fixtures/docx-corpus/strict-format.docx new file mode 100644 index 0000000..4495720 Binary files /dev/null and b/tests/fixtures/docx-corpus/strict-format.docx differ diff --git a/tests/fixtures/docx-corpus/strikethrough.docx b/tests/fixtures/docx-corpus/strikethrough.docx new file mode 100755 index 0000000..0d2b713 Binary files /dev/null and b/tests/fixtures/docx-corpus/strikethrough.docx differ diff --git a/tests/fixtures/docx-corpus/tables.docx b/tests/fixtures/docx-corpus/tables.docx new file mode 100755 index 0000000..86009a7 Binary files /dev/null and b/tests/fixtures/docx-corpus/tables.docx differ diff --git a/tests/fixtures/docx-corpus/text-box.docx b/tests/fixtures/docx-corpus/text-box.docx new file mode 100755 index 0000000..2d53703 Binary files /dev/null and b/tests/fixtures/docx-corpus/text-box.docx differ diff --git a/tests/fixtures/docx-corpus/tiny-picture-target-base-relative.docx b/tests/fixtures/docx-corpus/tiny-picture-target-base-relative.docx new file mode 100644 index 0000000..1d61988 Binary files /dev/null and b/tests/fixtures/docx-corpus/tiny-picture-target-base-relative.docx differ diff --git a/tests/fixtures/docx-corpus/tiny-picture.docx b/tests/fixtures/docx-corpus/tiny-picture.docx new file mode 100755 index 0000000..d45bb9e Binary files /dev/null and b/tests/fixtures/docx-corpus/tiny-picture.docx differ diff --git a/tests/fixtures/docx-corpus/underline.docx b/tests/fixtures/docx-corpus/underline.docx new file mode 100755 index 0000000..36604fc Binary files /dev/null and b/tests/fixtures/docx-corpus/underline.docx differ diff --git a/tests/fixtures/docx-corpus/utf8-bom.docx b/tests/fixtures/docx-corpus/utf8-bom.docx new file mode 100644 index 0000000..3881238 Binary files /dev/null and b/tests/fixtures/docx-corpus/utf8-bom.docx differ