diff --git a/doc/CustomConfig.md b/doc/CustomConfig.md index d3f8ad57..549827c6 100644 --- a/doc/CustomConfig.md +++ b/doc/CustomConfig.md @@ -26,6 +26,7 @@ The `Config` class has the following options: | `setPdfWhitespaces` | String | `\0\t\n\f\r ` | | | `setPdfWhitespacesRegex` | String | `[\0\t\n\f\r ]` | | | `setRetainImageContent` | Boolean | `true` | If parsing fails due to memory exhaustion, you can set the value to `false`. This will reduce memory usage, although it will no longer retain image content. | +| `setContentSpooling` | Boolean | `false` | If parsing large documents fails due to memory exhaustion, set this to `true` to spool decoded stream content to a temporary file instead of keeping it in memory. Lowers peak memory usage at the cost of some extra disk I/O. | ## option setDecodeMemoryLimit + setRetainImageContent (manage memory usage) @@ -41,6 +42,26 @@ $config->setDecodeMemoryLimit(1000000); $parser = new \Smalot\PdfParser\Parser([], $config); ``` +## option setContentSpooling (manage memory usage) + +When parsing large documents, the bulk of the memory a parsed document keeps +alive is the decoded content of its stream objects. Enabling content spooling +writes that content to a single temporary file as objects are parsed and reads +it back on demand, so the full set of decoded streams no longer has to reside in +memory at once. This noticeably lowers peak memory usage for large files in +exchange for a small amount of disk I/O. The extracted text and document details +are identical with the option on or off. + +```php +$config = new \Smalot\PdfParser\Config(); +// Spool decoded stream content to a temporary file instead of memory +$config->setContentSpooling(true); +$parser = new \Smalot\PdfParser\Parser([], $config); +``` + +The temporary file is created in the system temp directory and removed +automatically once the parsed document is no longer referenced. + ## option setHorizontalOffset When words are broken up or when the structure of a table is not preserved, you can use `setHorizontalOffset`. diff --git a/src/Smalot/PdfParser/Config.php b/src/Smalot/PdfParser/Config.php index e44b1640..384fdde8 100644 --- a/src/Smalot/PdfParser/Config.php +++ b/src/Smalot/PdfParser/Config.php @@ -89,6 +89,15 @@ class Config */ private $ignoreEncryption = false; + /** + * Whether decoded object stream content is spooled to a temporary file + * instead of being kept in memory. Trades disk I/O for a lower peak memory + * footprint when parsing large documents. + * + * @var bool + */ + private $contentSpooling = false; + public function getFontSpaceLimit() { return $this->fontSpaceLimit; @@ -172,4 +181,14 @@ public function setIgnoreEncryption(bool $ignoreEncryption): void { $this->ignoreEncryption = $ignoreEncryption; } + + public function getContentSpooling(): bool + { + return $this->contentSpooling; + } + + public function setContentSpooling(bool $contentSpooling): void + { + $this->contentSpooling = $contentSpooling; + } } diff --git a/src/Smalot/PdfParser/ContentSpool.php b/src/Smalot/PdfParser/ContentSpool.php new file mode 100644 index 00000000..1863615d --- /dev/null +++ b/src/Smalot/PdfParser/ContentSpool.php @@ -0,0 +1,153 @@ + + * + * @date 2026-06-18 + * + * @license LGPLv3 + * + * @url + * + * PdfParser is a pdf library written in PHP, extraction oriented. + * Copyright (C) 2017 - Sébastien MALOT + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. + * If not, see . + */ + +namespace Smalot\PdfParser; + +/** + * Temporary on-disk store for decoded stream content. + * + * When content spooling is enabled (see Config::setContentSpooling()), the + * decoded content of each parsed object is appended to a single temporary file + * instead of being held in memory, and read back on demand. This trades a small + * amount of disk I/O for a markedly lower peak memory footprint, since the full + * set of decoded streams - usually the largest thing a parsed Document keeps + * alive - no longer has to reside in RAM all at once. + * + * The backing temporary file is created lazily on first use and removed + * automatically once the spool (and the Document owning it) is destroyed. + * + * @internal + */ +class ContentSpool +{ + /** + * @var resource|null + */ + private $handle; + + /** + * Current size of the spool file, i.e. the offset at which the next chunk + * of content will be written. + * + * @var int + */ + private $size = 0; + + /** + * Append a chunk of content to the spool. + * + * @return array{0: int, 1: int}|null [offset, length] locating the stored + * content, or null if it could not be + * stored (the caller should then keep + * the content in memory) + */ + public function store(string $content): ?array + { + $length = \strlen($content); + if (0 === $length) { + return null; + } + + $handle = $this->handle(); + if (null === $handle) { + return null; + } + + $offset = $this->size; + if (0 === fseek($handle, $offset) && fwrite($handle, $content) === $length) { + $this->size += $length; + + return [$offset, $length]; + } + + return null; + } + + /** + * Read back content previously stored via store(). + * + * @param int $offset offset returned by store() + * @param int $length length returned by store() + */ + public function fetch(int $offset, int $length): string + { + if (!\is_resource($this->handle) || $length <= 0) { + return ''; + } + + // Seeking before reading also flushes any pending write buffer, which + // is required when reading data that was just written to the handle. + if (0 !== fseek($this->handle, $offset)) { + return ''; + } + + $content = ''; + $remaining = $length; + while ($remaining > 0 && !feof($this->handle)) { + $chunk = fread($this->handle, $remaining); + if (false === $chunk || '' === $chunk) { + break; + } + $content .= $chunk; + $remaining -= \strlen($chunk); + } + + return $content; + } + + /** + * Lazily open the backing temporary file. + * + * @return resource|null + */ + private function handle() + { + if (null === $this->handle) { + // tmpfile() opens a binary-safe read-write handle and removes the + // underlying file automatically when the handle is closed. + $handle = tmpfile(); + $this->handle = false !== $handle ? $handle : null; + } + + return $this->handle; + } + + /** + * Ensure the backing file is closed and removed when the spool is destroyed. + */ + public function __destruct() + { + if (\is_resource($this->handle)) { + fclose($this->handle); + } + } +} diff --git a/src/Smalot/PdfParser/Document.php b/src/Smalot/PdfParser/Document.php index 1fad8b1b..d2a5af3a 100644 --- a/src/Smalot/PdfParser/Document.php +++ b/src/Smalot/PdfParser/Document.php @@ -74,11 +74,29 @@ class Document */ protected $details; + /** + * Optional on-disk store for decoded object stream content, shared by all + * objects of this document. Only set when content spooling is enabled. + * + * @var ContentSpool|null + */ + protected $contentSpool; + public function __construct() { $this->trailer = new Header([], $this); } + public function getContentSpool(): ?ContentSpool + { + return $this->contentSpool; + } + + public function setContentSpool(?ContentSpool $contentSpool): void + { + $this->contentSpool = $contentSpool; + } + public function init() { $this->buildDictionary(); diff --git a/src/Smalot/PdfParser/PDFObject.php b/src/Smalot/PdfParser/PDFObject.php index 378ae15d..271e4acc 100644 --- a/src/Smalot/PdfParser/PDFObject.php +++ b/src/Smalot/PdfParser/PDFObject.php @@ -65,10 +65,19 @@ class PDFObject protected $header; /** - * @var string + * @var string|null */ protected $content; + /** + * Location of this object's content within the document's ContentSpool, + * as [offset, length], when the content has been spooled to disk instead + * of being kept in $content. Null when the content lives in memory. + * + * @var array{0: int, 1: int}|null + */ + protected $contentRef; + /** * @var Config|null */ @@ -130,9 +139,45 @@ public function getDetails(bool $deep = true): array public function getContent(): ?string { + // Content has been spooled to disk; read it back on demand. + if (null !== $this->contentRef) { + $spool = $this->document->getContentSpool(); + + return null !== $spool + ? $spool->fetch($this->contentRef[0], $this->contentRef[1]) + : null; + } + return $this->content; } + /** + * Move this object's in-memory content to the document's ContentSpool, if + * one is configured, freeing the in-memory copy. The content is transparently + * read back from disk by getContent() when needed. + * + * @internal + */ + public function spoolContent(): void + { + if (null !== $this->contentRef + || null === $this->content + || '' === $this->content) { + return; + } + + $spool = $this->document->getContentSpool(); + if (null === $spool) { + return; + } + + $ref = $spool->store($this->content); + if (null !== $ref) { + $this->contentRef = $ref; + $this->content = null; + } + } + /** * Creates a duplicate of the document stream with * strings and other items replaced by $char. Formerly @@ -429,65 +474,65 @@ public function getSectionsText(?string $content): array { $sections = []; - // A cleaned stream has one command on every line, so split the - // cleaned stream content on \r\n into an array - $textCleaned = preg_split( - '/(\r\n|\n|\r)/', - $this->formatContent($content), - -1, - \PREG_SPLIT_NO_EMPTY - ); + // A cleaned stream has one command on every line. Splitting the whole + // string into an array up front is simplest, but a graphics-heavy page + // can have hundreds of thousands of lines, of which only a handful are + // kept below. The resulting array can take a lot of memory. + // Instead split in bounded, line-aligned chunks first and process each + // chunk, so only a small slice is ever materialized at once. + $cleaned = $this->formatContent($content); + $length = \strlen($cleaned); $inTextBlock = false; - foreach ($textCleaned as $line) { - $line = trim($line); + $chunkSize = 1024 * 1024; // 1 MB; bounds the per-chunk line array + $offset = 0; + while ($offset < $length) { + // Cut the chunk at the next line boundary so a command is never + // split across chunks; the $inTextBlock flag carries across them. + $end = min($offset + $chunkSize, $length); + if ($end < $length) { + $end += strcspn($cleaned, "\r\n", $end); + } - // Skip empty lines - if ('' === $line) { - continue; + // Split into lines. When the whole stream fits in one chunk (the + // common case) split it directly to avoid copying it via substr(). + $chunk = (0 === $offset && $length === $end) + ? $cleaned + : substr($cleaned, $offset, $end - $offset); + $textCleaned = preg_split('/(\r\n|\n|\r)/', $chunk, -1, \PREG_SPLIT_NO_EMPTY); + + // Advance past the chunk and the run of delimiters following it. + $offset = $end + strspn($cleaned, "\r\n", $end); + + // On the final chunk the source stream is no longer needed; release + // it before filtering the lines so single-chunk pages peak no higher + // than a plain whole-string split would. + if ($offset >= $length) { + $cleaned = $chunk = ''; } - // If a 'BT' is encountered, set the $inTextBlock flag - if (preg_match('/BT$/', $line)) { - $inTextBlock = true; - $sections[] = $line; - - // If an 'ET' is encountered, unset the $inTextBlock flag - } elseif ('ET' == $line) { - $inTextBlock = false; - $sections[] = $line; - } elseif ($inTextBlock) { - // If we are inside a BT ... ET text block, save all lines - $sections[] = trim($line); - } else { - // Otherwise, if we are outside of a text block, only - // save specific, necessary lines. Care should be taken - // to ensure a command being checked for *only* matches - // that command. For instance, a simple search for 'c' - // may also match the 'sc' command. See the command - // list in the formatContent() method above. - // Add more commands to save here as you find them in - // weird PDFs! - if ('q' == $line[-1] || 'Q' == $line[-1]) { - // Save and restore graphics state commands - $sections[] = $line; - } elseif (preg_match('/(?isKeptOutsideTextBlock($line)) { $sections[] = $line; } } @@ -496,6 +541,26 @@ public function getSectionsText(?string $content): array return $sections; } + /** + * Whether a (trimmed, non-empty) line outside a BT...ET text block is one of + * the few commands worth keeping for text positioning/extraction. + * + * Care should be taken to ensure a command being checked for *only* matches + * that command. For instance, a simple search for 'c' may also match the + * 'sc' command. See the command list in the formatContent() method above. + * Add more commands to keep here as you find them in weird PDFs! + */ + private function isKeptOutsideTextBlock(string $line): bool + { + return 'q' == $line[-1] || 'Q' == $line[-1] // save/restore graphics state + || preg_match('/(?getSectionsText($this->content); + $sections = $this->getSectionsText($this->getContent()); $current_font = $this->getDefaultFont($page); $current_font_size = 1; $current_text_leading = 0; diff --git a/src/Smalot/PdfParser/Parser.php b/src/Smalot/PdfParser/Parser.php index b051f114..6196e114 100644 --- a/src/Smalot/PdfParser/Parser.php +++ b/src/Smalot/PdfParser/Parser.php @@ -99,24 +99,43 @@ public function parseFile(string $filename): Document */ public function parseContent(string $content): Document { - // Create structure from raw data. - list($xref, $data) = $this->rawDataParser->parseData($content); + // Normalize the raw data and decode the cross-reference/trailer table. + list($xref, $pdfData) = $this->rawDataParser->parseHeaderAndXref($content); + + // The original (possibly un-trimmed) input is no longer needed; drop it + // so it can be freed once the normalized $pdfData copy is also released. + unset($content); if (isset($xref['trailer']['encrypt']) && false === $this->config->getIgnoreEncryption()) { throw new \Exception('Secured pdf file are currently not supported.'); } - if (empty($data)) { - throw new \Exception('Object list not found. Possible secured file.'); - } - // Create destination object. $document = new Document(); $this->objects = []; - foreach ($data as $id => $structure) { + // When content spooling is enabled, give the document an on-disk store + // so each object's decoded content can be moved out of memory as soon + // as the object is built (see parseObject()). + if ($this->config->getContentSpooling()) { + $document->setContentSpool(new ContentSpool()); + } + + // Stream the raw objects one at a time instead of building the whole + // raw object graph up front. Each structure is turned into a PDFObject + // and then goes out of scope before the next is parsed, so the largest + // transient structure - the full raw object array - is never held, + // which markedly lowers peak memory on large documents. + foreach ($this->rawDataParser->getObjectsStream($pdfData, $xref) as $id => $structure) { $this->parseObject($id, $structure, $document); - unset($data[$id]); + } + + // Object parsing is done; the raw PDF string is no longer needed and + // can be released before text extraction is performed by the caller. + unset($pdfData); + + if (empty($this->objects)) { + throw new \Exception('Object list not found. Possible secured file.'); } $document->setTrailer($this->parseTrailer($xref['trailer'], $document)); @@ -234,7 +253,12 @@ protected function parseObject(string $id, array $structure, ?Document $document } if (!isset($this->objects[$id])) { - $this->objects[$id] = PDFObject::factory($document, $header, $content, $this->config); + $object = PDFObject::factory($document, $header, $content, $this->config); + // Free the just-decoded content from memory by moving it to the + // document's on-disk spool (no-op unless content spooling is on). + // The local $content copy is released when this method returns. + $object->spoolContent(); + $this->objects[$id] = $object; } } diff --git a/src/Smalot/PdfParser/RawData/RawDataParser.php b/src/Smalot/PdfParser/RawData/RawDataParser.php index ec8d01e5..a06dff91 100644 --- a/src/Smalot/PdfParser/RawData/RawDataParser.php +++ b/src/Smalot/PdfParser/RawData/RawDataParser.php @@ -945,16 +945,21 @@ protected function getXrefData(string $pdfData, int $offset = 0, array $xref = [ } /** - * Parses PDF data and returns extracted data as array. + * Normalize the raw PDF data and decode the cross-reference/trailer data. + * + * Returns the xref/trailer data together with the normalized PDF data so + * callers (e.g. Parser) can inspect the trailer (for instance to detect + * encryption) and then stream the objects one at a time via + * getObjectsStream() instead of materializing them all at once. * * @param string $data PDF data to parse * - * @return array array of parsed PDF document objects + * @return array{0: array, 1: string} [$xref, $pdfData] * * @throws EmptyPdfException if empty PDF data given * @throws MissingPdfHeaderException if PDF data missing `%PDF-` header */ - public function parseData(string $data): array + public function parseHeaderAndXref(string $data): array { if (empty($data)) { throw new EmptyPdfException('Empty PDF data given.'); @@ -976,15 +981,31 @@ public function parseData(string $data): array $xref = $this->getXrefData($pdfData); } - // parse all document objects - $objects = []; + return [$xref, $pdfData]; + } + + /** + * Yield each indirect object's raw structure one at a time. + * + * Yielding (rather than returning a fully built array) lets the consumer + * build its own representation of an object and discard the raw structure + * before the next one is parsed, so the complete raw object graph - by far + * the largest transient structure when parsing a document - never has to be + * held in memory at once. + * + * @param string $pdfData normalized PDF data, as returned by parseHeaderAndXref() + * @param array $xref xref/trailer data, as returned by parseHeaderAndXref() + * + * @return \Generator raw object structure keyed by object reference + */ + public function getObjectsStream(string $pdfData, array $xref): \Generator + { foreach ($xref['xref'] as $obj => $offset) { - if (!isset($objects[$obj]) && ($offset > 0)) { - // decode objects with positive offset - $objects[$obj] = $this->getIndirectObject($pdfData, $xref, $obj, $offset, true); + // decode objects with positive offset; xref is keyed by object + // reference so every $obj is unique and decoded exactly once + if ($offset > 0) { + yield $obj => $this->getIndirectObject($pdfData, $xref, $obj, $offset, true); } } - - return [$xref, $objects]; } } diff --git a/src/Smalot/PdfParser/XObject/Form.php b/src/Smalot/PdfParser/XObject/Form.php index 8e60647f..8c69dc94 100644 --- a/src/Smalot/PdfParser/XObject/Form.php +++ b/src/Smalot/PdfParser/XObject/Form.php @@ -44,7 +44,7 @@ class Form extends Page public function getText(?Page $page = null): string { $header = new Header([], $this->document); - $contents = new PDFObject($this->document, $header, $this->content, $this->config); + $contents = new PDFObject($this->document, $header, $this->getContent(), $this->config); return $contents->getText($this); } diff --git a/tests/PHPUnit/Integration/ConfigTest.php b/tests/PHPUnit/Integration/ConfigTest.php index 7b0ecc8e..2ab177b3 100644 --- a/tests/PHPUnit/Integration/ConfigTest.php +++ b/tests/PHPUnit/Integration/ConfigTest.php @@ -37,6 +37,7 @@ use PHPUnitTests\TestCase; use Smalot\PdfParser\Config; +use Smalot\PdfParser\ContentSpool; class ConfigTest extends TestCase { @@ -55,4 +56,29 @@ public function testHorizontalOffset() $firstLine = explode("\n", $text)[0]; $this->assertEquals($reference, $firstLine); } + + /** + * Spooling decoded stream content to a temporary file must not change the + * extracted text or document details - it only lowers peak memory usage. + */ + public function testContentSpoolingProducesIdenticalOutput() + { + $filename = $this->rootDir.'/samples/bugs/Issue356.pdf'; + + $inMemoryConfig = new Config(); + $inMemoryConfig->setContentSpooling(false); + $inMemory = $this->getParserInstance($inMemoryConfig)->parseFile($filename); + + $spoolingConfig = new Config(); + $spoolingConfig->setContentSpooling(true); + $spooled = $this->getParserInstance($spoolingConfig)->parseFile($filename); + + // The spooling document must actually use an on-disk spool ... + $this->assertNull($inMemory->getContentSpool()); + $this->assertInstanceOf(ContentSpool::class, $spooled->getContentSpool()); + + // ... while producing exactly the same results. + $this->assertSame($inMemory->getText(), $spooled->getText()); + $this->assertSame($inMemory->getDetails(), $spooled->getDetails()); + } } diff --git a/tests/PHPUnit/Unit/ConfigTest.php b/tests/PHPUnit/Unit/ConfigTest.php index 133c7eb1..f881c0fb 100644 --- a/tests/PHPUnit/Unit/ConfigTest.php +++ b/tests/PHPUnit/Unit/ConfigTest.php @@ -75,4 +75,15 @@ public function testRetainImageContentSetterGetter(): void $this->fixture->setRetainImageContent(false); $this->assertFalse($this->fixture->getRetainImageContent()); } + + /** + * Tests setter and getter for content spooling. + */ + public function testContentSpoolingSetterGetter(): void + { + $this->assertFalse($this->fixture->getContentSpooling()); + + $this->fixture->setContentSpooling(true); + $this->assertTrue($this->fixture->getContentSpooling()); + } } diff --git a/tests/PHPUnit/Unit/ContentSpoolTest.php b/tests/PHPUnit/Unit/ContentSpoolTest.php new file mode 100644 index 00000000..5afadea9 --- /dev/null +++ b/tests/PHPUnit/Unit/ContentSpoolTest.php @@ -0,0 +1,52 @@ +store($a); + $refB = $spool->store($b); + + $this->assertIsArray($refA); + $this->assertIsArray($refB); + + // Chunks are appended, so the second starts right after the first + $this->assertSame([0, \strlen($a)], $refA); + $this->assertSame([\strlen($a), \strlen($b)], $refB); + + // Fetching does not depend on order and can be repeated + $this->assertSame($b, $spool->fetch($refB[0], $refB[1])); + $this->assertSame($a, $spool->fetch($refA[0], $refA[1])); + $this->assertSame($a, $spool->fetch($refA[0], $refA[1])); + } + + public function testStoringEmptyContentReturnsNull(): void + { + $spool = new ContentSpool(); + + $this->assertNull($spool->store('')); + } + + public function testFetchIsBinarySafe(): void + { + $spool = new ContentSpool(); + + $binary = random_bytes(2048)."\x00\x1f\x80\xff"; + $ref = $spool->store($binary); + + $this->assertIsArray($ref); + $this->assertSame($binary, $spool->fetch($ref[0], $ref[1])); + } +}