Preserve empty pages in extraction

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fast-pdf-extract"
-version = "0.6.2"
+version = "0.6.3"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

src/lib.rs

@@ -167,9 +167,8 @@ fn remove_non_english(pages: Pages) -> Pages {
                 after_len += paragraph.len();
             }
         }
-        if after_page.len() > 0 {
-            after_pages.push(after_page);
-        };
+        // Preserve every page so extracted pages align with the source PDF.
+        after_pages.push(after_page);
     }
 
     if (after_len as f64) < (0.9 * before_len as f64) {
@@ -186,17 +185,24 @@ fn get_pages(filename: String) -> PyResult<Vec<String>> {
     let mut pages = document
         .pages()
         .map_err(to_pyerr)?
-        .filter_map(|page| {
-            let stext_json = page
-                .ok()?
-                .to_text_page(
-                    TextPageFlags::COLLECT_STYLES | TextPageFlags::USE_GID_FOR_UNKNOWN_UNICODE,
-                )
-                .ok()?
-                .to_json(1.0)
-                .ok()?;
-            let stext_page: StextPage = serde_json::from_str(stext_json.as_str()).ok()?;
-            Some(get_styled_paragraphs(stext_page))
+        .map(|page| {
+            // Page extraction can fail for malformed PDFs. Keep an empty page
+            // instead of shifting all later page indexes.
+            let Ok(page) = page else {
+                return vec![];
+            };
+            let Ok(text_page) = page.to_text_page(
+                TextPageFlags::COLLECT_STYLES | TextPageFlags::USE_GID_FOR_UNKNOWN_UNICODE,
+            ) else {
+                return vec![];
+            };
+            let Ok(stext_json) = text_page.to_json(1.0) else {
+                return vec![];
+            };
+            let Ok(stext_page) = serde_json::from_str(stext_json.as_str()) else {
+                return vec![];
+            };
+            get_styled_paragraphs(stext_page)
         })
         .collect::<Pages>();
 

tests/test_files/bad-json.txt

@@ -1,3 +1,5 @@
+
+
 30th Annual General Meeting 
 Date: 26th December, 2020 Day: 
 Saturday 

tests/tests.py

@@ -37,6 +37,9 @@ def test_i32_box(self):
 
     def test_bad_json(self):
         pages = fast_pdf_extract.get_pages("tests/test_files/bad-json.pdf")
+        # Malformed pages are returned as empty pages to keep page indexes stable.
+        self.assertEqual(len(pages), 64)
+        self.assertEqual(pages[0], "")
         compare_snapshot("\n\n".join(pages), "tests/test_files/bad-json.txt")
 
     def test_only_images_unicode_jatalia(self):
@@ -46,3 +49,14 @@ def test_only_images_unicode_jatalia(self):
     def test_strikethrough(self):
         pages = fast_pdf_extract.get_pages("tests/test_files/strike.pdf")
         compare_snapshot("\n\n".join(pages), "tests/test_files/strike.txt")
+
+    def test_empty_pages_bank_of_maharashtra(self):
+        # Source PDF has 3 physical pages. One page has no extractable text,
+        # so get_pages should preserve page count and return an empty string for it.
+        pages = fast_pdf_extract.get_pages(
+            "tests/test_files/BANK_OF_MAHARASHTRA-532525-MARCH-2021.pdf"
+        )
+        self.assertEqual(len(pages), 3)
+        self.assertEqual(pages[1], "")
+        self.assertTrue(pages[0].strip())
+        self.assertTrue(pages[2].strip())