Harden main-PDF validation and response recovery

src/scansci_pdf/institutional/publisher_batch.py

@@ -38,11 +38,24 @@
 MIN_PDF_BYTES = 5_000
 MAX_BROWSER_CONCURRENCY = 4
 PDF_URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.IGNORECASE)
+_UTF8_REPLACEMENT_BYTES = b"\xef\xbf\xbd"
+_UTF8_REPLACEMENT_PDF_THRESHOLD = 8
+
+
+def _is_usable_pdf_response(body: bytes) -> bool:
+    """Reject binary PDFs that were lossy-decoded and re-encoded as UTF-8."""
+    return (
+        body[:5] == b"%PDF-"
+        and len(body) > MIN_PDF_BYTES
+        and body.count(_UTF8_REPLACEMENT_BYTES) < _UTF8_REPLACEMENT_PDF_THRESHOLD
+    )
 
 NON_ARTICLE_PDF_MARKERS = (
     "plain language summary",
     "p l a i n l a n g u a g e s u m m a r y",
     "electronic supporting information",
+    "electronic supplementary material",
+    "supporting information to https://doi.org",
     "we are delighted to inform you that your manuscript",
     "department of health and human services food and drug administration",
     "new drug application",
@@ -1458,7 +1471,15 @@ def on_response(response: Any) -> None:
                     captured["deferred_url"] = url
                     return
                 body = response.body()
-                if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+                if (
+                    body[:5] == b"%PDF-"
+                    and len(body) > MIN_PDF_BYTES
+                    and not _is_usable_pdf_response(body)
+                ):
+                    captured["deferred_url"] = url
+                    self._event(result, "pdf_response_utf8_corruption", url)
+                    return
+                if _is_usable_pdf_response(body):
                     captured["bytes"] = body
                     captured["url"] = url
             except Exception:
@@ -1493,7 +1514,14 @@ def on_response(response: Any) -> None:
                             captured["deferred_url"] = response_url
                         else:
                             body = response.body()
-                            if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+                            if (
+                                body[:5] == b"%PDF-"
+                                and len(body) > MIN_PDF_BYTES
+                                and not _is_usable_pdf_response(body)
+                            ):
+                                captured["deferred_url"] = response_url
+                                self._event(result, "pdf_response_utf8_corruption", response_url)
+                            elif _is_usable_pdf_response(body):
                                 captured["bytes"] = body
                                 captured["url"] = response.url
                     if not captured["bytes"]:
@@ -1777,8 +1805,10 @@ def _capture_browser_download(self, page: Any, pdf_url: str, result: DownloadRes
             download = download_info.value
             path = download.path()
             body = Path(path).read_bytes()
-            if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+            if _is_usable_pdf_response(body):
                 return body, str(getattr(download, "url", "") or pdf_url)
+            if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+                self._event(result, "pdf_download_utf8_corruption", pdf_url)
         except Exception as exc:
             self._event(result, "download_capture_error", f"{type(exc).__name__}: {exc}")
         return None, pdf_url
@@ -1865,7 +1895,7 @@ def _fetch_pdf_url(self, url: str, *, page: Any | None = None) -> tuple[bytes |
                 allow_redirects=True,
             )
             body = resp.content
-            if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+            if _is_usable_pdf_response(body):
                 return body, resp.url
         except Exception:
             return None, url

src/scansci_pdf/publisher_batch.py

@@ -37,6 +37,8 @@
     "plain language summary",
     "p l a i n l a n g u a g e s u m m a r y",
     "electronic supporting information",
+    "electronic supplementary material",
+    "supporting information to https://doi.org",
     "we are delighted to inform you that your manuscript",
     "department of health and human services food and drug administration",
     "new drug application",
@@ -59,6 +61,17 @@
 # treated as an IP block signal. 403 is ACS's block-page status; 429 is the
 # generic rate-limit/block code.
 _IP_BLOCK_STATUS_CODES = {403, 429}
+_UTF8_REPLACEMENT_BYTES = b"\xef\xbf\xbd"
+_UTF8_REPLACEMENT_PDF_THRESHOLD = 8
+
+
+def _is_usable_pdf_response(body: bytes) -> bool:
+    """Reject binary PDFs that were lossy-decoded and re-encoded as UTF-8."""
+    return (
+        body[:5] == b"%PDF-"
+        and len(body) > MIN_PDF_BYTES
+        and body.count(_UTF8_REPLACEMENT_BYTES) < _UTF8_REPLACEMENT_PDF_THRESHOLD
+    )
 
 # Reusable JS helpers injected into page.evaluate() calls
 _JS_VISIBLE = """(el) => {
@@ -2001,7 +2014,15 @@ def on_response(response: Any) -> None:
                     if self._is_ip_block_response(None, body):
                         captured["block_reason"] = "ip_blocked"
                         self._event(result, "ip_blocked_body", url)
-                if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+                if (
+                    body[:5] == b"%PDF-"
+                    and len(body) > MIN_PDF_BYTES
+                    and not _is_usable_pdf_response(body)
+                ):
+                    captured["deferred_url"] = url
+                    self._event(result, "pdf_response_utf8_corruption", url)
+                    return
+                if _is_usable_pdf_response(body):
                     captured["bytes"] = body
                     captured["url"] = url
             except Exception:
@@ -2038,7 +2059,14 @@ def on_response(response: Any) -> None:
                             captured["deferred_url"] = response_url
                         else:
                             body = response.body()
-                            if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+                            if (
+                                body[:5] == b"%PDF-"
+                                and len(body) > MIN_PDF_BYTES
+                                and not _is_usable_pdf_response(body)
+                            ):
+                                captured["deferred_url"] = response_url
+                                self._event(result, "pdf_response_utf8_corruption", response_url)
+                            elif _is_usable_pdf_response(body):
                                 captured["bytes"] = body
                                 captured["url"] = response.url
                     if not captured["bytes"]:
@@ -2337,8 +2365,10 @@ def _capture_browser_download(self, page: Any, pdf_url: str, result: DownloadRes
             download = download_info.value
             path = download.path()
             body = Path(path).read_bytes()
-            if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+            if _is_usable_pdf_response(body):
                 return body, str(getattr(download, "url", "") or pdf_url)
+            if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+                self._event(result, "pdf_download_utf8_corruption", pdf_url)
         except Exception as exc:
             self._event(result, "download_capture_error", f"{type(exc).__name__}: {exc}")
         return None, pdf_url
@@ -2476,7 +2506,7 @@ def _fetch_pdf_url(self, url: str, *, page: Any | None = None) -> tuple[bytes |
             if self._is_ip_block_response(resp.status_code, resp.text):
                 return None, resp.url, "ip_blocked"
             body = resp.content
-            if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES:
+            if _is_usable_pdf_response(body):
                 return body, resp.url, None
         except Exception:
             return None, url, None

tests/test_publisher_batch_pdf_validation.py

@@ -0,0 +1,73 @@
+import pytest
+
+from scansci_pdf.institutional.publisher_batch import (
+    PaperRecord as InstitutionalPaperRecord,
+)
+from scansci_pdf.institutional.publisher_batch import (
+    PublisherBatchDownloader as InstitutionalPublisherBatchDownloader,
+)
+from scansci_pdf.institutional.publisher_batch import (
+    _is_usable_pdf_response as institutional_is_usable_pdf_response,
+)
+from scansci_pdf.publisher_batch import (
+    PaperRecord,
+    PublisherBatchDownloader,
+    _is_usable_pdf_response,
+)
+
+
+@pytest.mark.parametrize(
+    ("downloader", "record_type"),
+    [
+        (PublisherBatchDownloader, PaperRecord),
+        (InstitutionalPublisherBatchDownloader, InstitutionalPaperRecord),
+    ],
+)
+def test_springer_electronic_supplementary_material_is_not_main_article(
+    downloader,
+    record_type,
+) -> None:
+    text = """
+    Electronic Supplementary Material
+    Example article title
+    Supporting information to https://doi.org/10.1007/s12274-022-4138-4
+    """
+    record = record_type(doi="10.1007/s12274-022-4138-4")
+
+    assert not downloader._text_matches_record(text, record)
+
+
+def test_main_article_with_matching_doi_is_verified() -> None:
+    text = """
+    Example article title
+    https://doi.org/10.1007/s12274-022-4138-4
+    Abstract
+    """
+    record = PaperRecord(doi="10.1007/s12274-022-4138-4")
+
+    assert PublisherBatchDownloader._text_matches_record(text, record)
+
+
+@pytest.mark.parametrize(
+    "validator",
+    [_is_usable_pdf_response, institutional_is_usable_pdf_response],
+)
+def test_pdf_response_rejects_utf8_replacement_byte_corruption(validator) -> None:
+    valid_pdf = b"%PDF-1.7\n" + (b"binary stream data\n" * 400)
+    corrupted_pdf = (
+        b"%PDF-1.7\n"
+        + (b"x\xef\xbf\xbdcorrupted flate stream\n" * 400)
+    )
+
+    assert validator(valid_pdf)
+    assert not validator(corrupted_pdf)
+
+
+@pytest.mark.parametrize(
+    "validator",
+    [_is_usable_pdf_response, institutional_is_usable_pdf_response],
+)
+def test_pdf_response_tolerates_one_replacement_sequence_in_metadata(validator) -> None:
+    pdf = b"%PDF-1.7\nmetadata \xef\xbf\xbd\n" + (b"binary stream data\n" * 400)
+
+    assert validator(pdf)