diff --git a/src/scansci_pdf/institutional/publisher_batch.py b/src/scansci_pdf/institutional/publisher_batch.py index 48d18b6..d8d3a16 100644 --- a/src/scansci_pdf/institutional/publisher_batch.py +++ b/src/scansci_pdf/institutional/publisher_batch.py @@ -38,11 +38,24 @@ MIN_PDF_BYTES = 5_000 MAX_BROWSER_CONCURRENCY = 4 PDF_URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.IGNORECASE) +_UTF8_REPLACEMENT_BYTES = b"\xef\xbf\xbd" +_UTF8_REPLACEMENT_PDF_THRESHOLD = 8 + + +def _is_usable_pdf_response(body: bytes) -> bool: + """Reject binary PDFs that were lossy-decoded and re-encoded as UTF-8.""" + return ( + body[:5] == b"%PDF-" + and len(body) > MIN_PDF_BYTES + and body.count(_UTF8_REPLACEMENT_BYTES) < _UTF8_REPLACEMENT_PDF_THRESHOLD + ) NON_ARTICLE_PDF_MARKERS = ( "plain language summary", "p l a i n l a n g u a g e s u m m a r y", "electronic supporting information", + "electronic supplementary material", + "supporting information to https://doi.org", "we are delighted to inform you that your manuscript", "department of health and human services food and drug administration", "new drug application", @@ -1458,7 +1471,15 @@ def on_response(response: Any) -> None: captured["deferred_url"] = url return body = response.body() - if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + if ( + body[:5] == b"%PDF-" + and len(body) > MIN_PDF_BYTES + and not _is_usable_pdf_response(body) + ): + captured["deferred_url"] = url + self._event(result, "pdf_response_utf8_corruption", url) + return + if _is_usable_pdf_response(body): captured["bytes"] = body captured["url"] = url except Exception: @@ -1493,7 +1514,14 @@ def on_response(response: Any) -> None: captured["deferred_url"] = response_url else: body = response.body() - if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + if ( + body[:5] == b"%PDF-" + and len(body) > MIN_PDF_BYTES + and not _is_usable_pdf_response(body) + ): + captured["deferred_url"] = response_url + self._event(result, "pdf_response_utf8_corruption", response_url) + elif _is_usable_pdf_response(body): captured["bytes"] = body captured["url"] = response.url if not captured["bytes"]: @@ -1777,8 +1805,10 @@ def _capture_browser_download(self, page: Any, pdf_url: str, result: DownloadRes download = download_info.value path = download.path() body = Path(path).read_bytes() - if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + if _is_usable_pdf_response(body): return body, str(getattr(download, "url", "") or pdf_url) + if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + self._event(result, "pdf_download_utf8_corruption", pdf_url) except Exception as exc: self._event(result, "download_capture_error", f"{type(exc).__name__}: {exc}") return None, pdf_url @@ -1865,7 +1895,7 @@ def _fetch_pdf_url(self, url: str, *, page: Any | None = None) -> tuple[bytes | allow_redirects=True, ) body = resp.content - if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + if _is_usable_pdf_response(body): return body, resp.url except Exception: return None, url diff --git a/src/scansci_pdf/publisher_batch.py b/src/scansci_pdf/publisher_batch.py index e6bad28..3fc81d9 100644 --- a/src/scansci_pdf/publisher_batch.py +++ b/src/scansci_pdf/publisher_batch.py @@ -37,6 +37,8 @@ "plain language summary", "p l a i n l a n g u a g e s u m m a r y", "electronic supporting information", + "electronic supplementary material", + "supporting information to https://doi.org", "we are delighted to inform you that your manuscript", "department of health and human services food and drug administration", "new drug application", @@ -59,6 +61,17 @@ # treated as an IP block signal. 403 is ACS's block-page status; 429 is the # generic rate-limit/block code. _IP_BLOCK_STATUS_CODES = {403, 429} +_UTF8_REPLACEMENT_BYTES = b"\xef\xbf\xbd" +_UTF8_REPLACEMENT_PDF_THRESHOLD = 8 + + +def _is_usable_pdf_response(body: bytes) -> bool: + """Reject binary PDFs that were lossy-decoded and re-encoded as UTF-8.""" + return ( + body[:5] == b"%PDF-" + and len(body) > MIN_PDF_BYTES + and body.count(_UTF8_REPLACEMENT_BYTES) < _UTF8_REPLACEMENT_PDF_THRESHOLD + ) # Reusable JS helpers injected into page.evaluate() calls _JS_VISIBLE = """(el) => { @@ -2001,7 +2014,15 @@ def on_response(response: Any) -> None: if self._is_ip_block_response(None, body): captured["block_reason"] = "ip_blocked" self._event(result, "ip_blocked_body", url) - if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + if ( + body[:5] == b"%PDF-" + and len(body) > MIN_PDF_BYTES + and not _is_usable_pdf_response(body) + ): + captured["deferred_url"] = url + self._event(result, "pdf_response_utf8_corruption", url) + return + if _is_usable_pdf_response(body): captured["bytes"] = body captured["url"] = url except Exception: @@ -2038,7 +2059,14 @@ def on_response(response: Any) -> None: captured["deferred_url"] = response_url else: body = response.body() - if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + if ( + body[:5] == b"%PDF-" + and len(body) > MIN_PDF_BYTES + and not _is_usable_pdf_response(body) + ): + captured["deferred_url"] = response_url + self._event(result, "pdf_response_utf8_corruption", response_url) + elif _is_usable_pdf_response(body): captured["bytes"] = body captured["url"] = response.url if not captured["bytes"]: @@ -2337,8 +2365,10 @@ def _capture_browser_download(self, page: Any, pdf_url: str, result: DownloadRes download = download_info.value path = download.path() body = Path(path).read_bytes() - if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + if _is_usable_pdf_response(body): return body, str(getattr(download, "url", "") or pdf_url) + if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + self._event(result, "pdf_download_utf8_corruption", pdf_url) except Exception as exc: self._event(result, "download_capture_error", f"{type(exc).__name__}: {exc}") return None, pdf_url @@ -2476,7 +2506,7 @@ def _fetch_pdf_url(self, url: str, *, page: Any | None = None) -> tuple[bytes | if self._is_ip_block_response(resp.status_code, resp.text): return None, resp.url, "ip_blocked" body = resp.content - if body[:5] == b"%PDF-" and len(body) > MIN_PDF_BYTES: + if _is_usable_pdf_response(body): return body, resp.url, None except Exception: return None, url, None diff --git a/tests/test_publisher_batch_pdf_validation.py b/tests/test_publisher_batch_pdf_validation.py new file mode 100644 index 0000000..c57167e --- /dev/null +++ b/tests/test_publisher_batch_pdf_validation.py @@ -0,0 +1,73 @@ +import pytest + +from scansci_pdf.institutional.publisher_batch import ( + PaperRecord as InstitutionalPaperRecord, +) +from scansci_pdf.institutional.publisher_batch import ( + PublisherBatchDownloader as InstitutionalPublisherBatchDownloader, +) +from scansci_pdf.institutional.publisher_batch import ( + _is_usable_pdf_response as institutional_is_usable_pdf_response, +) +from scansci_pdf.publisher_batch import ( + PaperRecord, + PublisherBatchDownloader, + _is_usable_pdf_response, +) + + +@pytest.mark.parametrize( + ("downloader", "record_type"), + [ + (PublisherBatchDownloader, PaperRecord), + (InstitutionalPublisherBatchDownloader, InstitutionalPaperRecord), + ], +) +def test_springer_electronic_supplementary_material_is_not_main_article( + downloader, + record_type, +) -> None: + text = """ + Electronic Supplementary Material + Example article title + Supporting information to https://doi.org/10.1007/s12274-022-4138-4 + """ + record = record_type(doi="10.1007/s12274-022-4138-4") + + assert not downloader._text_matches_record(text, record) + + +def test_main_article_with_matching_doi_is_verified() -> None: + text = """ + Example article title + https://doi.org/10.1007/s12274-022-4138-4 + Abstract + """ + record = PaperRecord(doi="10.1007/s12274-022-4138-4") + + assert PublisherBatchDownloader._text_matches_record(text, record) + + +@pytest.mark.parametrize( + "validator", + [_is_usable_pdf_response, institutional_is_usable_pdf_response], +) +def test_pdf_response_rejects_utf8_replacement_byte_corruption(validator) -> None: + valid_pdf = b"%PDF-1.7\n" + (b"binary stream data\n" * 400) + corrupted_pdf = ( + b"%PDF-1.7\n" + + (b"x\xef\xbf\xbdcorrupted flate stream\n" * 400) + ) + + assert validator(valid_pdf) + assert not validator(corrupted_pdf) + + +@pytest.mark.parametrize( + "validator", + [_is_usable_pdf_response, institutional_is_usable_pdf_response], +) +def test_pdf_response_tolerates_one_replacement_sequence_in_metadata(validator) -> None: + pdf = b"%PDF-1.7\nmetadata \xef\xbf\xbd\n" + (b"binary stream data\n" * 400) + + assert validator(pdf)