From 84333946c414623f0d33ac73cd81ff27724b15b1 Mon Sep 17 00:00:00 2001 From: Tobias Perelstein <5562156+tobocop2@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:51:43 -0400 Subject: [PATCH] feat: add scanned math pages exercising formula extraction Four synthetic pages rendered with matplotlib mathtext, rasterized at 300 DPI, binarized, and packed as CCITT G4 PDFs with no text layer, mirroring the shape of a scanned textbook page. One carries a 1.2 degree skew. A rasterized memo page joins them as the negative case: a scanned page with no mathematics must yield no formulas. The binaries follow the storage split: corpus.lock.json pins them, and the objects need a publish_corpus.py run to reach the bucket. --- ATTRIBUTIONS.md | 16 ++++++++++++++++ MATH_PROVENANCE.md | 7 ++++++- corpus.lock.json | 20 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/ATTRIBUTIONS.md b/ATTRIBUTIONS.md index 906e70b..24c80d6 100644 --- a/ATTRIBUTIONS.md +++ b/ATTRIBUTIONS.md @@ -190,3 +190,19 @@ Vendored WordPerfect-family test files. Per-file provenance (upstream path, revi - **License:** CC-BY-SA-4.0. ShareAlike is copyleft, so these are reference only. - **Used here:** 178 mathematics articles, reference. - **Modifications:** none. + +## Synthetic scanned math pages + +- **Citation:** none; generated for this corpus. +- **Source:** rendered with matplotlib mathtext, rasterized at 300 DPI, binarized, and packed as CCITT G4 PDFs with no text layer, to mirror the shape of a scanned textbook page (xberg issue #1385). +- **License:** MIT, as our own work. +- **Used here:** `pdf_scanned/synthetic_math_*.pdf`, four pages that carry display equations only as pixels. They exercise formula detection and recognition on scanned input, including a skewed scan. +- **Modifications:** not applicable. + +## xberg benchmark-harness fixtures + +- **Citation:** none; part of the xberg repository. +- **Source:** , `tools/benchmark-harness/fixtures/split/memo_marketing_form.pdf`, page 1, rasterized at 300 DPI and packed as a bilevel CCITT G4 scan. +- **License:** MIT, as our own work. +- **Used here:** `pdf_scanned/memo_prose_scanned.pdf`, the negative case: a scanned page with no mathematics, which must yield no formulas. +- **Modifications:** rasterized and binarized from the vector original. diff --git a/MATH_PROVENANCE.md b/MATH_PROVENANCE.md index 8519b45..f951439 100644 --- a/MATH_PROVENANCE.md +++ b/MATH_PROVENANCE.md @@ -160,7 +160,12 @@ That is real formula-level ground truth, independent of any extractor. `README.m | `xml/math/journal_pcbi_1005589_type_manuscript_58c02e.xml` | [PLOS Computational Biology](https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1005589&type=manuscript) | CC-BY-4.0 | MathML inside JATS , JATS Journa | | `xml/math/manuscript_933c4c.xml` | [Texture](https://raw.githubusercontent.com/substance/texture/master/data/kitchen-sink/manuscript.xml) | MIT | LaTeX in CDATA, JATS disp-formula / | -Committed: 136. The binary formats among them live in the bucket and are pinned by `corpus.lock.json`, following the storage split in `README.md`. +| `pdf_scanned/synthetic_math_interpolation_scanned.pdf` | Generated for this corpus (matplotlib mathtext, 300 DPI bilevel CCITT G4, no text layer) | MIT | Rasterized display equations (interpolation) | +| `pdf_scanned/synthetic_math_interpolation_skew_scanned.pdf` | Generated for this corpus; same page rotated 1.2 degrees before binarization | MIT | Rasterized display equations, skewed scan | +| `pdf_scanned/synthetic_math_matrix_scanned.pdf` | Generated for this corpus (matplotlib mathtext, 300 DPI bilevel CCITT G4, no text layer) | MIT | Rasterized display equations (determinants) | +| `pdf_scanned/synthetic_math_series_scanned.pdf` | Generated for this corpus (matplotlib mathtext, 300 DPI bilevel CCITT G4, no text layer) | MIT | Rasterized display equations (series) | + +Committed: 140. The binary formats among them live in the bucket and are pinned by `corpus.lock.json`, following the storage split in `README.md`. ## Reference only, not committed diff --git a/corpus.lock.json b/corpus.lock.json index 1c72042..568aa1b 100644 --- a/corpus.lock.json +++ b/corpus.lock.json @@ -1625,6 +1625,10 @@ "sha256": "0350f7e061a91df9c11548f061ac308360f0a1a0e8389c74e1b76c1bb70d5184", "size": 126609 }, + "pdf_scanned/memo_prose_scanned.pdf": { + "sha256": "9f75e84ec7e322b71715a626f06df1ce1ebe5bd7d921e6b046807d18a981b484", + "size": 42912 + }, "pdf_scanned/multi_page_scanned.pdf": { "sha256": "cc9455385e3fa2621664ef6fc3afc58f5df29ea37139e0fdd8d5dbb3b74073e1", "size": 2527028 @@ -1677,6 +1681,22 @@ "sha256": "2da8b562f74b778fcba92efe07fb8892aad1b279dbf84daba412ee148a4b56c8", "size": 1233386 }, + "pdf_scanned/synthetic_math_interpolation_scanned.pdf": { + "sha256": "99dd205450e6dfbd638a2a4ce03058ba595436acdafe0e736e126753c5b71834", + "size": 14894 + }, + "pdf_scanned/synthetic_math_interpolation_skew_scanned.pdf": { + "sha256": "553023190b90aa46e6196e74a55bf71939853db478c3109aea3ef1e90c4e4fe7", + "size": 15586 + }, + "pdf_scanned/synthetic_math_matrix_scanned.pdf": { + "sha256": "8505715c5362ce8eb733cd3cc783d27cffa218d51dabbf2a971820cc37b09090", + "size": 10464 + }, + "pdf_scanned/synthetic_math_series_scanned.pdf": { + "sha256": "6dcf181fa25d84409f7d99531bb4f1cd6607f003b73f81de8dbeb34e91fb3074", + "size": 11751 + }, "ppt/simple.ppt": { "sha256": "731b1a5478f36b996e02a0a2151df3de46c1c8293b3f0e10e7eda8aa24f249ab", "size": 504320