From e88141adf6dd3b33092fa3f657eae8c1f202d87b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 17:09:59 +0000 Subject: [PATCH 1/2] Add exhaustive text-faithfulness evaluator (oracle vs export surfaces) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/text_faithfulness_eval.py compares independent oracle extractors (pypdfium2 primary, blind pdfplumber second opinion) against Warp's two exported text surfaces — the OC PAWLS token stream (page-level) and the engine block_text/content (doc-level) — via multiset token recall with a re-tokenization forgiveness pass, so only real text loss is flagged. Doc-parallel with JSONL checkpointing over every PDF in the repo. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017ctAeFX3qYwAs1fhupd8xR --- scripts/text_faithfulness_eval.py | 384 ++++++++++++++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 scripts/text_faithfulness_eval.py diff --git a/scripts/text_faithfulness_eval.py b/scripts/text_faithfulness_eval.py new file mode 100644 index 0000000..13698f3 --- /dev/null +++ b/scripts/text_faithfulness_eval.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python +"""Exhaustive text-faithfulness evaluation for the Warp-Ingest pipeline. + +Question answered: **does Warp ever LOSE text?** Structure (labels, hierarchy, +reading order) is out of scope — only content loss is scored. + +Oracle: pypdfium2 (PDFium — Chrome's PDF text stack), a text extractor fully +independent of the pdfplumber/pdfminer front-end, plus optionally a blind +pdfplumber flat extract as a second opinion. For every page, each oracle's +token multiset is compared against Warp's exported text surfaces: + + * ``pawls`` — the OpenContracts PAWLS token stream (rebuilt from the + front-end XHTML; page-level comparison). + * ``content`` — the OC export's ``content`` field == the engine's + ``block_text`` (the json/html render surface; doc-level + comparison, since blocks may move across page boundaries). + +Scoring is multiset token recall with a *re-tokenization forgiveness* pass: +an oracle token missing from Warp's token multiset is forgiven when its +whitespace-free form still appears verbatim in Warp's whitespace-free page +character stream (covers different word segmentation, glued/split words and +soft-hyphen joins — the text is present, just tokenized differently). Missing +tokens containing at least one alphanumeric are *material*; punctuation-only +tokens are counted separately as trivial. + +Usage:: + + uv run python scripts/text_faithfulness_eval.py \ + --out eval_results.jsonl [--workers 3] [--docs substr ...] [--limit N] + +Results are checkpointed one JSON line per document; a re-run skips documents +already present in the output file. Summarize with --summarize. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import unicodedata +from collections import Counter +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] + +# --------------------------------------------------------------------------- +# text normalization +# --------------------------------------------------------------------------- + +# characters different extractors render differently but that are the "same" +# text for faithfulness purposes +_CHAR_MAP = { + 0x00AD: None, # soft hyphen + 0x200B: None, # zero-width space + 0x200C: None, # zero-width non-joiner + 0x200D: None, # zero-width joiner + 0xFEFF: None, # BOM / zero-width no-break + 0xFFFE: None, # PDFium end-of-line marker artifacts + 0x2018: ord("'"), + 0x2019: ord("'"), + 0x201C: ord('"'), + 0x201D: ord('"'), + 0x2010: ord("-"), # hyphen + 0x2011: ord("-"), # non-breaking hyphen + 0x2012: ord("-"), # figure dash + 0x2013: ord("-"), # en dash + 0x2014: ord("-"), # em dash + 0x2015: ord("-"), # horizontal bar + 0x00A0: ord(" "), # nbsp + 0x2022: None, # bullet (glyph choice varies across extractors) + 0x00B7: None, # middle dot + 0xF0B7: None, # private-use bullet (Symbol font) + 0xF0A7: None, # private-use bullet + 0x25CF: None, # black circle + 0x25AA: None, # small black square + 0x25A0: None, # black square + 0x25E6: None, # white bullet + 0x2219: None, # bullet operator + 0x25BC: None, # down triangle (often decorative) + 0x25B2: None, # up triangle +} + + +def norm_text(s: str) -> str: + """Normalize extractor output for comparison (case/ligature/dash folded).""" + s = unicodedata.normalize("NFKC", s) + s = s.translate(_CHAR_MAP) + # drop remaining control / format chars except whitespace + s = "".join( + c if not unicodedata.category(c).startswith(("Cc", "Cf")) else " " for c in s + ) + return s.lower() + + +def toks(s: str) -> list[str]: + return norm_text(s).split() + + +_ALNUM = re.compile(r"[a-z0-9]") + + +def material(token: str) -> bool: + """A missing token matters if it carries any alphanumeric content.""" + return bool(_ALNUM.search(token)) + + +# --------------------------------------------------------------------------- +# scoring +# --------------------------------------------------------------------------- + + +def score(oracle_tokens: list[str], warp_tokens: list[str], warp_streams: list[str]): + """Multiset recall of oracle tokens in warp tokens, with substring forgiveness. + + ``warp_streams`` are whitespace-free character streams (page-level and/or + doc-level) used to forgive re-tokenization: a deficit token is forgiven if + it appears as a contiguous substring beyond what token matches account for. + """ + want = Counter(oracle_tokens) + have = Counter(warp_tokens) + matched = sum(min(c, have.get(t, 0)) for t, c in want.items()) + missing: Counter[str] = Counter() + forgiven = 0 + for t, c in want.items(): + deficit = c - min(c, have.get(t, 0)) + if not deficit: + continue + t_sub = t.strip("-") or t + extra = 0 + for stream in warp_streams: + occ = stream.count(t_sub) if t_sub else 0 + extra = max(extra, occ - have.get(t, 0)) + if extra >= deficit: + break + f = min(deficit, max(0, extra)) + forgiven += f + if deficit - f: + missing[t] += deficit - f + n = sum(want.values()) + miss_mat = sum(c for t, c in missing.items() if material(t)) + miss_triv = sum(c for t, c in missing.items() if not material(t)) + return { + "n_oracle": n, + "matched": matched, + "forgiven": forgiven, + "missing_material": miss_mat, + "missing_trivial": miss_triv, + "recall_strict": round(matched / n, 6) if n else 1.0, + "recall_effective": round((matched + forgiven) / n, 6) if n else 1.0, + "missing_samples": [ + {"token": t, "count": c} + for t, c in sorted( + ((t, c) for t, c in missing.items() if material(t)), + key=lambda x: -x[1], + )[:25] + ], + } + + +# --------------------------------------------------------------------------- +# oracles +# --------------------------------------------------------------------------- + + +def pdfium_pages(path: str) -> list[str]: + import pypdfium2 as pdfium + + pdf = pdfium.PdfDocument(path) + try: + out = [] + for page in pdf: + tp = page.get_textpage() + out.append(tp.get_text_bounded() or "") + tp.close() + page.close() + return out + finally: + pdf.close() + + +def plumber_pages(path: str) -> list[str]: + import pdfplumber + + with pdfplumber.open(path) as pdf: + return [(p.extract_text() or "") for p in pdf.pages] + + +# --------------------------------------------------------------------------- +# per-document evaluation (worker) +# --------------------------------------------------------------------------- + + +def eval_doc(path: str, oracles: list[str]) -> dict: + os.environ.setdefault("WARP_FE_WORKERS", "1") # no nested pools in workers + + from warp_ingest.ingestor.opencontracts_exporter import to_opencontracts_export + from warp_ingest.ingestor.pdf_ingestor import parse_blocks, parse_pdf + + rec: dict = {"doc": os.path.relpath(path, REPO)} + try: + xhtml = parse_pdf(path, {"render_format": "all"}) + blocks, *_ = parse_blocks(xhtml, render_format="all", parse_pages=()) + export = to_opencontracts_export( + xhtml, blocks, pdf_bytes=Path(path).read_bytes() + ) + except Exception as e: # noqa: BLE001 - a crash IS a total-text-loss finding + rec["error"] = f"{type(e).__name__}: {e}" + return rec + + pawls = export["pawls_file_content"] + warp_page_tokens = [toks(" ".join(t["text"] for t in pg["tokens"])) for pg in pawls] + warp_page_streams = ["".join(ts) for ts in warp_page_tokens] + warp_doc_stream = "".join(warp_page_streams) + content_tokens = toks(export["content"]) + content_stream = "".join(content_tokens) + + rec["pages"] = len(pawls) + rec["oracles"] = {} + for name in oracles: + try: + pages = pdfium_pages(path) if name == "pdfium" else plumber_pages(path) + except Exception as e: # noqa: BLE001 + rec["oracles"][name] = {"error": f"{type(e).__name__}: {e}"} + continue + o_page_tokens = [toks(t) for t in pages] + o_all = [t for ts in o_page_tokens for t in ts] + + # pawls surface: page-level (fall back to doc stream for cross-page moves) + page_rows = [] + agg = Counter() + for i, ots in enumerate(o_page_tokens): + wts = warp_page_tokens[i] if i < len(warp_page_tokens) else [] + ws = warp_page_streams[i] if i < len(warp_page_streams) else "" + s = score(ots, wts, [ws, warp_doc_stream]) + for k in ( + "n_oracle", + "matched", + "forgiven", + "missing_material", + "missing_trivial", + ): + agg[k] += s[k] + if s["missing_material"]: + page_rows.append({"page": i, **s}) + n = agg["n_oracle"] + rec["oracles"][name] = { + "pawls": { + **dict(agg), + "recall_strict": round(agg["matched"] / n, 6) if n else 1.0, + "recall_effective": ( + round((agg["matched"] + agg["forgiven"]) / n, 6) if n else 1.0 + ), + "flagged_pages": page_rows[:2000], + }, + "content": score(o_all, content_tokens, [content_stream]), + } + return rec + + +def _worker(args): + path, oracles = args + try: + return eval_doc(path, oracles) + except Exception as e: # noqa: BLE001 + return { + "doc": os.path.relpath(path, REPO), + "error": f"worker: {type(e).__name__}: {e}", + } + + +# --------------------------------------------------------------------------- +# driver +# --------------------------------------------------------------------------- + + +def find_pdfs() -> list[str]: + roots = [REPO / "tests" / "fixtures", REPO / "files"] + out: list[str] = [] + for r in roots: + out.extend(str(p) for p in r.rglob("*.pdf")) + return sorted(set(out)) + + +def page_count(path: str) -> int: + import pypdfium2 as pdfium + + pdf = pdfium.PdfDocument(path) + try: + return len(pdf) + finally: + pdf.close() + + +def summarize(out_path: Path, flag_threshold: float) -> int: + rows = [json.loads(l) for l in out_path.read_text().splitlines() if l.strip()] + print(f"{len(rows)} documents evaluated") + bad = 0 + for surface in ("pawls", "content"): + for oracle in ("pdfium", "plumber"): + tot = Counter() + worst: list[tuple[float, str]] = [] + for r in rows: + o = (r.get("oracles") or {}).get(oracle) + if not o or "error" in o: + continue + s = o[surface] + for k in ( + "n_oracle", + "matched", + "forgiven", + "missing_material", + "missing_trivial", + ): + tot[k] += s[k] + worst.append((s["recall_effective"], r["doc"])) + if not tot["n_oracle"]: + continue + eff = (tot["matched"] + tot["forgiven"]) / tot["n_oracle"] + print( + f"\n== {surface} vs {oracle}: eff-recall {eff:.6f} " + f"({tot['missing_material']} material / {tot['missing_trivial']} trivial " + f"missing of {tot['n_oracle']})" + ) + for rec_, doc in sorted(worst)[:12]: + mark = " <-- FLAG" if rec_ < flag_threshold else "" + if mark: + bad += 1 + print(f" {rec_:.6f} {doc}{mark}") + errs = [r for r in rows if "error" in r] + for r in errs: + print(f"ERROR {r['doc']}: {r['error']}") + return 1 if (bad or errs) else 0 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--out", default=str(REPO / "eval_results.jsonl")) + ap.add_argument("--workers", type=int, default=3) + ap.add_argument("--oracles", default="pdfium,plumber") + ap.add_argument("--docs", nargs="*", help="substring filters on doc path") + ap.add_argument("--limit", type=int) + ap.add_argument("--summarize", action="store_true") + ap.add_argument("--flag-threshold", type=float, default=0.995) + args = ap.parse_args() + + out_path = Path(args.out) + if args.summarize: + return summarize(out_path, args.flag_threshold) + + oracles = [o for o in args.oracles.split(",") if o] + pdfs = find_pdfs() + if args.docs: + pdfs = [p for p in pdfs if any(d in p for d in args.docs)] + done = set() + if out_path.exists(): + for line in out_path.read_text().splitlines(): + if line.strip(): + done.add(json.loads(line)["doc"]) + pdfs = [p for p in pdfs if os.path.relpath(p, REPO) not in done] + pdfs.sort(key=page_count, reverse=True) # big docs first for load balance + if args.limit: + pdfs = pdfs[: args.limit] + print(f"{len(pdfs)} documents to evaluate ({len(done)} already done)") + + os.environ["WARP_FE_WORKERS"] = "1" + with out_path.open("a") as fh, ProcessPoolExecutor(max_workers=args.workers) as ex: + futs = {ex.submit(_worker, (p, oracles)): p for p in pdfs} + for i, fut in enumerate(as_completed(futs), 1): + rec = fut.result() + fh.write(json.dumps(rec) + "\n") + fh.flush() + status = f"ERROR {rec.get('error')}" if "error" in rec else "" + if not status: + pw = rec["oracles"].get("pdfium", {}).get("pawls", {}) + status = f"pawls eff {pw.get('recall_effective')}" + print(f"[{i}/{len(futs)}] {rec['doc']}: {status}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From ce80e837e327b3440a264d8985c1acdb33e9fe1a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 17:34:05 +0000 Subject: [PATCH 2/2] Add text-faithfulness findings report, raw results, and CI floor test Exhaustive evaluation over all 179 repo PDFs (5,850 pages, 2.59M oracle tokens): PAWLS-surface effective recall 0.999855 vs PDFium and 0.999893 vs pdfplumber, with every one of the 61 flagged pages triaged. No material body-text loss found; residual genuine losses are ~10 tokens of rotated margin text and ~6 embedded-layer tokens on OCR-rerouted scan pages. Report documents the garble classes (overprint char-doubling, letter-space char-explosion), the content-surface furniture stripping, and the repeated-signature-block stripping finding, with recommended follow-ups. tests/test_text_faithfulness.py locks the PAWLS >= 0.999 effective-recall floor vs the PDFium oracle on the canonical fixtures. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017ctAeFX3qYwAs1fhupd8xR --- docs/text_faithfulness_eval.md | 129 ++++++++++++++++ docs/text_faithfulness_eval_results.jsonl | 179 ++++++++++++++++++++++ tests/test_text_faithfulness.py | 57 +++++++ 3 files changed, 365 insertions(+) create mode 100644 docs/text_faithfulness_eval.md create mode 100644 docs/text_faithfulness_eval_results.jsonl create mode 100644 tests/test_text_faithfulness.py diff --git a/docs/text_faithfulness_eval.md b/docs/text_faithfulness_eval.md new file mode 100644 index 0000000..e931718 --- /dev/null +++ b/docs/text_faithfulness_eval.md @@ -0,0 +1,129 @@ +# Text-faithfulness evaluation: does Warp-Ingest ever lose text? + +**Date:** 2026-07-06 · **Scope:** every PDF in the repo — **179 documents / 5,850 +pages** (`tests/fixtures/**` incl. the 50 S-1 bodies, contracts, hetero-100, +legal-100, OCR fixtures, plus `files/pdf/`) · **Tool:** +`scripts/text_faithfulness_eval.py` · **Raw results:** +`docs/text_faithfulness_eval_results.jsonl` + +## Question + +Structure (labels, hierarchy, reading order) may be imperfect — but the parser +must never **lose** text. This evaluation compares Warp's exported text against +known-good independent extractors, exhaustively, and triages every page that +shows a material deficit. + +## Method + +- **Oracles** (what a page "really says"): + - **pypdfium2** (PDFium — Chrome's PDF text stack). Fully independent of + Warp's pdfplumber/pdfminer front-end, so agreement is meaningful evidence. + - **pdfplumber flat extract** as a second opinion (same library family as the + front-end, so it checks the pipeline *after* extraction). +- **Warp surfaces** (what Warp exports): + - **`pawls`** — the OpenContracts PAWLS token stream, rebuilt from the + front-end XHTML. This is the export's designed no-loss surface (compared + page-by-page). + - **`content`** — the OC export `content` field == the engine's + `block_text`, i.e. the `json`/`html` render surface (compared doc-level). +- **Scoring**: multiset token recall after normalization (NFKC, case, quote/dash + folding, bullet-glyph removal), with a *re-tokenization forgiveness* pass — a + missing token is forgiven when its whitespace-free form still appears + contiguously in Warp's whitespace-free character stream (different word + segmentation is not loss). Missing tokens with no alphanumeric content are + counted separately as trivial. + +## Headline results (2,586,748 oracle tokens) + +| surface | oracle | strict recall | **effective recall** | material missing | docs fully clean | flagged pages | +|---|---|---|---|---|---|---| +| pawls | pdfium | 0.9899 | **0.999855** | 375 | 136 / 179 | 61 / 5,850 | +| pawls | pdfplumber | 0.9947 | **0.999893** | 282 | 144 / 179 | 44 / 5,850 | +| content | pdfium | 0.9865 | 0.997491 | 5,729 | 57 / 179 | — | +| content | pdfplumber | 0.9900 | 0.997497 | 5,826 | 59 / 179 | — | + +**Verdict: no material body-text loss was found on the PAWLS surface.** Every +one of the 61 flagged pages (375 tokens, 0.0145% of the corpus) was triaged — +by an automated classifier over re-parsed pages plus manual inspection of every +class — and decomposes into the categories below. The residual *genuine* losses +are a handful of tokens of page-margin/decorative text, detailed in §"Genuine +losses". + +## PAWLS surface triage (the no-loss guarantee) + +Classifier over all flagged pages (token-weighted; big-doc singletons +pattern-matched to the same classes and spot-verified): + +| class | ~tokens | verdict | example | +|---|---|---|---| +| **Doubled characters** | 126 | *garble, not absence* — fake-bold overprint (text drawn twice at a sub-pixel offset) comes through pdfplumber as `ccaappiittaall::` where PDFium dedupes to `capital:`. The words are unreadable/unsearchable in that form. | `GBFR22-value-creation-model_p1` infographic; the diagonal “SUBJECT TO COMPLETION” overlay on draft S-1 cover pages (the pdfplumber-oracle mirror of this class is the reversed/doubled watermark tokens: `seitiruces`, `sseeiittiirruucceess`) | +| **Oracle-side gluing** | 62+ | *not Warp's fault* — PDFium welds vertically-stacked table-header cells into one token (`dateexhibit`, `numberfiled`, `capitalretained`, `holdersdividend`); Warp has both words separately. Verified by splitting each glued token into two tokens present on the same Warp page. | `files/pdf/primary-document.pdf` exhibit index pp.114–117; forbright/liftoff S-1 balance-sheet headers | +| **Per-character explosion** | 45 | *garble, not absence* — wide letter-spaced headings and rotated/diagonal banner text arrive as single-char tokens (`t`,`e`,`i`,`r`,…). Text present, words destroyed. | `chapter_10_p2` (letter-spaced heads), infleqtion/exyn/swarmer S-1 p1–2 diagonal red banner (`changed.`) | +| **Unmapped CID glyphs** | 10 | *decode gap* — pdfminer emits `(cid:47)…` for fonts whose cmap it can't decode; PDFium decodes them. | `GBFR22` display-font labels | +| **Soft-hyphen compounds** | ~15 | *hyphen lost, words kept* — the visible hyphen is encoded U+00AD (`low\xadcarbon`); Warp splits the pair, dropping only the hyphen glyph. | ESEF annual-report pp.22–23, `t-co2e` | +| **Sub/superscript & math splits** | ~20 | *present, tokenized apart* — `0.01` + `%`, `co` + `2`, `kh2po4`, math glyph-order (`γ5q`, `~x∈v`); also accent-decoding disagreements (`tramèr` vs `tramer`). Spot-checked present. | `ar2024e_p49` donut-chart labels, physics/CS papers | +| **Rotated margin text** | ~10 | **GENUINE LOSS** — 90°-rotated margin/sidebar text is absent from Warp output (pdfplumber's own word extraction can't see it either; a pdfminer-stack limitation). All instances are marginal/decorative, not body text. | arXiv sidebar `arXiv:2411.18478v1 [cs.CL]`; Federal Register processing keys (`jspears on DSK121TN23PROD`, `E:\FR\FM\27MYR2.SGM`) | +| **Sparse-scan OCR reroute** | ~6 | **GENUINE LOSS (with huge offsetting gain)** — a scanned page with a tiny embedded text layer (<4 lines) is rerouted to OCR; the embedded tokens are discarded. generate_bio p111: embedded layer is just the folio `C-9`; OCR recovered **299 tokens** of drawing text the oracle can't see, but the folio was dropped. Same class: `fw_garver` p4, `fw_southern_computer` (`order:`). | `generate_bio_s1__ex1015` p111, FortWorth scans (whose remaining “missing” gibberish — `4popr`, `uoq` — is oracle noise from the scans' junk text layer) | + +## Content surface (`block_text` / json render) + +Doc-level effective recall 0.9975. The deficit vocabulary is dominated by the +engine's **deliberate repeated-furniture stripping** (`should_ignore_line`, +`find_true_header_footers`): running `TABLE OF CONTENTS` banners (724 tokens), +EDGAR `Source: … 10-Q, 11/14/2019` footers, `The accompanying notes are an +integral part …` banners, page folios, dot-leader runs (the trivial-token +column). Duplicated TOC lines are also deduped (`sample.pdf`). This is by +design and the text remains in the PAWLS layer — but consumers of the +`json`/`html`/markdown renders should know it is *removed there*. + +One class deserves attention rather than a shrug: **repeated signature blocks**. +On `neutron_lime_s1__exhibit45sx1` (295-page guarantee supplement with ~177 +near-identical consent pages), the signature lines `NEUTRON HOLDINGS, INC. / +By: /s/ Chris Maher / Title: Authorized Signatory` are stripped from +`block_text` on every page — verified present in the XHTML/PAWLS layer on all +177 pages, absent from all blocks. The furniture heuristics treat +same-position-every-page repetition as headers/footers, and a mass-signature +exhibit meets that signature. For a legal parser, `/s/` execution lines are +substantive content; this is the one place the deliberate stripping deletes +text a consumer may care about (901 tokens on that doc). Recommended follow-up +below. + +## Genuine losses — complete inventory + +Across 5,850 pages and 2.59M tokens, the text Warp actually fails to carry on +its no-loss (PAWLS) surface: + +1. **~10 tokens of 90°-rotated margin text** on 4 pages (arXiv sidebar, Federal + Register margin keys) — pdfminer-stack limitation, decorative text. +2. **~6 embedded-text-layer tokens on rerouted scan pages** (page folio `C-9`, + a few label tokens on FortWorth scans) — where OCR simultaneously *added* + hundreds of tokens per page that no text-layer extractor can see. + +No sentence, clause, paragraph, table cell, or heading of body text was lost +anywhere in the corpus. Additionally, two *garble* classes (overprint +char-doubling; letter-spacing/rotation char-explosion) keep the text in the +export but in a form that defeats search — they are fidelity, not loss, but are +the highest-value fixes. + +## Recommended follow-ups (not in scope of this eval) + +1. **Overprint dedupe** in the front-end (pdfplumber `dedupe_chars()` or + equivalent) — kills the doubled-char class at the root, incl. the S-1 + draft-cover watermark garble. +2. **Signature-block exemption** in the furniture stripper (never strip a line + containing `/s/` or `By:`-style execution fields) — restores signature text + to the json/html renders. +3. **Merge embedded text with OCR** on rerouted sparse pages (union rather than + replace), recovering folios like `C-9`. +4. Rotated-text capture is a pdfminer-stack limitation; revisit only if + sidebar/margin text ever matters to a consumer. + +## Reproducing + +```bash +uv run python scripts/text_faithfulness_eval.py --out eval_results.jsonl # full corpus, ~40 min on 4 cores +uv run python scripts/text_faithfulness_eval.py --out eval_results.jsonl --summarize +``` + +`tests/test_text_faithfulness.py` locks the guarantee in CI: PAWLS +effective recall vs the PDFium oracle ≥ 0.999 on the canonical fixture docs. diff --git a/docs/text_faithfulness_eval_results.jsonl b/docs/text_faithfulness_eval_results.jsonl new file mode 100644 index 0000000..fa67250 --- /dev/null +++ b/docs/text_faithfulness_eval_results.jsonl @@ -0,0 +1,179 @@ +{"doc": "tests/fixtures/s1/innio_s1__ck000210915020_101b8d.pdf", "pages": 394, "oracles": {"pdfium": {"pawls": {"n_oracle": 176130, "matched": 175917, "forgiven": 213, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998791, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 176130, "matched": 175432, "forgiven": 583, "missing_material": 114, "missing_trivial": 1, "recall_strict": 0.996037, "recall_effective": 0.999347, "missing_samples": [{"token": "december", "count": 12}, {"token": "accompanying", "count": 6}, {"token": "integral", "count": 6}, {"token": "statements.", "count": 6}, {"token": "financial", "count": 3}, {"token": "consolidated", "count": 3}, {"token": "*listed", "count": 2}, {"token": "31,", "count": 2}, {"token": "source:", "count": 2}, {"token": "f-54", "count": 2}, {"token": "f-55", "count": 2}, {"token": "1101", "count": 1}, {"token": "paul", "count": 1}, {"token": "ave.", "count": 1}, {"token": "waukesha,", "count": 1}, {"token": "53188", "count": 1}, {"token": "estimated", "count": 1}, {"token": "discounts", "count": 1}, {"token": "j.p.", "count": 1}, {"token": "morgan*", "count": 1}, {"token": "stanley*", "count": 1}, {"token": "barclays", "count": 1}, {"token": "citigroup", "count": 1}, {"token": "bnp", "count": 1}, {"token": "paribas", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 176129, "matched": 176107, "forgiven": 22, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999875, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 176129, "matched": 175625, "forgiven": 389, "missing_material": 114, "missing_trivial": 1, "recall_strict": 0.997138, "recall_effective": 0.999347, "missing_samples": [{"token": "december", "count": 12}, {"token": "accompanying", "count": 6}, {"token": "integral", "count": 6}, {"token": "statements.", "count": 6}, {"token": "financial", "count": 3}, {"token": "consolidated", "count": 3}, {"token": "*listed", "count": 2}, {"token": "31,", "count": 2}, {"token": "source:", "count": 2}, {"token": "f-54", "count": 2}, {"token": "f-55", "count": 2}, {"token": "1101", "count": 1}, {"token": "paul", "count": 1}, {"token": "ave.", "count": 1}, {"token": "waukesha,", "count": 1}, {"token": "53188", "count": 1}, {"token": "estimated", "count": 1}, {"token": "discounts", "count": 1}, {"token": "j.p.", "count": 1}, {"token": "morgan*", "count": 1}, {"token": "stanley*", "count": 1}, {"token": "barclays", "count": 1}, {"token": "citigroup", "count": 1}, {"token": "bnp", "count": 1}, {"token": "paribas", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/forbright_s1__forbrightsx1pu_4bba16.pdf", "pages": 441, "oracles": {"pdfium": {"pawls": {"n_oracle": 159715, "matched": 159533, "forgiven": 180, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.99886, "recall_effective": 0.999987, "flagged_pages": [{"page": 303, "n_oracle": 160, "matched": 158, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.9875, "recall_effective": 0.99375, "missing_samples": [{"token": "capitalretained", "count": 1}]}, {"page": 383, "n_oracle": 172, "matched": 170, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.988372, "recall_effective": 0.994186, "missing_samples": [{"token": "capitalretained", "count": 1}]}]}, "content": {"n_oracle": 159715, "matched": 158936, "forgiven": 601, "missing_material": 153, "missing_trivial": 25, "recall_strict": 0.995123, "recall_effective": 0.998886, "missing_samples": [{"token": "gross", "count": 12}, {"token": "unrealized", "count": 12}, {"token": "december", "count": 7}, {"token": "march", "count": 4}, {"token": "fair", "count": 3}, {"token": "thousands)", "count": 3}, {"token": "j.p.", "count": 2}, {"token": "morgan", "count": 2}, {"token": "barclays", "count": 2}, {"token": "santander", "count": 2}, {"token": "centerview", "count": 2}, {"token": "2025:", "count": 2}, {"token": "estimated", "count": 2}, {"token": "4/15/2021", "count": 2}, {"token": "4/15/2031", "count": 2}, {"token": "capitalretained", "count": 2}, {"token": "(level", "count": 2}, {"token": "f-66", "count": 2}, {"token": "edgar", "count": 1}, {"token": "lewandowski", "count": 1}, {"token": "zuckerman", "count": 1}, {"token": "simpson", "count": 1}, {"token": "thacher", "count": 1}, {"token": "bartlett", "count": 1}, {"token": "lexington", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 159694, "matched": 159689, "forgiven": 5, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999969, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 159694, "matched": 159092, "forgiven": 427, "missing_material": 152, "missing_trivial": 23, "recall_strict": 0.99623, "recall_effective": 0.998904, "missing_samples": [{"token": "gross", "count": 12}, {"token": "unrealized", "count": 12}, {"token": "december", "count": 7}, {"token": "march", "count": 4}, {"token": "fair", "count": 3}, {"token": "thousands)", "count": 3}, {"token": "j.p.", "count": 2}, {"token": "morgan", "count": 2}, {"token": "barclays", "count": 2}, {"token": "santander", "count": 2}, {"token": "centerview", "count": 2}, {"token": "2025:", "count": 2}, {"token": "estimated", "count": 2}, {"token": "4/15/2021", "count": 2}, {"token": "4/15/2031", "count": 2}, {"token": "(level", "count": 2}, {"token": "f-66", "count": 2}, {"token": "edgar", "count": 1}, {"token": "lewandowski", "count": 1}, {"token": "zuckerman", "count": 1}, {"token": "simpson", "count": 1}, {"token": "thacher", "count": 1}, {"token": "bartlett", "count": 1}, {"token": "lexington", "count": 1}, {"token": "until", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/xenergy_s1__drsa_1ae6c6.pdf", "pages": 338, "oracles": {"pdfium": {"pawls": {"n_oracle": 139903, "matched": 133554, "forgiven": 6349, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.954619, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 139903, "matched": 134169, "forgiven": 5662, "missing_material": 71, "missing_trivial": 1, "recall_strict": 0.959014, "recall_effective": 0.999485, "missing_samples": [{"token": "contents", "count": 37}, {"token": "buying", "count": 1}, {"token": "changed.", "count": 1}, {"token": "growth.", "count": 1}, {"token": "footprint.", "count": 1}, {"token": "119", "count": 1}, {"token": "efficiency.", "count": 1}, {"token": "136", "count": 1}, {"token": "143", "count": 1}, {"token": "145", "count": 1}, {"token": "sell.", "count": 1}, {"token": "mr.", "count": 1}, {"token": "176", "count": 1}, {"token": "182", "count": 1}, {"token": "186", "count": 1}, {"token": "189", "count": 1}, {"token": "194", "count": 1}, {"token": "f-10", "count": 1}, {"token": "f-15", "count": 1}, {"token": "f-16", "count": 1}, {"token": "f-21", "count": 1}, {"token": "f-32", "count": 1}, {"token": "f-36", "count": 1}, {"token": "ii-2", "count": 1}, {"token": "*1.1", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 155502, "matched": 155405, "forgiven": 88, "missing_material": 9, "missing_trivial": 0, "recall_strict": 0.999376, "recall_effective": 0.999942, "flagged_pages": [{"page": 1, "n_oracle": 770, "matched": 741, "forgiven": 22, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.962338, "recall_effective": 0.990909, "missing_samples": [{"token": "egnahcxe", "count": 1}, {"token": "delif", "count": 1}, {"token": "tnemetats", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": "litnu", "count": 1}, {"token": ".degnahc", "count": 1}, {"token": "etelpmoc", "count": 1}]}, {"page": 2, "n_oracle": 35, "matched": 0, "forgiven": 33, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.0, "recall_effective": 0.942857, "missing_samples": [{"token": "noissimmoc", "count": 1}, {"token": ".dettimrep", "count": 1}]}]}, "content": {"n_oracle": 155502, "matched": 155078, "forgiven": 322, "missing_material": 100, "missing_trivial": 2, "recall_strict": 0.997273, "recall_effective": 0.999344, "missing_samples": [{"token": "contents", "count": 37}, {"token": "mr.", "count": 3}, {"token": "seitiruces", "count": 2}, {"token": "f-10", "count": 2}, {"token": "before", "count": 1}, {"token": "buying", "count": 1}, {"token": "egnahcxe", "count": 1}, {"token": "delif", "count": 1}, {"token": "tnemetats", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": "litnu", "count": 1}, {"token": "eseht", "count": 1}, {"token": ".degnahc", "count": 1}, {"token": "etelpmoc", "count": 1}, {"token": "sutcepsorp", "count": 1}, {"token": "yranimilerp", "count": 1}, {"token": "siht", "count": 1}, {"token": "noitamrofni", "count": 1}, {"token": "noissimmoc", "count": 1}, {"token": ".dettimrep", "count": 1}, {"token": "nomura", "count": 1}, {"token": "dilution", "count": 1}, {"token": "f-5", "count": 1}, {"token": "risk.", "count": 1}, {"token": "assets,", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/fervo_s1__exhibit1016sx1_9df974.pdf", "pages": 295, "oracles": {"pdfium": {"pawls": {"n_oracle": 116708, "matched": 116680, "forgiven": 28, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99976, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 116708, "matched": 116338, "forgiven": 251, "missing_material": 119, "missing_trivial": 0, "recall_strict": 0.99683, "recall_effective": 0.99898, "missing_samples": [{"token": "granite", "count": 28}, {"token": "loans.", "count": 3}, {"token": "167", "count": 2}, {"token": "182", "count": 2}, {"token": "210", "count": 2}, {"token": "lenders.", "count": 2}, {"token": "collateral.", "count": 2}, {"token": "requirement.", "count": 2}, {"token": "105", "count": 1}, {"token": "111", "count": 1}, {"token": "137", "count": 1}, {"token": "139", "count": 1}, {"token": "142", "count": 1}, {"token": "146", "count": 1}, {"token": "149", "count": 1}, {"token": "156", "count": 1}, {"token": "159", "count": 1}, {"token": "162", "count": 1}, {"token": "170", "count": 1}, {"token": "173", "count": 1}, {"token": "174", "count": 1}, {"token": "181", "count": 1}, {"token": "10.4", "count": 1}, {"token": "188", "count": 1}, {"token": "191", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 116608, "matched": 116516, "forgiven": 92, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999211, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 116608, "matched": 116179, "forgiven": 310, "missing_material": 119, "missing_trivial": 0, "recall_strict": 0.996321, "recall_effective": 0.998979, "missing_samples": [{"token": "granite", "count": 28}, {"token": "loans.", "count": 3}, {"token": "167", "count": 2}, {"token": "182", "count": 2}, {"token": "210", "count": 2}, {"token": "lenders.", "count": 2}, {"token": "collateral.", "count": 2}, {"token": "requirement.", "count": 2}, {"token": "105", "count": 1}, {"token": "111", "count": 1}, {"token": "137", "count": 1}, {"token": "139", "count": 1}, {"token": "142", "count": 1}, {"token": "146", "count": 1}, {"token": "149", "count": 1}, {"token": "156", "count": 1}, {"token": "159", "count": 1}, {"token": "162", "count": 1}, {"token": "170", "count": 1}, {"token": "173", "count": 1}, {"token": "174", "count": 1}, {"token": "181", "count": 1}, {"token": "10.4", "count": 1}, {"token": "188", "count": 1}, {"token": "191", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/solv_energy_s1__s1_becbd8.pdf", "pages": 299, "oracles": {"pdfium": {"pawls": {"n_oracle": 143106, "matched": 138991, "forgiven": 4115, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.971245, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 143106, "matched": 138493, "forgiven": 4012, "missing_material": 601, "missing_trivial": 0, "recall_strict": 0.967765, "recall_effective": 0.9958, "missing_samples": [{"token": "contents", "count": 279}, {"token": "table", "count": 201}, {"token": "condensed", "count": 16}, {"token": "consolidated", "count": 10}, {"token": "notes", "count": 9}, {"token": "energy,", "count": 7}, {"token": "accumulated", "count": 2}, {"token": "accompanying", "count": 2}, {"token": "151", "count": 1}, {"token": "1,956,606", "count": 1}, {"token": "134", "count": 1}, {"token": "bonus(2)", "count": 1}, {"token": "options(3)", "count": 1}, {"token": "non-equity", "count": 1}, {"token": "(compensation(4)", "count": 1}, {"token": "compensation(5)", "count": 1}, {"token": "($)(1)", "count": 1}, {"token": "173", "count": 1}, {"token": "184", "count": 1}, {"token": "f-2", "count": 1}, {"token": "f-7", "count": 1}, {"token": "f-11", "count": 1}, {"token": "f-14", "count": 1}, {"token": "f-15", "count": 1}, {"token": "f-16", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 150334, "matched": 149440, "forgiven": 894, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.994053, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 150334, "matched": 147933, "forgiven": 1697, "missing_material": 704, "missing_trivial": 0, "recall_strict": 0.984029, "recall_effective": 0.995317, "missing_samples": [{"token": "contents", "count": 279}, {"token": "table", "count": 201}, {"token": "energy,", "count": 29}, {"token": "condensed", "count": 21}, {"token": "financial", "count": 18}, {"token": "consolidated", "count": 17}, {"token": "notes", "count": 12}, {"token": "remaining", "count": 4}, {"token": "assets,", "count": 4}, {"token": "information.", "count": 3}, {"token": "intangible", "count": 3}, {"token": "contractual", "count": 3}, {"token": "accompanying", "count": 3}, {"token": "mr.", "count": 3}, {"token": "total", "count": 2}, {"token": "february", "count": 2}, {"token": "2026,", "count": 1}, {"token": "inc.,", "count": 1}, {"token": "j.p.", "count": 1}, {"token": "155", "count": 1}, {"token": "174", "count": 1}, {"token": "agreement,", "count": 1}, {"token": "among", "count": 1}, {"token": "accumulated", "count": 1}, {"token": "151", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/neutron_lime_s1__exhibit45sx1_4208ac.pdf", "pages": 295, "oracles": {"pdfium": {"pawls": {"n_oracle": 63563, "matched": 63527, "forgiven": 36, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999434, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 63563, "matched": 62113, "forgiven": 544, "missing_material": 901, "missing_trivial": 5, "recall_strict": 0.977188, "recall_effective": 0.985746, "missing_samples": [{"token": "chris", "count": 71}, {"token": "maher", "count": 71}, {"token": "title:", "count": 62}, {"token": "by:", "count": 60}, {"token": "/s/", "count": 51}, {"token": "signatory", "count": 49}, {"token": "neutron", "count": 46}, {"token": "holdings,", "count": 46}, {"token": "authorized", "count": 45}, {"token": "inc.", "count": 34}, {"token": "name:", "count": 33}, {"token": "fidelity", "count": 18}, {"token": "address:", "count": 16}, {"token": "agreement.", "count": 10}, {"token": "counterparts.", "count": 10}, {"token": "canada", "count": 10}, {"token": "ulc", "count": 10}, {"token": "etc.", "count": 9}, {"token": "severability.", "count": 9}, {"token": "amendment.", "count": 9}, {"token": "third", "count": 8}, {"token": "fifth", "count": 8}, {"token": "omnibus", "count": 8}, {"token": "first", "count": 7}, {"token": "purchasers:", "count": 7}]}}, "plumber": {"pawls": {"n_oracle": 63541, "matched": 63529, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999811, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 63541, "matched": 62116, "forgiven": 519, "missing_material": 901, "missing_trivial": 5, "recall_strict": 0.977574, "recall_effective": 0.985741, "missing_samples": [{"token": "chris", "count": 71}, {"token": "maher", "count": 71}, {"token": "title:", "count": 62}, {"token": "by:", "count": 60}, {"token": "/s/", "count": 51}, {"token": "signatory", "count": 49}, {"token": "neutron", "count": 46}, {"token": "holdings,", "count": 46}, {"token": "authorized", "count": 45}, {"token": "inc.", "count": 34}, {"token": "name:", "count": 33}, {"token": "fidelity", "count": 18}, {"token": "address:", "count": 16}, {"token": "agreement.", "count": 10}, {"token": "counterparts.", "count": 10}, {"token": "canada", "count": 10}, {"token": "ulc", "count": 10}, {"token": "etc.", "count": 9}, {"token": "severability.", "count": 9}, {"token": "amendment.", "count": 9}, {"token": "third", "count": 8}, {"token": "fifth", "count": 8}, {"token": "omnibus", "count": 8}, {"token": "first", "count": 7}, {"token": "purchasers:", "count": 7}]}}}} +{"doc": "tests/fixtures/s1/yesway_s1__s1_558e28.pdf", "pages": 324, "oracles": {"pdfium": {"pawls": {"n_oracle": 135895, "matched": 135075, "forgiven": 820, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.993966, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 135895, "matched": 134774, "forgiven": 1021, "missing_material": 100, "missing_trivial": 0, "recall_strict": 0.991751, "recall_effective": 0.999264, "missing_samples": [{"token": "contents", "count": 28}, {"token": "accompanying", "count": 7}, {"token": "statements.", "count": 4}, {"token": "inc.,", "count": 3}, {"token": "esq.", "count": 2}, {"token": "(212)", "count": 2}, {"token": "notes", "count": 2}, {"token": "p.c.,", "count": 2}, {"token": "telephone:", "count": 1}, {"token": "avenue", "count": 1}, {"token": "york,", "count": 1}, {"token": "fax:", "count": 1}, {"token": "751-4864", "count": 1}, {"token": "michael", "count": 1}, {"token": "kim,", "count": 1}, {"token": "nallengara,", "count": 1}, {"token": "shearman", "count": 1}, {"token": "sterling", "count": 1}, {"token": "lexington", "count": 1}, {"token": "10022", "count": 1}, {"token": "848-4000", "count": 1}, {"token": "effective.", "count": 1}, {"token": "until", "count": 1}, {"token": "voting", "count": 1}, {"token": "goldman", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 135519, "matched": 135388, "forgiven": 126, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.999033, "recall_effective": 0.999963, "flagged_pages": [{"page": 1, "n_oracle": 708, "matched": 678, "forgiven": 25, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.957627, "recall_effective": 0.992938, "missing_samples": [{"token": "noissimmoc", "count": 1}, {"token": "htiw", "count": 1}, {"token": "tnemetats", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": ".degnahc", "count": 1}]}]}, "content": {"n_oracle": 135519, "matched": 135064, "forgiven": 341, "missing_material": 114, "missing_trivial": 0, "recall_strict": 0.996643, "recall_effective": 0.999159, "missing_samples": [{"token": "contents", "count": 28}, {"token": "accompanying", "count": 7}, {"token": "statements.", "count": 4}, {"token": "inc.,", "count": 3}, {"token": "esq.", "count": 2}, {"token": "(212)", "count": 2}, {"token": "notes", "count": 2}, {"token": "p.c.,", "count": 2}, {"token": "telephone:", "count": 1}, {"token": "michael", "count": 1}, {"token": "kim,", "count": 1}, {"token": "nallengara,", "count": 1}, {"token": "shearman", "count": 1}, {"token": "sterling", "count": 1}, {"token": "lexington", "count": 1}, {"token": "avenue", "count": 1}, {"token": "york,", "count": 1}, {"token": "10022", "count": 1}, {"token": "848-4000", "count": 1}, {"token": "fax:", "count": 1}, {"token": "751-4864", "count": 1}, {"token": "voting", "count": 1}, {"token": "noissimmoc", "count": 1}, {"token": "seitiruces", "count": 1}, {"token": "htiw", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/cerebras_s1__exhibit1018sx1_56fdc8.pdf", "pages": 276, "oracles": {"pdfium": {"pawls": {"n_oracle": 115673, "matched": 115646, "forgiven": 27, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999767, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 115673, "matched": 115526, "forgiven": 89, "missing_material": 58, "missing_trivial": 0, "recall_strict": 0.998729, "recall_effective": 0.999499, "missing_samples": [{"token": "credit.", "count": 3}, {"token": "agreement]", "count": 3}, {"token": "[cerebras", "count": 3}, {"token": "121", "count": 2}, {"token": "134", "count": 2}, {"token": "160", "count": 2}, {"token": "177", "count": 2}, {"token": "115", "count": 1}, {"token": "117", "count": 1}, {"token": "120", "count": 1}, {"token": "135", "count": 1}, {"token": "145", "count": 1}, {"token": "149", "count": 1}, {"token": "154", "count": 1}, {"token": "157", "count": 1}, {"token": "161", "count": 1}, {"token": "163", "count": 1}, {"token": "165", "count": 1}, {"token": "168", "count": 1}, {"token": "174", "count": 1}, {"token": "178", "count": 1}, {"token": "185", "count": 1}, {"token": "188", "count": 1}, {"token": "debt.", "count": 1}, {"token": "managing", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 115671, "matched": 115665, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999948, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 115671, "matched": 115546, "forgiven": 67, "missing_material": 58, "missing_trivial": 0, "recall_strict": 0.998919, "recall_effective": 0.999499, "missing_samples": [{"token": "credit.", "count": 3}, {"token": "agreement]", "count": 3}, {"token": "[cerebras", "count": 3}, {"token": "121", "count": 2}, {"token": "134", "count": 2}, {"token": "160", "count": 2}, {"token": "177", "count": 2}, {"token": "115", "count": 1}, {"token": "117", "count": 1}, {"token": "120", "count": 1}, {"token": "135", "count": 1}, {"token": "145", "count": 1}, {"token": "149", "count": 1}, {"token": "154", "count": 1}, {"token": "157", "count": 1}, {"token": "161", "count": 1}, {"token": "163", "count": 1}, {"token": "165", "count": 1}, {"token": "168", "count": 1}, {"token": "174", "count": 1}, {"token": "178", "count": 1}, {"token": "185", "count": 1}, {"token": "188", "count": 1}, {"token": "debt.", "count": 1}, {"token": "managing", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/liftoff_s1__iron20260113_231d60.pdf", "pages": 289, "oracles": {"pdfium": {"pawls": {"n_oracle": 146029, "matched": 142150, "forgiven": 3877, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.973437, "recall_effective": 0.999986, "flagged_pages": [{"page": 11, "n_oracle": 423, "matched": 414, "forgiven": 7, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.978723, "recall_effective": 0.995272, "missing_samples": [{"token": "transactionsforward-looking", "count": 1}, {"token": "holdersdividend", "count": 1}]}]}, "content": {"n_oracle": 146029, "matched": 142515, "forgiven": 3422, "missing_material": 91, "missing_trivial": 1, "recall_strict": 0.975936, "recall_effective": 0.99937, "missing_samples": [{"token": "december", "count": 7}, {"token": "remaining", "count": 4}, {"token": "useful", "count": 3}, {"token": "(650)", "count": 2}, {"token": "telephone:", "count": 2}, {"token": "effective.", "count": 2}, {"token": "permitted.", "count": 2}, {"token": "carrying", "count": 2}, {"token": "messrs.", "count": 2}, {"token": "2026.", "count": 1}, {"token": "mobile,", "count": 1}, {"token": "middlefield", "count": 1}, {"token": "redwood", "count": 1}, {"token": "319-7151", "count": 1}, {"token": "counsel", "count": 1}, {"token": "michael", "count": 1}, {"token": "kaplan", "count": 1}, {"token": "keshvargar", "count": 1}, {"token": "davis", "count": 1}, {"token": "polk", "count": 1}, {"token": "wardwell", "count": 1}, {"token": "lexington", "count": 1}, {"token": "york,", "count": 1}, {"token": "10017", "count": 1}, {"token": "(212)", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 147625, "matched": 147506, "forgiven": 70, "missing_material": 49, "missing_trivial": 0, "recall_strict": 0.999194, "recall_effective": 0.999668, "flagged_pages": [{"page": 2, "n_oracle": 38, "matched": 0, "forgiven": 19, "missing_material": 19, "missing_trivial": 0, "recall_strict": 0.0, "recall_effective": 0.5, "missing_samples": [{"token": "sseeiittiirruucceess", "count": 3}, {"token": "rreeffffoo", "count": 3}, {"token": "ddnnaa", "count": 2}, {"token": "eerreehhww", "count": 1}, {"token": "yyuubb", "count": 1}, {"token": "ggnniittiicciillooss", "count": 1}, {"token": "ttoonn", "count": 1}, {"token": "ssuuttcceeppssoorrpp", "count": 1}, {"token": "..eevviittcceeffffee", "count": 1}, {"token": "nnooiissssiimmmmoocc", "count": 1}, {"token": "eeggnnaahhccxxee", "count": 1}, {"token": "..ddeettttiimmrreepp", "count": 1}, {"token": "eellaass", "count": 1}, {"token": "rroo", "count": 1}]}, {"page": 3, "n_oracle": 512, "matched": 445, "forgiven": 37, "missing_material": 30, "missing_trivial": 0, "recall_strict": 0.869141, "recall_effective": 0.941406, "missing_samples": [{"token": "seitiruces", "count": 4}, {"token": "eseht", "count": 3}, {"token": "sutcepsorp", "count": 2}, {"token": "siht", "count": 2}, {"token": "eht", "count": 1}, {"token": "htiw", "count": 1}, {"token": "delif", "count": 1}, {"token": "tnemetats", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": "litnu", "count": 1}, {"token": "sredlohkcots", "count": 1}, {"token": "gnilles", "count": 1}, {"token": "rehtien", "count": 1}, {"token": ".degnahc", "count": 1}, {"token": "etelpmoc", "count": 1}, {"token": "noitamrofni", "count": 1}, {"token": "erehw", "count": 1}, {"token": "noitcidsiruj", "count": 1}, {"token": "gniticilos", "count": 1}, {"token": ".evitceffe", "count": 1}, {"token": "noissimmoc", "count": 1}, {"token": "egnahcxe", "count": 1}, {"token": ".dettimrep", "count": 1}]}]}, "content": {"n_oracle": 147625, "matched": 147167, "forgiven": 299, "missing_material": 158, "missing_trivial": 1, "recall_strict": 0.996898, "recall_effective": 0.998923, "missing_samples": [{"token": "december", "count": 13}, {"token": "seitiruces", "count": 4}, {"token": "carrying", "count": 4}, {"token": "remaining", "count": 4}, {"token": "sseeiittiirruucceess", "count": 3}, {"token": "rreeffffoo", "count": 3}, {"token": "eseht", "count": 3}, {"token": "useful", "count": 3}, {"token": "(650)", "count": 2}, {"token": "telephone:", "count": 2}, {"token": "ddnnaa", "count": 2}, {"token": "eht", "count": 2}, {"token": "sutcepsorp", "count": 2}, {"token": "siht", "count": 2}, {"token": "september", "count": 2}, {"token": "messrs.", "count": 2}, {"token": "2026.", "count": 1}, {"token": "mobile,", "count": 1}, {"token": "middlefield", "count": 1}, {"token": "redwood", "count": 1}, {"token": "319-7151", "count": 1}, {"token": "rickard", "count": 1}, {"token": "michael", "count": 1}, {"token": "kaplan", "count": 1}, {"token": "counsel", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/pershing_square_s1__s1_f498b3.pdf", "pages": 280, "oracles": {"pdfium": {"pawls": {"n_oracle": 170152, "matched": 170063, "forgiven": 89, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999477, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 170152, "matched": 169674, "forgiven": 390, "missing_material": 84, "missing_trivial": 4, "recall_strict": 0.997191, "recall_effective": 0.999483, "missing_samples": [{"token": "contents", "count": 18}, {"token": "december", "count": 4}, {"token": "january", "count": 2}, {"token": "s&p", "count": 2}, {"token": "vs.", "count": 2}, {"token": "1,165,766,679", "count": 2}, {"token": "1,019", "count": 2}, {"token": "bofa", "count": 1}, {"token": "jefferies", "count": 1}, {"token": "wells", "count": 1}, {"token": "fargo", "count": 1}, {"token": "international,", "count": 1}, {"token": "ltd.,", "count": 1}, {"token": "existing", "count": 1}, {"token": "funds,", "count": 1}, {"token": "2025.", "count": 1}, {"token": "mr.", "count": 1}, {"token": "overview", "count": 1}, {"token": "(159,064)", "count": 1}, {"token": "(151,840)", "count": 1}, {"token": "underwriters'", "count": 1}, {"token": "(1,425)", "count": 1}, {"token": "159,821", "count": 1}, {"token": "(318,885)", "count": 1}, {"token": "7,224", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 170412, "matched": 170399, "forgiven": 13, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999924, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 170412, "matched": 169989, "forgiven": 334, "missing_material": 85, "missing_trivial": 4, "recall_strict": 0.997518, "recall_effective": 0.999478, "missing_samples": [{"token": "contents", "count": 18}, {"token": "december", "count": 4}, {"token": "january", "count": 2}, {"token": "s&p", "count": 2}, {"token": "vs.", "count": 2}, {"token": "1,165,766,679", "count": 2}, {"token": "1,019", "count": 2}, {"token": "bofa", "count": 1}, {"token": "jefferies", "count": 1}, {"token": "wells", "count": 1}, {"token": "fargo", "count": 1}, {"token": "international,", "count": 1}, {"token": "ltd.,", "count": 1}, {"token": "existing", "count": 1}, {"token": "funds,", "count": 1}, {"token": "2025.", "count": 1}, {"token": "mr.", "count": 1}, {"token": "overview", "count": 1}, {"token": "(159,064)", "count": 1}, {"token": "(151,840)", "count": 1}, {"token": "underwriters'", "count": 1}, {"token": "(1,425)", "count": 1}, {"token": "159,821", "count": 1}, {"token": "(318,885)", "count": 1}, {"token": "7,224", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/infleqtion_s1__s1_60ea51.pdf", "pages": 252, "oracles": {"pdfium": {"pawls": {"n_oracle": 126502, "matched": 122797, "forgiven": 3704, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.970712, "recall_effective": 0.999992, "flagged_pages": [{"page": 2, "n_oracle": 804, "matched": 731, "forgiven": 72, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.909204, "recall_effective": 0.998756, "missing_samples": [{"token": "changed.", "count": 1}]}]}, "content": {"n_oracle": 126502, "matched": 123049, "forgiven": 3297, "missing_material": 156, "missing_trivial": 0, "recall_strict": 0.972704, "recall_effective": 0.998767, "missing_samples": [{"token": "contents", "count": 66}, {"token": "(f/k/a", "count": 5}, {"token": "integral", "count": 5}, {"token": "statements.", "count": 2}, {"token": "accompanying", "count": 2}, {"token": "mr.", "count": 2}, {"token": "preliminary", "count": 1}, {"token": "changed.", "count": 1}, {"token": "coldquanta,", "count": 1}, {"token": "alumni", "count": 1}, {"token": "pranav", "count": 1}, {"token": "gokhale,", "count": 1}, {"token": "messrs.", "count": 1}, {"token": "l.p.,", "count": 1}, {"token": "129", "count": 1}, {"token": "140", "count": 1}, {"token": "152", "count": 1}, {"token": "f-5", "count": 1}, {"token": "f-10", "count": 1}, {"token": "f-11", "count": 1}, {"token": "f-12", "count": 1}, {"token": "inc..", "count": 1}, {"token": "f-13", "count": 1}, {"token": "f-14", "count": 1}, {"token": "f-15", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 133282, "matched": 132816, "forgiven": 429, "missing_material": 37, "missing_trivial": 0, "recall_strict": 0.996504, "recall_effective": 0.999722, "flagged_pages": [{"page": 2, "n_oracle": 807, "matched": 739, "forgiven": 31, "missing_material": 37, "missing_trivial": 0, "recall_strict": 0.915737, "recall_effective": 0.954151, "missing_samples": [{"token": "eht", "count": 5}, {"token": "seitiruces", "count": 4}, {"token": "eseht", "count": 3}, {"token": "sutcepsorp", "count": 2}, {"token": "yranimilerp", "count": 2}, {"token": "sredloh", "count": 1}, {"token": "ytiruces", "count": 1}, {"token": "gnilles", "count": 1}, {"token": "rehtien", "count": 1}, {"token": ".degnahc", "count": 1}, {"token": "etelpmoc", "count": 1}, {"token": "siht", "count": 1}, {"token": "noitamrofni", "count": 1}, {"token": ".evitceffe", "count": 1}, {"token": "noissimmoc", "count": 1}, {"token": "egnahcxe", "count": 1}, {"token": "htiw", "count": 1}, {"token": "delif", "count": 1}, {"token": "tnemetats", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": "litnu", "count": 1}, {"token": "erehw", "count": 1}, {"token": "noitcidsiruj", "count": 1}, {"token": "yub", "count": 1}, {"token": "seod", "count": 1}]}]}, "content": {"n_oracle": 133282, "matched": 132201, "forgiven": 849, "missing_material": 232, "missing_trivial": 0, "recall_strict": 0.991889, "recall_effective": 0.998259, "missing_samples": [{"token": "contents", "count": 66}, {"token": "eht", "count": 5}, {"token": "(f/k/a", "count": 5}, {"token": "integral", "count": 5}, {"token": "seitiruces", "count": 4}, {"token": "combined", "count": 4}, {"token": "accompanying", "count": 4}, {"token": "statements.", "count": 4}, {"token": "mr.", "count": 4}, {"token": "eseht", "count": 3}, {"token": "sutcepsorp", "count": 2}, {"token": "yranimilerp", "count": 2}, {"token": "churchill", "count": 2}, {"token": "february", "count": 2}, {"token": "consolidated", "count": 2}, {"token": "company's", "count": 2}, {"token": "dr.", "count": 2}, {"token": "(incorporated", "count": 2}, {"token": "8-k,", "count": 2}, {"token": "2026).", "count": 2}, {"token": "filed", "count": 1}, {"token": "sredloh", "count": 1}, {"token": "ytiruces", "count": 1}, {"token": "gnilles", "count": 1}, {"token": "rehtien", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/yesway_s1__ex105b_379ea6.pdf", "pages": 232, "oracles": {"pdfium": {"pawls": {"n_oracle": 114356, "matched": 114321, "forgiven": 34, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.999694, "recall_effective": 0.999991, "flagged_pages": [{"page": 19, "n_oracle": 765, "matched": 764, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.998693, "recall_effective": 0.998693, "missing_samples": [{"token": "1/2", "count": 1}]}]}, "content": {"n_oracle": 114356, "matched": 113929, "forgiven": 243, "missing_material": 142, "missing_trivial": 42, "recall_strict": 0.996266, "recall_effective": 0.998391, "missing_samples": [{"token": "(attached", "count": 5}, {"token": "lenders.", "count": 5}, {"token": "fees.", "count": 5}, {"token": "title:", "count": 4}, {"token": "loans.", "count": 4}, {"token": "payments.", "count": 4}, {"token": "documents.", "count": 3}, {"token": "generally.", "count": 3}, {"token": "severability.", "count": 2}, {"token": "agent.", "count": 2}, {"token": "hereto)", "count": 2}, {"token": "law.", "count": 2}, {"token": "optional.", "count": 2}, {"token": "fee.", "count": 2}, {"token": "survival.", "count": 2}, {"token": "covenants.", "count": 2}, {"token": "persons.", "count": 2}, {"token": "terms.", "count": 1}, {"token": "agreement.", "count": 1}, {"token": "date.", "count": 1}, {"token": "document.", "count": 1}, {"token": "control.", "count": 1}, {"token": "etc.", "count": 1}, {"token": "jurisdiction.", "count": 1}, {"token": "signatures.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 114361, "matched": 114279, "forgiven": 82, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999283, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 114361, "matched": 113890, "forgiven": 288, "missing_material": 140, "missing_trivial": 43, "recall_strict": 0.995881, "recall_effective": 0.9984, "missing_samples": [{"token": "(attached", "count": 5}, {"token": "lenders.", "count": 5}, {"token": "fees.", "count": 5}, {"token": "title:", "count": 4}, {"token": "loans.", "count": 4}, {"token": "payments.", "count": 4}, {"token": "documents.", "count": 3}, {"token": "generally.", "count": 3}, {"token": "severability.", "count": 2}, {"token": "agent.", "count": 2}, {"token": "hereto)", "count": 2}, {"token": "law.", "count": 2}, {"token": "optional.", "count": 2}, {"token": "fee.", "count": 2}, {"token": "survival.", "count": 2}, {"token": "covenants.", "count": 2}, {"token": "persons.", "count": 2}, {"token": "terms.", "count": 1}, {"token": "agreement.", "count": 1}, {"token": "date.", "count": 1}, {"token": "document.", "count": 1}, {"token": "control.", "count": 1}, {"token": "etc.", "count": 1}, {"token": "jurisdiction.", "count": 1}, {"token": "signatures.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/exyn_s1__s1_b6bd4c.pdf", "pages": 234, "oracles": {"pdfium": {"pawls": {"n_oracle": 104015, "matched": 103384, "forgiven": 630, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.993934, "recall_effective": 0.99999, "flagged_pages": [{"page": 1, "n_oracle": 550, "matched": 522, "forgiven": 27, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.949091, "recall_effective": 0.998182, "missing_samples": [{"token": "changed.", "count": 1}]}]}, "content": {"n_oracle": 104015, "matched": 103048, "forgiven": 892, "missing_material": 74, "missing_trivial": 1, "recall_strict": 0.990703, "recall_effective": 0.999279, "missing_samples": [{"token": "contents", "count": 27}, {"token": "december", "count": 3}, {"token": "prospectus);", "count": 2}, {"token": "effect.", "count": 2}, {"token": "mr.", "count": 2}, {"token": "filed", "count": 1}, {"token": "registration", "count": 1}, {"token": "until", "count": 1}, {"token": "preliminary", "count": 1}, {"token": "offering,", "count": 1}, {"token": "between", "count": 1}, {"token": "changed.", "count": 1}, {"token": "148", "count": 1}, {"token": "full),", "count": 1}, {"token": "assumed", "count": 1}, {"token": "western", "count": 1}, {"token": "september", "count": 1}, {"token": "2023,", "count": 1}, {"token": "2025,", "count": 1}, {"token": "bank.", "count": 1}, {"token": "incorporation,", "count": 1}, {"token": "thousands,", "count": 1}, {"token": "data)", "count": 1}, {"token": "bylaws,", "count": 1}, {"token": "l.p.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 103753, "matched": 103668, "forgiven": 77, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.999181, "recall_effective": 0.999923, "flagged_pages": [{"page": 1, "n_oracle": 553, "matched": 523, "forgiven": 25, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.94575, "recall_effective": 0.990958, "missing_samples": [{"token": "htiw", "count": 1}, {"token": "delif", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": ".degnahc", "count": 1}, {"token": "etelpmoc", "count": 1}]}, {"page": 2, "n_oracle": 36, "matched": 0, "forgiven": 33, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.0, "recall_effective": 0.916667, "missing_samples": [{"token": "noissimmoc", "count": 1}, {"token": "egnahcxe", "count": 1}, {"token": ".dettimrep", "count": 1}]}]}, "content": {"n_oracle": 103753, "matched": 103342, "forgiven": 324, "missing_material": 86, "missing_trivial": 1, "recall_strict": 0.996039, "recall_effective": 0.999161, "missing_samples": [{"token": "contents", "count": 27}, {"token": "december", "count": 3}, {"token": "seitiruces", "count": 2}, {"token": "prospectus);", "count": 2}, {"token": "effect.", "count": 2}, {"token": "mr.", "count": 2}, {"token": "offering,", "count": 1}, {"token": "between", "count": 1}, {"token": "htiw", "count": 1}, {"token": "delif", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": "litnu", "count": 1}, {"token": "eseht", "count": 1}, {"token": ".degnahc", "count": 1}, {"token": "etelpmoc", "count": 1}, {"token": "sutcepsorp", "count": 1}, {"token": "yranimilerp", "count": 1}, {"token": "siht", "count": 1}, {"token": "deniatnoc", "count": 1}, {"token": "noitamrofni", "count": 1}, {"token": "noissimmoc", "count": 1}, {"token": "egnahcxe", "count": 1}, {"token": ".dettimrep", "count": 1}, {"token": "148", "count": 1}, {"token": "full),", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/generate_bio_s1__ex1015_68a8fd.pdf", "pages": 195, "oracles": {"pdfium": {"pawls": {"n_oracle": 63740, "matched": 63543, "forgiven": 196, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.996909, "recall_effective": 0.999984, "flagged_pages": [{"page": 111, "n_oracle": 1, "matched": 0, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.0, "recall_effective": 0.0, "missing_samples": [{"token": "c-9", "count": 1}]}]}, "content": {"n_oracle": 63740, "matched": 63264, "forgiven": 266, "missing_material": 141, "missing_trivial": 69, "recall_strict": 0.992532, "recall_effective": 0.996705, "missing_samples": [{"token": "4831-9849-5468,", "count": 6}, {"token": "v.", "count": 6}, {"token": "access.", "count": 4}, {"token": "default.", "count": 3}, {"token": "improvements.", "count": 3}, {"token": "insurance.", "count": 3}, {"token": "premises.", "count": 2}, {"token": "63", "count": 2}, {"token": "a-1", "count": 2}, {"token": "definitions.", "count": 2}, {"token": "notice.", "count": 2}, {"token": "removal.", "count": 2}, {"token": "work.", "count": 2}, {"token": "required.", "count": 2}, {"token": "enjoyment.", "count": 2}, {"token": "agreement.", "count": 2}, {"token": "essence.", "count": 2}, {"token": "20__", "count": 2}, {"token": "services.", "count": 1}, {"token": "liability.", "count": 1}, {"token": "b-1", "count": 1}, {"token": "e-1", "count": 1}, {"token": "data.", "count": 1}, {"token": "date:", "count": 1}, {"token": "55", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 63766, "matched": 63722, "forgiven": 43, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.99931, "recall_effective": 0.999984, "flagged_pages": [{"page": 111, "n_oracle": 1, "matched": 0, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.0, "recall_effective": 0.0, "missing_samples": [{"token": "c-9", "count": 1}]}]}, "content": {"n_oracle": 63766, "matched": 63423, "forgiven": 132, "missing_material": 143, "missing_trivial": 68, "recall_strict": 0.994621, "recall_effective": 0.996691, "missing_samples": [{"token": "4831-9849-5468,", "count": 6}, {"token": "v.", "count": 6}, {"token": "access.", "count": 4}, {"token": "default.", "count": 3}, {"token": "improvements.", "count": 3}, {"token": "insurance.", "count": 3}, {"token": "premises.", "count": 2}, {"token": "63", "count": 2}, {"token": "a-1", "count": 2}, {"token": "definitions.", "count": 2}, {"token": "notice.", "count": 2}, {"token": "removal.", "count": 2}, {"token": "work.", "count": 2}, {"token": "required.", "count": 2}, {"token": "enjoyment.", "count": 2}, {"token": "agreement.", "count": 2}, {"token": "essence.", "count": 2}, {"token": "20__", "count": 2}, {"token": "services.", "count": 1}, {"token": "liability.", "count": 1}, {"token": "b-1", "count": 1}, {"token": "e-1", "count": 1}, {"token": "data.", "count": 1}, {"token": "date:", "count": 1}, {"token": "55", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/pershing_square_s1__ex109_9133b3.pdf", "pages": 167, "oracles": {"pdfium": {"pawls": {"n_oracle": 113452, "matched": 113430, "forgiven": 22, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999806, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 113452, "matched": 113067, "forgiven": 127, "missing_material": 168, "missing_trivial": 90, "recall_strict": 0.996606, "recall_effective": 0.997726, "missing_samples": [{"token": "insurance.", "count": 7}, {"token": "rights.", "count": 6}, {"token": "date.", "count": 4}, {"token": "definitions.", "count": 4}, {"token": "services.", "count": 3}, {"token": "agreement.", "count": 3}, {"token": "intentionally", "count": 3}, {"token": "permits.", "count": 3}, {"token": "taking.", "count": 3}, {"token": "signature_____________________________________", "count": 3}, {"token": "rent.", "count": 2}, {"token": "omitted.", "count": 2}, {"token": "alterations.", "count": 2}, {"token": "restrictions.", "count": 2}, {"token": "area.", "count": 2}, {"token": "work.", "count": 2}, {"token": "contribution.", "count": 2}, {"token": "property.", "count": 2}, {"token": "repair.", "count": 2}, {"token": "survival.", "count": 2}, {"token": "uses.", "count": 2}, {"token": "indemnity.", "count": 2}, {"token": "damages.", "count": 2}, {"token": "default.", "count": 2}, {"token": "offer.", "count": 2}]}}, "plumber": {"pawls": {"n_oracle": 113412, "matched": 113386, "forgiven": 26, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999771, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 113412, "matched": 113027, "forgiven": 127, "missing_material": 170, "missing_trivial": 88, "recall_strict": 0.996605, "recall_effective": 0.997725, "missing_samples": [{"token": "insurance.", "count": 7}, {"token": "rights.", "count": 6}, {"token": "date.", "count": 4}, {"token": "definitions.", "count": 4}, {"token": "services.", "count": 3}, {"token": "agreement.", "count": 3}, {"token": "intentionally", "count": 3}, {"token": "permits.", "count": 3}, {"token": "taking.", "count": 3}, {"token": "signature_____________________________________", "count": 3}, {"token": "rent.", "count": 2}, {"token": "omitted.", "count": 2}, {"token": "alterations.", "count": 2}, {"token": "restrictions.", "count": 2}, {"token": "area.", "count": 2}, {"token": "work.", "count": 2}, {"token": "contribution.", "count": 2}, {"token": "property.", "count": 2}, {"token": "repair.", "count": 2}, {"token": "survival.", "count": 2}, {"token": "uses.", "count": 2}, {"token": "indemnity.", "count": 2}, {"token": "damages.", "count": 2}, {"token": "default.", "count": 2}, {"token": "offer.", "count": 2}]}}}} +{"doc": "tests/fixtures/s1/swarmer_s1__s1_fe1f49.pdf", "pages": 184, "oracles": {"pdfium": {"pawls": {"n_oracle": 84066, "matched": 83559, "forgiven": 506, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.993969, "recall_effective": 0.999988, "flagged_pages": [{"page": 1, "n_oracle": 562, "matched": 538, "forgiven": 23, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.957295, "recall_effective": 0.998221, "missing_samples": [{"token": "changed.", "count": 1}]}]}, "content": {"n_oracle": 84066, "matched": 83396, "forgiven": 632, "missing_material": 38, "missing_trivial": 0, "recall_strict": 0.99203, "recall_effective": 0.999548, "missing_samples": [{"token": "contents", "count": 15}, {"token": "mr.", "count": 6}, {"token": "f-7", "count": 2}, {"token": "swarmer,", "count": 1}, {"token": "changed.", "count": 1}, {"token": "december", "count": 1}, {"token": "(3)(4)", "count": 1}, {"token": "2024:", "count": 1}, {"token": "f-4", "count": 1}, {"token": "f-5", "count": 1}, {"token": "f-12", "count": 1}, {"token": "f-13", "count": 1}, {"token": "f-20", "count": 1}, {"token": "f-23", "count": 1}, {"token": "f-24", "count": 1}, {"token": "f-25", "count": 1}, {"token": "ii-5", "count": 1}, {"token": "ii-6", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 83864, "matched": 83788, "forgiven": 68, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.999094, "recall_effective": 0.999905, "flagged_pages": [{"page": 1, "n_oracle": 567, "matched": 539, "forgiven": 20, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.950617, "recall_effective": 0.985891, "missing_samples": [{"token": "egnahcxe", "count": 1}, {"token": "htiw", "count": 1}, {"token": "delif", "count": 1}, {"token": "tnemetats", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": "litnu", "count": 1}, {"token": ".degnahc", "count": 1}, {"token": "etelpmoc", "count": 1}]}]}, "content": {"n_oracle": 83864, "matched": 83625, "forgiven": 186, "missing_material": 53, "missing_trivial": 0, "recall_strict": 0.99715, "recall_effective": 0.999368, "missing_samples": [{"token": "contents", "count": 15}, {"token": "mr.", "count": 3}, {"token": "seitiruces", "count": 2}, {"token": "f-7", "count": 2}, {"token": "swarmer,", "count": 1}, {"token": "eht", "count": 1}, {"token": "htiw", "count": 1}, {"token": "delif", "count": 1}, {"token": "tnemetats", "count": 1}, {"token": "noitartsiger", "count": 1}, {"token": "litnu", "count": 1}, {"token": "eseht", "count": 1}, {"token": ".degnahc", "count": 1}, {"token": "etelpmoc", "count": 1}, {"token": "sutcepsorp", "count": 1}, {"token": "yranimilerp", "count": 1}, {"token": "siht", "count": 1}, {"token": "noitamrofni", "count": 1}, {"token": "december", "count": 1}, {"token": "forma(2)", "count": 1}, {"token": "adjusted(3)(4)", "count": 1}, {"token": "2024:", "count": 1}, {"token": "(12)mr.", "count": 1}, {"token": "(13)mr.", "count": 1}, {"token": "(14)mr.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/kardigan_s1__ex1017_6628ff.pdf", "pages": 126, "oracles": {"pdfium": {"pawls": {"n_oracle": 54991, "matched": 53374, "forgiven": 1616, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.970595, "recall_effective": 0.999982, "flagged_pages": [{"page": 107, "n_oracle": 341, "matched": 327, "forgiven": 13, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.958944, "recall_effective": 0.997067, "missing_samples": [{"token": "yearinstalled", "count": 1}]}]}, "content": {"n_oracle": 54991, "matched": 53170, "forgiven": 1599, "missing_material": 170, "missing_trivial": 52, "recall_strict": 0.966885, "recall_effective": 0.995963, "missing_samples": [{"token": "general.", "count": 6}, {"token": "default.", "count": 5}, {"token": "remedy.", "count": 4}, {"token": "rent.", "count": 4}, {"token": "offer.", "count": 3}, {"token": "obligations.", "count": 3}, {"token": "insurance.", "count": 3}, {"token": "signage.", "count": 3}, {"token": "warranty.", "count": 2}, {"token": "improvements.", "count": 2}, {"token": "space.", "count": 2}, {"token": "abatement.", "count": 2}, {"token": "use.", "count": 2}, {"token": "confidentiality.", "count": 2}, {"token": "transfers.", "count": 2}, {"token": "rights.", "count": 2}, {"token": "law.", "count": 2}, {"token": "management.", "count": 2}, {"token": "costs.", "count": 2}, {"token": "proceeds.", "count": 2}, {"token": "areas.", "count": 1}, {"token": "possession.", "count": 1}, {"token": "effect.", "count": 1}, {"token": "compliance.", "count": 1}, {"token": "letter.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 58893, "matched": 58845, "forgiven": 48, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999185, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 58893, "matched": 58449, "forgiven": 197, "missing_material": 201, "missing_trivial": 46, "recall_strict": 0.992461, "recall_effective": 0.995806, "missing_samples": [{"token": "general.", "count": 6}, {"token": "rent.", "count": 5}, {"token": "default.", "count": 5}, {"token": "lease.", "count": 4}, {"token": "remedy.", "count": 4}, {"token": "improvements.", "count": 4}, {"token": "tenant.", "count": 4}, {"token": "premises.", "count": 3}, {"token": "offer.", "count": 3}, {"token": "obligations.", "count": 3}, {"token": "insurance.", "count": 3}, {"token": "signage.", "count": 3}, {"token": "landlord.", "count": 2}, {"token": "(i.e.,", "count": 2}, {"token": "warranty.", "count": 2}, {"token": "term.", "count": 2}, {"token": "space.", "count": 2}, {"token": "abatement.", "count": 2}, {"token": "use.", "count": 2}, {"token": "clean-up.", "count": 2}, {"token": "confidentiality.", "count": 2}, {"token": "transfers.", "count": 2}, {"token": "rights.", "count": 2}, {"token": "law.", "count": 2}, {"token": "management.", "count": 2}]}}}} +{"doc": "tests/fixtures/s1/hawkeye360_s1__exhibit21sx1_fa98cc.pdf", "pages": 112, "oracles": {"pdfium": {"pawls": {"n_oracle": 45091, "matched": 45078, "forgiven": 13, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999712, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 45091, "matched": 44952, "forgiven": 49, "missing_material": 90, "missing_trivial": 0, "recall_strict": 0.996917, "recall_effective": 0.998004, "missing_samples": [{"token": "agreement.", "count": 5}, {"token": "definitions", "count": 4}, {"token": "agreements.", "count": 4}, {"token": "merger.", "count": 3}, {"token": "annex", "count": 2}, {"token": "closing.", "count": 2}, {"token": "officers.", "count": 2}, {"token": "spreadsheet.", "count": 2}, {"token": "shares.", "count": 2}, {"token": "conflicts.", "count": 2}, {"token": "confidentiality.", "count": 2}, {"token": "certificate.", "count": 2}, {"token": "termination.", "count": 2}, {"token": "plans.", "count": 1}, {"token": "laws.", "count": 1}, {"token": "taxes.", "count": 1}, {"token": "insurance.", "count": 1}, {"token": "policy.", "count": 1}, {"token": "time.", "count": 1}, {"token": "stock.", "count": 1}, {"token": "amount.", "count": 1}, {"token": "payments.", "count": 1}, {"token": "transfers.", "count": 1}, {"token": "withholding.", "count": 1}, {"token": "authority.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 45080, "matched": 45067, "forgiven": 13, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999712, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 45080, "matched": 44941, "forgiven": 49, "missing_material": 90, "missing_trivial": 0, "recall_strict": 0.996917, "recall_effective": 0.998004, "missing_samples": [{"token": "agreement.", "count": 5}, {"token": "definitions", "count": 4}, {"token": "agreements.", "count": 4}, {"token": "merger.", "count": 3}, {"token": "annex", "count": 2}, {"token": "closing.", "count": 2}, {"token": "officers.", "count": 2}, {"token": "spreadsheet.", "count": 2}, {"token": "shares.", "count": 2}, {"token": "conflicts.", "count": 2}, {"token": "confidentiality.", "count": 2}, {"token": "certificate.", "count": 2}, {"token": "termination.", "count": 2}, {"token": "plans.", "count": 1}, {"token": "laws.", "count": 1}, {"token": "taxes.", "count": 1}, {"token": "insurance.", "count": 1}, {"token": "policy.", "count": 1}, {"token": "time.", "count": 1}, {"token": "stock.", "count": 1}, {"token": "amount.", "count": 1}, {"token": "payments.", "count": 1}, {"token": "transfers.", "count": 1}, {"token": "withholding.", "count": 1}, {"token": "authority.", "count": 1}]}}}} +{"doc": "files/pdf/credit_aon.pdf", "pages": 118, "oracles": {"pdfium": {"pawls": {"n_oracle": 49909, "matched": 49898, "forgiven": 11, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99978, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 49909, "matched": 49801, "forgiven": 80, "missing_material": 26, "missing_trivial": 2, "recall_strict": 0.997836, "recall_effective": 0.999439, "missing_samples": [{"token": "title:", "count": 18}, {"token": "authorized", "count": 3}, {"token": "managing", "count": 3}, {"token": "signatory", "count": 2}]}}, "plumber": {"pawls": {"n_oracle": 49895, "matched": 49889, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99988, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 49895, "matched": 49790, "forgiven": 77, "missing_material": 26, "missing_trivial": 2, "recall_strict": 0.997896, "recall_effective": 0.999439, "missing_samples": [{"token": "title:", "count": 18}, {"token": "authorized", "count": 3}, {"token": "managing", "count": 3}, {"token": "signatory", "count": 2}]}}}} +{"doc": "files/pdf/primary-document.pdf", "pages": 121, "oracles": {"pdfium": {"pawls": {"n_oracle": 68816, "matched": 68706, "forgiven": 83, "missing_material": 27, "missing_trivial": 0, "recall_strict": 0.998402, "recall_effective": 0.999608, "flagged_pages": [{"page": 36, "n_oracle": 450, "matched": 449, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.997778, "recall_effective": 0.997778, "missing_samples": [{"token": "reductionconsolidation", "count": 1}]}, {"page": 72, "n_oracle": 344, "matched": 340, "forgiven": 2, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.988372, "recall_effective": 0.994186, "missing_samples": [{"token": "pricesnumber", "count": 1}, {"token": "valuenumber", "count": 1}]}, {"page": 80, "n_oracle": 442, "matched": 441, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.997738, "recall_effective": 0.997738, "missing_samples": [{"token": "marketsdiagnostics", "count": 1}]}, {"page": 81, "n_oracle": 458, "matched": 456, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.995633, "recall_effective": 0.997817, "missing_samples": [{"token": "amortizationnet", "count": 1}]}, {"page": 97, "n_oracle": 496, "matched": 493, "forgiven": 2, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.993952, "recall_effective": 0.997984, "missing_samples": [{"token": "obligationbenefit", "count": 1}]}, {"page": 99, "n_oracle": 333, "matched": 332, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.996997, "recall_effective": 0.996997, "missing_samples": [{"token": "reductionconsolidation", "count": 1}]}, {"page": 105, "n_oracle": 541, "matched": 539, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.996303, "recall_effective": 0.998152, "missing_samples": [{"token": "componentsamounts", "count": 1}]}, {"page": 114, "n_oracle": 414, "matched": 405, "forgiven": 0, "missing_material": 9, "missing_trivial": 0, "recall_strict": 0.978261, "recall_effective": 0.978261, "missing_samples": [{"token": "dateexhibit", "count": 1}, {"token": "numberfiled", "count": 1}, {"token": "2001.8-k", "count": 1}, {"token": "20268-k", "count": 1}, {"token": "association8-k", "count": 1}, {"token": "20298-k", "count": 1}, {"token": "20308-k", "count": 1}, {"token": "2031.8-k", "count": 1}, {"token": "2006).*10-k", "count": 1}]}, {"page": 115, "n_oracle": 433, "matched": 430, "forgiven": 0, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.993072, "recall_effective": 0.993072, "missing_samples": [{"token": "dateexhibit", "count": 1}, {"token": "numberfiled", "count": 1}, {"token": "2010).*10-k", "count": 1}]}, {"page": 116, "n_oracle": 412, "matched": 409, "forgiven": 0, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.992718, "recall_effective": 0.992718, "missing_samples": [{"token": "dateexhibit", "count": 1}, {"token": "numberfiled", "count": 1}, {"token": "company*10-q", "count": 1}]}, {"page": 117, "n_oracle": 319, "matched": 315, "forgiven": 0, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.987461, "recall_effective": 0.987461, "missing_samples": [{"token": "dateexhibit", "count": 1}, {"token": "numberfiled", "count": 1}, {"token": "company*10-k", "count": 1}, {"token": "2014)*10-k", "count": 1}]}]}, "content": {"n_oracle": 68816, "matched": 67919, "forgiven": 452, "missing_material": 439, "missing_trivial": 6, "recall_strict": 0.986965, "recall_effective": 0.993533, "missing_samples": [{"token": "technologies,", "count": 56}, {"token": "financial", "count": 55}, {"token": "consolidated", "count": 53}, {"token": "(continued)", "count": 50}, {"token": "inc.", "count": 46}, {"token": "contents", "count": 44}, {"token": "notes", "count": 29}, {"token": "statements", "count": 26}, {"token": "agilent", "count": 24}, {"token": "table", "count": 9}, {"token": "accompanying", "count": 5}, {"token": "integral", "count": 5}, {"token": "statements.", "count": 4}, {"token": "dateexhibit", "count": 4}, {"token": "numberfiled", "count": 4}, {"token": "reductionconsolidation", "count": 2}, {"token": "118", "count": 1}, {"token": "none.", "count": 1}, {"token": "pricesnumber", "count": 1}, {"token": "valuenumber", "count": 1}, {"token": "marketsdiagnostics", "count": 1}, {"token": "amortizationnet", "count": 1}, {"token": "105", "count": 1}, {"token": "plansnon-u.s.", "count": 1}, {"token": "obligationbenefit", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 69019, "matched": 68747, "forgiven": 272, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996059, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 69019, "matched": 67966, "forgiven": 636, "missing_material": 411, "missing_trivial": 6, "recall_strict": 0.984743, "recall_effective": 0.993958, "missing_samples": [{"token": "technologies,", "count": 56}, {"token": "financial", "count": 55}, {"token": "consolidated", "count": 53}, {"token": "(continued)", "count": 50}, {"token": "inc.", "count": 46}, {"token": "contents", "count": 44}, {"token": "notes", "count": 29}, {"token": "statements", "count": 26}, {"token": "agilent", "count": 24}, {"token": "table", "count": 9}, {"token": "accompanying", "count": 5}, {"token": "integral", "count": 5}, {"token": "statements.", "count": 4}, {"token": "118", "count": 1}, {"token": "none.", "count": 1}, {"token": "105", "count": 1}, {"token": "106", "count": 1}, {"token": "108", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/parabilis_s1__ex1011_424531.pdf", "pages": 96, "oracles": {"pdfium": {"pawls": {"n_oracle": 51095, "matched": 51086, "forgiven": 9, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999824, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 51095, "matched": 50953, "forgiven": 85, "missing_material": 57, "missing_trivial": 0, "recall_strict": 0.997221, "recall_effective": 0.998884, "missing_samples": [{"token": "right.", "count": 4}, {"token": "lease.", "count": 3}, {"token": "warranty).", "count": 2}, {"token": "risk).", "count": 2}, {"token": "10.2", "count": 1}, {"token": "12.4", "count": 1}, {"token": "12.7", "count": 1}, {"token": "12.8", "count": 1}, {"token": "13.3", "count": 1}, {"token": "inc.,", "count": 1}, {"token": "*notwithstanding", "count": 1}, {"token": "rent.", "count": 1}, {"token": "costs.", "count": 1}, {"token": "building.", "count": 1}, {"token": "work).", "count": 1}, {"token": "taxes.", "count": 1}, {"token": "availability.", "count": 1}, {"token": "space.", "count": 1}, {"token": "default.", "count": 1}, {"token": "warranty.", "count": 1}, {"token": "representatives.", "count": 1}, {"token": "allowance.", "count": 1}, {"token": "removal.", "count": 1}, {"token": "services.", "count": 1}, {"token": "systems.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 51083, "matched": 51078, "forgiven": 5, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999902, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 51083, "matched": 50945, "forgiven": 81, "missing_material": 57, "missing_trivial": 0, "recall_strict": 0.997299, "recall_effective": 0.998884, "missing_samples": [{"token": "right.", "count": 4}, {"token": "lease.", "count": 3}, {"token": "warranty).", "count": 2}, {"token": "risk).", "count": 2}, {"token": "10.2", "count": 1}, {"token": "12.4", "count": 1}, {"token": "12.7", "count": 1}, {"token": "12.8", "count": 1}, {"token": "13.3", "count": 1}, {"token": "inc.,", "count": 1}, {"token": "*notwithstanding", "count": 1}, {"token": "rent.", "count": 1}, {"token": "costs.", "count": 1}, {"token": "building.", "count": 1}, {"token": "work).", "count": 1}, {"token": "taxes.", "count": 1}, {"token": "availability.", "count": 1}, {"token": "space.", "count": 1}, {"token": "default.", "count": 1}, {"token": "warranty.", "count": 1}, {"token": "representatives.", "count": 1}, {"token": "allowance.", "count": 1}, {"token": "removal.", "count": 1}, {"token": "services.", "count": 1}, {"token": "systems.", "count": 1}]}}}} +{"doc": "files/pdf/credit_walt_disney.pdf", "pages": 102, "oracles": {"pdfium": {"pawls": {"n_oracle": 46666, "matched": 46619, "forgiven": 47, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998993, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 46666, "matched": 45956, "forgiven": 346, "missing_material": 364, "missing_trivial": 0, "recall_strict": 0.984785, "recall_effective": 0.9922, "missing_samples": [{"token": "page", "count": 54}, {"token": "credit", "count": 48}, {"token": "signature", "count": 47}, {"token": "disney", "count": 40}, {"token": "364-day", "count": 28}, {"token": "walt", "count": 26}, {"token": "2024", "count": 26}, {"token": "the364-day", "count": 25}, {"token": "company", "count": 20}, {"token": "institution:", "count": 20}, {"token": "advances.", "count": 5}, {"token": "29", "count": 1}, {"token": "31", "count": 1}, {"token": "32", "count": 1}, {"token": "etc.", "count": 1}, {"token": "37", "count": 1}, {"token": "44", "count": 1}, {"token": "45", "count": 1}, {"token": "53", "count": 1}, {"token": "55", "count": 1}, {"token": "58", "count": 1}, {"token": "63", "count": 1}, {"token": "67", "count": 1}, {"token": "2.01.", "count": 1}, {"token": "3.01.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 46687, "matched": 46683, "forgiven": 4, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999914, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 46687, "matched": 45968, "forgiven": 353, "missing_material": 366, "missing_trivial": 0, "recall_strict": 0.9846, "recall_effective": 0.992161, "missing_samples": [{"token": "364-day", "count": 55}, {"token": "page", "count": 54}, {"token": "credit", "count": 48}, {"token": "signature", "count": 47}, {"token": "disney", "count": 40}, {"token": "walt", "count": 26}, {"token": "2024", "count": 26}, {"token": "company", "count": 20}, {"token": "institution:", "count": 20}, {"token": "advances.", "count": 5}, {"token": "29", "count": 1}, {"token": "31", "count": 1}, {"token": "32", "count": 1}, {"token": "etc.", "count": 1}, {"token": "37", "count": 1}, {"token": "44", "count": 1}, {"token": "45", "count": 1}, {"token": "53", "count": 1}, {"token": "55", "count": 1}, {"token": "58", "count": 1}, {"token": "63", "count": 1}, {"token": "67", "count": 1}, {"token": "2.01.", "count": 1}, {"token": "3.01.", "count": 1}, {"token": "interest.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/eikon_s1__ex1011_d4f7e0.pdf", "pages": 86, "oracles": {"pdfium": {"pawls": {"n_oracle": 36434, "matched": 35335, "forgiven": 1099, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.969836, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 36434, "matched": 35222, "forgiven": 1106, "missing_material": 106, "missing_trivial": 0, "recall_strict": 0.966734, "recall_effective": 0.997091, "missing_samples": [{"token": "activities.", "count": 4}, {"token": "reports.", "count": 4}, {"token": "eikon.", "count": 3}, {"token": "records.", "count": 3}, {"token": "obligations.", "count": 3}, {"token": "obligation.", "count": 3}, {"token": "milestones.", "count": 3}, {"token": "trademarks.", "count": 3}, {"token": "cooperation.", "count": 3}, {"token": "subcontracting.", "count": 2}, {"token": "general.", "count": 2}, {"token": "notice.", "count": 2}, {"token": "payment.", "count": 2}, {"token": "disclosures.", "count": 2}, {"token": "warranties.", "count": 2}, {"token": "defense.", "count": 2}, {"token": "remedies.", "count": 2}, {"token": "technology.", "count": 1}, {"token": "patents.", "count": 1}, {"token": "control.", "count": 1}, {"token": "impact.", "count": 1}, {"token": "registration.", "count": 1}, {"token": "exception.", "count": 1}, {"token": "request.", "count": 1}, {"token": "products.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 38764, "matched": 38503, "forgiven": 261, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.993267, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 38764, "matched": 38321, "forgiven": 323, "missing_material": 120, "missing_trivial": 0, "recall_strict": 0.988572, "recall_effective": 0.996904, "missing_samples": [{"token": "eikon.", "count": 4}, {"token": "activities.", "count": 4}, {"token": "reports.", "count": 4}, {"token": "law.", "count": 3}, {"token": "impact.", "count": 3}, {"token": "records.", "count": 3}, {"token": "obligations.", "count": 3}, {"token": "obligation.", "count": 3}, {"token": "milestones.", "count": 3}, {"token": "trademarks.", "count": 3}, {"token": "transfer.", "count": 2}, {"token": "patents.", "count": 2}, {"token": "parties.", "count": 2}, {"token": "payment.", "count": 2}, {"token": "disclosures.", "count": 2}, {"token": "warranties.", "count": 2}, {"token": "defense.", "count": 2}, {"token": "remedies.", "count": 2}, {"token": "term.", "count": 1}, {"token": "technology.", "count": 1}, {"token": "territory.", "count": 1}, {"token": "control.", "count": 1}, {"token": "registration.", "count": 1}, {"token": "exception.", "count": 1}, {"token": "3.1.3subcontracting.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/exyn_s1__ex11_a8500b.pdf", "pages": 57, "oracles": {"pdfium": {"pawls": {"n_oracle": 27803, "matched": 27793, "forgiven": 10, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99964, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 27803, "matched": 27657, "forgiven": 55, "missing_material": 91, "missing_trivial": 0, "recall_strict": 0.994749, "recall_effective": 0.996727, "missing_samples": [{"token": "company.", "count": 5}, {"token": "underwriters.", "count": 5}, {"token": "agreements.", "count": 3}, {"token": "agreement.", "count": 2}, {"token": "effect.", "count": 2}, {"token": "warrant.", "count": 2}, {"token": "date.", "count": 2}, {"token": "delivery.", "count": 2}, {"token": "prospectuses.", "count": 2}, {"token": "expenses.", "count": 2}, {"token": "letter.", "count": 2}, {"token": "certificate.", "count": 2}, {"token": "securities.", "count": 1}, {"token": "statement.", "count": 1}, {"token": "requirements.", "count": 1}, {"token": "disclosure.", "count": 1}, {"token": "required.", "count": 1}, {"token": "show.", "count": 1}, {"token": "prospectus.", "count": 1}, {"token": "materials.", "count": 1}, {"token": "information.", "count": 1}, {"token": "filing.", "count": 1}, {"token": "organization.", "count": 1}, {"token": "subsidiaries.", "count": 1}, {"token": "rights.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 27807, "matched": 27795, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999568, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 27807, "matched": 27659, "forgiven": 57, "missing_material": 91, "missing_trivial": 0, "recall_strict": 0.994678, "recall_effective": 0.996727, "missing_samples": [{"token": "company.", "count": 5}, {"token": "underwriters.", "count": 5}, {"token": "agreements.", "count": 3}, {"token": "agreement.", "count": 2}, {"token": "effect.", "count": 2}, {"token": "warrant.", "count": 2}, {"token": "date.", "count": 2}, {"token": "delivery.", "count": 2}, {"token": "prospectuses.", "count": 2}, {"token": "expenses.", "count": 2}, {"token": "letter.", "count": 2}, {"token": "certificate.", "count": 2}, {"token": "securities.", "count": 1}, {"token": "statement.", "count": 1}, {"token": "requirements.", "count": 1}, {"token": "disclosure.", "count": 1}, {"token": "required.", "count": 1}, {"token": "show.", "count": 1}, {"token": "prospectus.", "count": 1}, {"token": "materials.", "count": 1}, {"token": "information.", "count": 1}, {"token": "filing.", "count": 1}, {"token": "organization.", "count": 1}, {"token": "subsidiaries.", "count": 1}, {"token": "rights.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/swarmer_s1__ex1018_765a25.pdf", "pages": 46, "oracles": {"pdfium": {"pawls": {"n_oracle": 16337, "matched": 16324, "forgiven": 13, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999204, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 16337, "matched": 16251, "forgiven": 39, "missing_material": 44, "missing_trivial": 3, "recall_strict": 0.994736, "recall_effective": 0.997123, "missing_samples": [{"token": "alexander", "count": 3}, {"token": "name:", "count": 3}, {"token": "title:", "count": 2}, {"token": "insurance.", "count": 2}, {"token": "page", "count": 1}, {"token": "premises.", "count": 1}, {"token": "summary", "count": 1}, {"token": "*during", "count": 1}, {"token": "fink", "count": 1}, {"token": "costs.", "count": 1}, {"token": "statement.", "count": 1}, {"token": "abatement.", "count": 1}, {"token": "signage.", "count": 1}, {"token": "furniture.", "count": 1}, {"token": "painting.", "count": 1}, {"token": "agreement.", "count": 1}, {"token": "agents.", "count": 1}, {"token": "subrogation.", "count": 1}, {"token": "indemnity.", "count": 1}, {"token": "attornment.", "count": 1}, {"token": "equipment.", "count": 1}, {"token": "liability.", "count": 1}, {"token": "enjoyment.", "count": 1}, {"token": "captions.", "count": 1}, {"token": "offer.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 16335, "matched": 16329, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999633, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 16335, "matched": 16253, "forgiven": 38, "missing_material": 41, "missing_trivial": 3, "recall_strict": 0.99498, "recall_effective": 0.997306, "missing_samples": [{"token": "name:alexander", "count": 3}, {"token": "title:", "count": 2}, {"token": "insurance.", "count": 2}, {"token": "page", "count": 1}, {"token": "premises.", "count": 1}, {"token": "summary", "count": 1}, {"token": "*during", "count": 1}, {"token": "fink", "count": 1}, {"token": "costs.", "count": 1}, {"token": "statement.", "count": 1}, {"token": "abatement.", "count": 1}, {"token": "signage.", "count": 1}, {"token": "furniture.", "count": 1}, {"token": "painting.", "count": 1}, {"token": "agreement.", "count": 1}, {"token": "agents.", "count": 1}, {"token": "subrogation.", "count": 1}, {"token": "indemnity.", "count": 1}, {"token": "attornment.", "count": 1}, {"token": "equipment.", "count": 1}, {"token": "liability.", "count": 1}, {"token": "enjoyment.", "count": 1}, {"token": "captions.", "count": 1}, {"token": "offer.", "count": 1}, {"token": "exhibits.", "count": 1}]}}}} +{"doc": "files/pdf/credit_citizens.pdf", "pages": 69, "oracles": {"pdfium": {"pawls": {"n_oracle": 38522, "matched": 38512, "forgiven": 10, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99974, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 38522, "matched": 38297, "forgiven": 186, "missing_material": 39, "missing_trivial": 0, "recall_strict": 0.994159, "recall_effective": 0.998988, "missing_samples": [{"token": "53", "count": 5}, {"token": "52", "count": 3}, {"token": "38", "count": 2}, {"token": "39", "count": 2}, {"token": "48", "count": 2}, {"token": "51", "count": 2}, {"token": "28", "count": 1}, {"token": "32", "count": 1}, {"token": "33", "count": 1}, {"token": "44", "count": 1}, {"token": "46", "count": 1}, {"token": "8.3", "count": 1}, {"token": "restricted", "count": 1}, {"token": "8.4", "count": 1}, {"token": "burdensome", "count": 1}, {"token": "8.5", "count": 1}, {"token": "8.7", "count": 1}, {"token": "fundamental", "count": 1}, {"token": "changes;", "count": 1}, {"token": "8.8", "count": 1}, {"token": "8.9", "count": 1}, {"token": "8.10", "count": 1}, {"token": "8.11", "count": 1}, {"token": "fiscal", "count": 1}, {"token": "54", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 38521, "matched": 38521, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 38521, "matched": 38306, "forgiven": 176, "missing_material": 39, "missing_trivial": 0, "recall_strict": 0.994419, "recall_effective": 0.998988, "missing_samples": [{"token": "53", "count": 5}, {"token": "52", "count": 3}, {"token": "38", "count": 2}, {"token": "39", "count": 2}, {"token": "48", "count": 2}, {"token": "51", "count": 2}, {"token": "28", "count": 1}, {"token": "32", "count": 1}, {"token": "33", "count": 1}, {"token": "44", "count": 1}, {"token": "46", "count": 1}, {"token": "8.3", "count": 1}, {"token": "restricted", "count": 1}, {"token": "8.4", "count": 1}, {"token": "burdensome", "count": 1}, {"token": "8.5", "count": 1}, {"token": "8.7", "count": 1}, {"token": "fundamental", "count": 1}, {"token": "changes;", "count": 1}, {"token": "8.8", "count": 1}, {"token": "8.9", "count": 1}, {"token": "8.10", "count": 1}, {"token": "8.11", "count": 1}, {"token": "fiscal", "count": 1}, {"token": "54", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/eikon_s1__ex1010_2e241e.pdf", "pages": 36, "oracles": {"pdfium": {"pawls": {"n_oracle": 12580, "matched": 12201, "forgiven": 379, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.969873, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 12580, "matched": 12183, "forgiven": 385, "missing_material": 12, "missing_trivial": 0, "recall_strict": 0.968442, "recall_effective": 0.999046, "missing_samples": [{"token": "obligations.", "count": 2}, {"token": "appointed", "count": 2}, {"token": "title:", "count": 2}, {"token": "shareholder", "count": 2}, {"token": "signatory", "count": 2}, {"token": "indications.", "count": 1}, {"token": "survival.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 13359, "matched": 13271, "forgiven": 88, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.993413, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 13359, "matched": 13232, "forgiven": 114, "missing_material": 12, "missing_trivial": 1, "recall_strict": 0.990493, "recall_effective": 0.999027, "missing_samples": [{"token": "obligations.", "count": 2}, {"token": "appointed", "count": 2}, {"token": "title:", "count": 2}, {"token": "shareholder", "count": 2}, {"token": "signatory", "count": 2}, {"token": "indications.", "count": 1}, {"token": "9.4survival.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/quantinuum_s1__exhibit33sx1_eefad8.pdf", "pages": 28, "oracles": {"pdfium": {"pawls": {"n_oracle": 10739, "matched": 10738, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999907, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 10739, "matched": 10724, "forgiven": 10, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.998603, "recall_effective": 0.999534, "missing_samples": [{"token": "18", "count": 2}, {"token": "17", "count": 1}, {"token": "19", "count": 1}, {"token": "meeting.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 10739, "matched": 10739, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 10739, "matched": 10725, "forgiven": 9, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.998696, "recall_effective": 0.999534, "missing_samples": [{"token": "18", "count": 2}, {"token": "17", "count": 1}, {"token": "19", "count": 1}, {"token": "meeting.", "count": 1}]}}}} +{"doc": "tests/fixtures/EtonPharmaceuticalsInc_20191114_10-Q_EX-10.1_11893941_EX-10.1_Development_Agreement_ZrZJLLv.pdf", "pages": 23, "oracles": {"pdfium": {"pawls": {"n_oracle": 10256, "matched": 10254, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999805, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 10256, "matched": 10047, "forgiven": 57, "missing_material": 152, "missing_trivial": 0, "recall_strict": 0.979622, "recall_effective": 0.985179, "missing_samples": [{"token": "pharmaceuticals,", "count": 23}, {"token": "inc.,", "count": 23}, {"token": "source:", "count": 23}, {"token": "10-q,", "count": 23}, {"token": "11/14/2019", "count": 23}, {"token": "confidential", "count": 10}, {"token": "warranties.", "count": 3}, {"token": "products.", "count": 1}, {"token": "numbers.", "count": 1}, {"token": "recalls.", "count": 1}, {"token": "fees.", "count": 1}, {"token": "milestones.", "count": 1}, {"token": "true-ups.", "count": 1}, {"token": "term.", "count": 1}, {"token": "audits.", "count": 1}, {"token": "information.", "count": 1}, {"token": "accounting.", "count": 1}, {"token": "infringement.", "count": 1}, {"token": "announcement.", "count": 1}, {"token": "breach.", "count": 1}, {"token": "bankruptcy.", "count": 1}, {"token": "parties.", "count": 1}, {"token": "obligations.", "count": 1}, {"token": "language.", "count": 1}, {"token": "majeure.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 10256, "matched": 10256, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 10256, "matched": 10049, "forgiven": 55, "missing_material": 152, "missing_trivial": 0, "recall_strict": 0.979817, "recall_effective": 0.985179, "missing_samples": [{"token": "pharmaceuticals,", "count": 23}, {"token": "inc.,", "count": 23}, {"token": "source:", "count": 23}, {"token": "10-q,", "count": 23}, {"token": "11/14/2019", "count": 23}, {"token": "confidential", "count": 10}, {"token": "warranties.", "count": 3}, {"token": "products.", "count": 1}, {"token": "numbers.", "count": 1}, {"token": "recalls.", "count": 1}, {"token": "fees.", "count": 1}, {"token": "milestones.", "count": 1}, {"token": "true-ups.", "count": 1}, {"token": "term.", "count": 1}, {"token": "audits.", "count": 1}, {"token": "information.", "count": 1}, {"token": "accounting.", "count": 1}, {"token": "infringement.", "count": 1}, {"token": "announcement.", "count": 1}, {"token": "breach.", "count": 1}, {"token": "bankruptcy.", "count": 1}, {"token": "parties.", "count": 1}, {"token": "obligations.", "count": 1}, {"token": "language.", "count": 1}, {"token": "majeure.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/generate_bio_s1__ex33_a3ab0f.pdf", "pages": 21, "oracles": {"pdfium": {"pawls": {"n_oracle": 7398, "matched": 7397, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999865, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 7398, "matched": 7341, "forgiven": 7, "missing_material": 50, "missing_trivial": 0, "recall_strict": 0.992295, "recall_effective": 0.993241, "missing_samples": [{"token": "meetings.", "count": 5}, {"token": "meeting.", "count": 4}, {"token": "quorum.", "count": 2}, {"token": "consent.", "count": 2}, {"token": "directors.", "count": 2}, {"token": "qualification.", "count": 2}, {"token": "tenure.", "count": 2}, {"token": "vacancies.", "count": 2}, {"token": "authority.", "count": 2}, {"token": "list.", "count": 1}, {"token": "adjournments.", "count": 1}, {"token": "proxies.", "count": 1}, {"token": "incorporation.", "count": 1}, {"token": "election.", "count": 1}, {"token": "action.", "count": 1}, {"token": "consents.", "count": 1}, {"token": "powers.", "count": 1}, {"token": "board.", "count": 1}, {"token": "removal.", "count": 1}, {"token": "resignation.", "count": 1}, {"token": "equipment.", "count": 1}, {"token": "titles.", "count": 1}, {"token": "secretaries.", "count": 1}, {"token": "officer.", "count": 1}, {"token": "presidents.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 7386, "matched": 7374, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998375, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 7386, "matched": 7319, "forgiven": 17, "missing_material": 50, "missing_trivial": 0, "recall_strict": 0.990929, "recall_effective": 0.99323, "missing_samples": [{"token": "meetings.", "count": 5}, {"token": "meeting.", "count": 4}, {"token": "quorum.", "count": 2}, {"token": "consent.", "count": 2}, {"token": "directors.", "count": 2}, {"token": "qualification.", "count": 2}, {"token": "tenure.", "count": 2}, {"token": "vacancies.", "count": 2}, {"token": "authority.", "count": 2}, {"token": "list.", "count": 1}, {"token": "adjournments.", "count": 1}, {"token": "proxies.", "count": 1}, {"token": "incorporation.", "count": 1}, {"token": "election.", "count": 1}, {"token": "action.", "count": 1}, {"token": "consents.", "count": 1}, {"token": "powers.", "count": 1}, {"token": "board.", "count": 1}, {"token": "removal.", "count": 1}, {"token": "resignation.", "count": 1}, {"token": "equipment.", "count": 1}, {"token": "titles.", "count": 1}, {"token": "secretaries.", "count": 1}, {"token": "officer.", "count": 1}, {"token": "presidents.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/forbright_s1__exhibit1012sx1_11ea38.pdf", "pages": 17, "oracles": {"pdfium": {"pawls": {"n_oracle": 6871, "matched": 6870, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999854, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 6871, "matched": 6848, "forgiven": 6, "missing_material": 17, "missing_trivial": 0, "recall_strict": 0.996653, "recall_effective": 0.997526, "missing_samples": [{"token": "etc.", "count": 3}, {"token": "expenses.", "count": 1}, {"token": "control.", "count": 1}, {"token": "claims.", "count": 1}, {"token": "proof.", "count": 1}, {"token": "harbor.", "count": 1}, {"token": "presumptions.", "count": 1}, {"token": "insurance.", "count": 1}, {"token": "subrogation.", "count": 1}, {"token": "severability.", "count": 1}, {"token": "notices.", "count": 1}, {"token": "headings.", "count": 1}, {"token": "counterparts.", "count": 1}, {"token": "jurisdiction.", "count": 1}, {"token": "trial.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 6871, "matched": 6871, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 6871, "matched": 6849, "forgiven": 5, "missing_material": 17, "missing_trivial": 0, "recall_strict": 0.996798, "recall_effective": 0.997526, "missing_samples": [{"token": "etc.", "count": 3}, {"token": "expenses.", "count": 1}, {"token": "control.", "count": 1}, {"token": "claims.", "count": 1}, {"token": "proof.", "count": 1}, {"token": "harbor.", "count": 1}, {"token": "presumptions.", "count": 1}, {"token": "insurance.", "count": 1}, {"token": "subrogation.", "count": 1}, {"token": "severability.", "count": 1}, {"token": "notices.", "count": 1}, {"token": "headings.", "count": 1}, {"token": "counterparts.", "count": 1}, {"token": "jurisdiction.", "count": 1}, {"token": "trial.", "count": 1}]}}}} +{"doc": "files/pdf/CreditcardscomInc_20070810_S-1_EX-10.33_362297_EX-10.33_Affiliate Agreement.pdf", "pages": 12, "oracles": {"pdfium": {"pawls": {"n_oracle": 3955, "matched": 3955, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3955, "matched": 3895, "forgiven": 0, "missing_material": 60, "missing_trivial": 0, "recall_strict": 0.984829, "recall_effective": 0.984829, "missing_samples": [{"token": "source:", "count": 12}, {"token": "creditcards.com,", "count": 12}, {"token": "inc.,", "count": 12}, {"token": "s-1,", "count": 12}, {"token": "8/10/2007", "count": 12}]}}, "plumber": {"pawls": {"n_oracle": 3955, "matched": 3955, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3955, "matched": 3895, "forgiven": 0, "missing_material": 60, "missing_trivial": 0, "recall_strict": 0.984829, "recall_effective": 0.984829, "missing_samples": [{"token": "source:", "count": 12}, {"token": "creditcards.com,", "count": 12}, {"token": "inc.,", "count": 12}, {"token": "s-1,", "count": 12}, {"token": "8/10/2007", "count": 12}]}}}} +{"doc": "tests/fixtures/s1/fervo_s1__exhibit993sx1_6f30d7.pdf", "pages": 16, "oracles": {"pdfium": {"pawls": {"n_oracle": 4071, "matched": 3999, "forgiven": 72, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.982314, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4071, "matched": 3989, "forgiven": 75, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.979858, "recall_effective": 0.998281, "missing_samples": [{"token": "(10", "count": 3}, {"token": "degolyer", "count": 1}, {"token": "macnaughton", "count": 1}, {"token": "11", "count": 1}, {"token": "12", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 4040, "matched": 4015, "forgiven": 25, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.993812, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4040, "matched": 4004, "forgiven": 33, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.991089, "recall_effective": 0.999257, "missing_samples": [{"token": "macnaughton", "count": 1}, {"token": "11", "count": 1}, {"token": "12", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/cerebras_s1__exhibit1013esx_f124a6.pdf", "pages": 11, "oracles": {"pdfium": {"pawls": {"n_oracle": 2227, "matched": 2223, "forgiven": 4, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998204, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 2227, "matched": 2212, "forgiven": 9, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.993264, "recall_effective": 0.997306, "missing_samples": [{"token": "project", "count": 2}, {"token": "#2024", "count": 1}, {"token": "/s/", "count": 1}, {"token": "andrew", "count": 1}, {"token": "feldman", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 2224, "matched": 2217, "forgiven": 7, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996853, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 2224, "matched": 2206, "forgiven": 12, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.991906, "recall_effective": 0.997302, "missing_samples": [{"token": "project", "count": 2}, {"token": "#2024", "count": 1}, {"token": "/s/", "count": 1}, {"token": "andrew", "count": 1}, {"token": "feldman", "count": 1}]}}}} +{"doc": "tests/fixtures/USC Title 1 - CHAPTER 1.pdf", "pages": 9, "oracles": {"pdfium": {"pawls": {"n_oracle": 4855, "matched": 4849, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998764, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4855, "matched": 4834, "forgiven": 19, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.995675, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 4852, "matched": 4852, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4852, "matched": 4837, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.996908, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}}} +{"doc": "files/pdf/Beyond Examples- High-level Automated Reasoning Paradigm in In-Context Learning via MCTS.pdf", "pages": 13, "oracles": {"pdfium": {"pawls": {"n_oracle": 8938, "matched": 8323, "forgiven": 599, "missing_material": 16, "missing_trivial": 0, "recall_strict": 0.931193, "recall_effective": 0.99821, "flagged_pages": [{"page": 0, "n_oracle": 622, "matched": 598, "forgiven": 22, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.961415, "recall_effective": 0.996785, "missing_samples": [{"token": "arxiv:2411.18478v1", "count": 1}, {"token": "[cs.cl]", "count": 1}]}, {"page": 3, "n_oracle": 641, "matched": 259, "forgiven": 376, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.404056, "recall_effective": 0.99064, "missing_samples": [{"token": "sa\u2192ost\u2192ost:", "count": 2}, {"token": "qt", "count": 1}, {"token": "cot:", "count": 1}, {"token": "count,", "count": 1}, {"token": "semantics", "count": 1}]}, {"page": 4, "n_oracle": 786, "matched": 764, "forgiven": 21, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.97201, "recall_effective": 0.998728, "missing_samples": [{"token": "szepesvari", "count": 1}]}, {"page": 8, "n_oracle": 549, "matched": 532, "forgiven": 14, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.969035, "recall_effective": 0.994536, "missing_samples": [{"token": "\u0373\u03ef\u03f3\u0358\u03c2y", "count": 1}, {"token": "\u0373\u03ed\u03ed\u0358\u03b8y", "count": 1}, {"token": "\u0373\u03ba\u03f3\u0358\u03efy", "count": 1}]}, {"page": 10, "n_oracle": 706, "matched": 682, "forgiven": 22, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.966006, "recall_effective": 0.997167, "missing_samples": [{"token": "tramer,", "count": 1}, {"token": "szepesvari,", "count": 1}]}, {"page": 11, "n_oracle": 702, "matched": 681, "forgiven": 19, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.970085, "recall_effective": 0.997151, "missing_samples": [{"token": "swiechowski,", "count": 1}, {"token": "mandziuk,", "count": 1}]}]}, "content": {"n_oracle": 8938, "matched": 8139, "forgiven": 663, "missing_material": 117, "missing_trivial": 19, "recall_strict": 0.910606, "recall_effective": 0.984784, "missing_samples": [{"token": "beyond", "count": 12}, {"token": "examples:", "count": 12}, {"token": "automated", "count": 12}, {"token": "via", "count": 12}, {"token": "in-context", "count": 11}, {"token": "high-level", "count": 9}, {"token": "paradigm", "count": 3}, {"token": "that...", "count": 3}, {"token": "follows...", "count": 3}, {"token": "time...", "count": 3}, {"token": "learning", "count": 2}, {"token": "subquestion", "count": 2}, {"token": "count,", "count": 2}, {"token": "complexity,", "count": 2}, {"token": "sa\u2192ost\u2192ost:", "count": 2}, {"token": "necessity", "count": 1}, {"token": "challenging", "count": 1}, {"token": "scenarios.", "count": 1}, {"token": "limitations,", "count": 1}, {"token": "*equal", "count": 1}, {"token": "arxiv:2411.18478v1", "count": 1}, {"token": "[cs.cl]", "count": 1}, {"token": "s0...t", "count": 1}, {"token": "(c)", "count": 1}, {"token": "(ours)", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 3754, "matched": 2303, "forgiven": 1421, "missing_material": 30, "missing_trivial": 0, "recall_strict": 0.613479, "recall_effective": 0.992009, "flagged_pages": [{"page": 0, "n_oracle": 251, "matched": 177, "forgiven": 72, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.705179, "recall_effective": 0.992032, "missing_samples": [{"token": "]lc.sc[", "count": 1}, {"token": "1v87481.1142:vixra", "count": 1}]}, {"page": 1, "n_oracle": 362, "matched": 232, "forgiven": 129, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.640884, "recall_effective": 0.997238, "missing_samples": [{"token": "\u0301wiechowskietal.,2023;kohetal.,2024).", "count": 1}]}, {"page": 3, "n_oracle": 530, "matched": 157, "forgiven": 347, "missing_material": 26, "missing_trivial": 0, "recall_strict": 0.296226, "recall_effective": 0.950943, "missing_samples": [{"token": "rrrreeeewwwwaaaarrrrdddd", "count": 2}, {"token": "mmmmooooddddeeeellll", "count": 2}, {"token": "qqqqqqqqq...", "count": 1}, {"token": "123456789", "count": 1}, {"token": "oaoaaaaaa", "count": 1}, {"token": "t\u2192t\u2192\u2192\u2192\u2192\u2192\u2192", "count": 1}, {"token": "odoodo", "count": 1}, {"token": "scsscs", "count": 1}, {"token": "lleeggeenndd", "count": 1}, {"token": "qqqquuuueeeessssttttiiiioooonnnn", "count": 1}, {"token": "aaaattttoooommmm", "count": 1}, {"token": "aaaaccccttttiiiioooonnnn", "count": 1}, {"token": "ffffoooorrrrwwwwaaaarrrrdddd", "count": 1}, {"token": "rrrreeeeaaaassssoooonnnniiiinnnngggg", "count": 1}, {"token": "oooouuuuttttppppuuuutttt", "count": 1}, {"token": "((((oooorrrrmmmm))))", "count": 1}, {"token": "pppprrrroooocccceeeessssssss", "count": 1}, {"token": "((((pppprrrrmmmm))))", "count": 1}, {"token": "ccccoooonnnnssssiiiisssstttteeeennnnccccyyyy----bbbbaaaasssseeeedddd", "count": 1}, {"token": "vvvveeeerrrriiiiffffiiiiccccaaaattttiiiioooonnnn", "count": 1}, {"token": "ffiinnaall", "count": 1}, {"token": "ssoolluuttiioonn", "count": 1}, {"token": "cot:", "count": 1}, {"token": "count,", "count": 1}]}, {"page": 10, "n_oracle": 287, "matched": 196, "forgiven": 90, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.682927, "recall_effective": 0.996516, "missing_samples": [{"token": "\u0308rnkranz,j.,scheffer,t.,andspiliopoulou,", "count": 1}]}]}, "content": {"n_oracle": 3754, "matched": 2415, "forgiven": 1262, "missing_material": 62, "missing_trivial": 15, "recall_strict": 0.643314, "recall_effective": 0.979489, "missing_samples": [{"token": "beyondexamples:high-levelautomatedreasoningparadigminin-contextlearningviamcts", "count": 11}, {"token": ",...,s", "count": 3}, {"token": "that...", "count": 3}, {"token": "follows...", "count": 3}, {"token": "time...", "count": 3}, {"token": "rrrreeeewwwwaaaarrrrdddd", "count": 2}, {"token": "mmmmooooddddeeeellll", "count": 2}, {"token": "necessityforhumaninterventioninchallenging", "count": 1}, {"token": "limitations,", "count": 1}, {"token": "*equal", "count": 1}, {"token": "]lc.sc[", "count": 1}, {"token": "1v87481.1142:vixra", "count": 1}, {"token": "\u0301wiechowskietal.,2023;kohetal.,2024).", "count": 1}, {"token": "0...t", "count": 1}, {"token": "(ours)", "count": 1}, {"token": "qqqqqqqqq...", "count": 1}, {"token": "123456789", "count": 1}, {"token": "oaoaaaaaa", "count": 1}, {"token": "t\u2192t\u2192\u2192\u2192\u2192\u2192\u2192", "count": 1}, {"token": "odoodo", "count": 1}, {"token": "scsscs", "count": 1}, {"token": "lleeggeenndd", "count": 1}, {"token": "qqqquuuueeeessssttttiiiioooonnnn", "count": 1}, {"token": "aaaattttoooommmm", "count": 1}, {"token": "aaaaccccttttiiiioooonnnn", "count": 1}]}}}} +{"doc": "tests/fixtures/files/doc_1_pdf_file.pdf", "pages": 9, "oracles": {"pdfium": {"pawls": {"n_oracle": 4855, "matched": 4849, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998764, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4855, "matched": 4834, "forgiven": 19, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.995675, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 4852, "matched": 4852, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4852, "matched": 4837, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.996908, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}}} +{"doc": "tests/fixtures/files/doc_2_pdf_file.pdf", "pages": 9, "oracles": {"pdfium": {"pawls": {"n_oracle": 4855, "matched": 4849, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998764, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4855, "matched": 4834, "forgiven": 19, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.995675, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 4852, "matched": 4852, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4852, "matched": 4837, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.996908, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}}} +{"doc": "tests/fixtures/files/doc_3_pdf_file.pdf", "pages": 9, "oracles": {"pdfium": {"pawls": {"n_oracle": 4855, "matched": 4849, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998764, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4855, "matched": 4834, "forgiven": 19, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.995675, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 4852, "matched": 4852, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4852, "matched": 4837, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.996908, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}}} +{"doc": "tests/fixtures/files/doc_4_pdf_file.pdf", "pages": 9, "oracles": {"pdfium": {"pawls": {"n_oracle": 4855, "matched": 4849, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998764, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4855, "matched": 4834, "forgiven": 19, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.995675, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 4852, "matched": 4852, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4852, "matched": 4837, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.996908, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}}} +{"doc": "files/pdf/SouthernStarEnergyInc_20051202_SB-2A_EX-9_801890_EX-9_Affiliate Agreement.pdf", "pages": 8, "oracles": {"pdfium": {"pawls": {"n_oracle": 3607, "matched": 3605, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999446, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3607, "matched": 3547, "forgiven": 4, "missing_material": 56, "missing_trivial": 0, "recall_strict": 0.983366, "recall_effective": 0.984475, "missing_samples": [{"token": "source:", "count": 8}, {"token": "southern", "count": 8}, {"token": "star", "count": 8}, {"token": "energy", "count": 8}, {"token": "inc.,", "count": 8}, {"token": "sb-2/a,", "count": 8}, {"token": "12/2/2005", "count": 8}]}}, "plumber": {"pawls": {"n_oracle": 3607, "matched": 3607, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3607, "matched": 3549, "forgiven": 2, "missing_material": 56, "missing_trivial": 0, "recall_strict": 0.98392, "recall_effective": 0.984475, "missing_samples": [{"token": "source:", "count": 8}, {"token": "southern", "count": 8}, {"token": "star", "count": 8}, {"token": "energy", "count": 8}, {"token": "inc.,", "count": 8}, {"token": "sb-2/a,", "count": 8}, {"token": "12/2/2005", "count": 8}]}}}} +{"doc": "tests/fixtures/sample.pdf", "pages": 9, "oracles": {"pdfium": {"pawls": {"n_oracle": 4855, "matched": 4849, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998764, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4855, "matched": 4834, "forgiven": 19, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.995675, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 4852, "matched": 4852, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 4852, "matched": 4837, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.996908, "recall_effective": 0.999588, "missing_samples": [{"token": "\"company\"", "count": 1}, {"token": "assigns.", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/exyn_s1__ex1010_de36c8.pdf", "pages": 7, "oracles": {"pdfium": {"pawls": {"n_oracle": 3534, "matched": 3534, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3534, "matched": 3508, "forgiven": 5, "missing_material": 21, "missing_trivial": 0, "recall_strict": 0.992643, "recall_effective": 0.994058, "missing_samples": [{"token": "washington", "count": 1}, {"token": "lender:", "count": 1}, {"token": "western", "count": 1}, {"token": "alliance", "count": 1}, {"token": "bank,", "count": 1}, {"token": "arizona", "count": 1}, {"token": "street", "count": 1}, {"token": "phoenix,", "count": 1}, {"token": "85004", "count": 1}, {"token": "private", "count": 1}, {"token": "limited", "count": 1}, {"token": "4th", "count": 1}, {"token": "floor,", "count": 1}, {"token": "plot", "count": 1}, {"token": "1269,", "count": 1}, {"token": "jubilee", "count": 1}, {"token": "hills", "count": 1}, {"token": "hyderabad", "count": 1}, {"token": "500033", "count": 1}, {"token": "telangana,", "count": 1}, {"token": "india", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 3532, "matched": 3530, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.999434, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3532, "matched": 3504, "forgiven": 7, "missing_material": 21, "missing_trivial": 0, "recall_strict": 0.992072, "recall_effective": 0.994054, "missing_samples": [{"token": "lender:", "count": 1}, {"token": "western", "count": 1}, {"token": "alliance", "count": 1}, {"token": "bank,", "count": 1}, {"token": "washington", "count": 1}, {"token": "arizona", "count": 1}, {"token": "street", "count": 1}, {"token": "phoenix,", "count": 1}, {"token": "85004", "count": 1}, {"token": "private", "count": 1}, {"token": "limited", "count": 1}, {"token": "4th", "count": 1}, {"token": "floor,", "count": 1}, {"token": "plot", "count": 1}, {"token": "1269,", "count": 1}, {"token": "jubilee", "count": 1}, {"token": "hills", "count": 1}, {"token": "hyderabad", "count": 1}, {"token": "500033", "count": 1}, {"token": "telangana,", "count": 1}, {"token": "india", "count": 1}]}}}} +{"doc": "tests/fixtures/contracts/fw_garver.pdf", "pages": 6, "oracles": {"pdfium": {"pawls": {"n_oracle": 1176, "matched": 1131, "forgiven": 38, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.961735, "recall_effective": 0.994048, "flagged_pages": [{"page": 1, "n_oracle": 237, "matched": 214, "forgiven": 19, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.902954, "recall_effective": 0.983122, "missing_samples": [{"token": "4popr", "count": 1}, {"token": "uoq", "count": 1}, {"token": "c\\,,]],,", "count": 1}, {"token": "nbzp5.q.4", "count": 1}]}, {"page": 4, "n_oracle": 68, "matched": 55, "forgiven": 10, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.808824, "recall_effective": 0.955882, "missing_samples": [{"token": "1.64", "count": 1}, {"token": "gpu", "count": 1}, {"token": "3.3%", "count": 1}]}]}, "content": {"n_oracle": 1176, "matched": 1099, "forgiven": 53, "missing_material": 24, "missing_trivial": 0, "recall_strict": 0.934524, "recall_effective": 0.979592, "missing_samples": [{"token": "june", "count": 1}, {"token": "1790", "count": 1}, {"token": "revision", "count": 1}, {"token": "13,", "count": 1}, {"token": "2025", "count": 1}, {"token": "page", "count": 1}, {"token": "executed", "count": 1}, {"token": "4popr", "count": 1}, {"token": "uoq", "count": 1}, {"token": "c\\,,]],,", "count": 1}, {"token": "nbzp5.q.4", "count": 1}, {"token": "109290", "count": 1}, {"token": "1.64", "count": 1}, {"token": "44226", "count": 1}, {"token": "st,", "count": 1}, {"token": "gpu", "count": 1}, {"token": "15d", "count": 1}, {"token": "3.3%", "count": 1}, {"token": "rsii", "count": 1}, {"token": "did", "count": 1}, {"token": "you", "count": 1}, {"token": "text", "count": 1}, {"token": "field", "count": 1}, {"token": "csc)", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 1220, "matched": 1097, "forgiven": 120, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.89918, "recall_effective": 0.997541, "flagged_pages": [{"page": 1, "n_oracle": 213, "matched": 183, "forgiven": 29, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.859155, "recall_effective": 0.995305, "missing_samples": [{"token": "c\\,,]],,", "count": 1}]}, {"page": 4, "n_oracle": 169, "matched": 99, "forgiven": 68, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.585799, "recall_effective": 0.988166, "missing_samples": [{"token": "ikrcigonc-uvi_", "count": 1}, {"token": "b`g", "count": 1}]}]}, "content": {"n_oracle": 1220, "matched": 1067, "forgiven": 131, "missing_material": 21, "missing_trivial": 1, "recall_strict": 0.87459, "recall_effective": 0.981967, "missing_samples": [{"token": "june", "count": 1}, {"token": "1790", "count": 1}, {"token": "revision", "count": 1}, {"token": "13,", "count": 1}, {"token": "2025", "count": 1}, {"token": "executed", "count": 1}, {"token": "c\\,,]],,", "count": 1}, {"token": "bijay", "count": 1}, {"token": "109290", "count": 1}, {"token": "163..16.fi0.", "count": 1}, {"token": "ikrcigonc-uvi_", "count": 1}, {"token": "b`g", "count": 1}, {"token": "33.%", "count": 1}, {"token": "st,st,44226", "count": 1}, {"token": "rsii", "count": 1}, {"token": "15d", "count": 1}, {"token": "did", "count": 1}, {"token": "you", "count": 1}, {"token": "text", "count": 1}, {"token": "field", "count": 1}, {"token": "csc)", "count": 1}]}}}} +{"doc": "tests/fixtures/legal100/fw_komatsu_rangel.pdf", "pages": 6, "oracles": {"pdfium": {"pawls": {"n_oracle": 925, "matched": 925, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 925, "matched": 889, "forgiven": 12, "missing_material": 18, "missing_trivial": 6, "recall_strict": 0.961081, "recall_effective": 0.974054, "missing_samples": [{"token": "date:", "count": 3}, {"token": "12/", "count": 3}, {"token": "08/", "count": 3}, {"token": "2023", "count": 3}, {"token": "page", "count": 3}, {"token": "revision", "count": 1}, {"token": "secretary", "count": 1}, {"token": "0____________________________", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 850, "matched": 810, "forgiven": 40, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.952941, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 850, "matched": 787, "forgiven": 45, "missing_material": 12, "missing_trivial": 6, "recall_strict": 0.925882, "recall_effective": 0.978824, "missing_samples": [{"token": "page", "count": 3}, {"token": "date:", "count": 3}, {"token": "12/08/2023", "count": 3}, {"token": "secretary", "count": 1}, {"token": "revision", "count": 1}, {"token": "0____________________________", "count": 1}]}}}} +{"doc": "tests/fixtures/contracts/fw_vertigis.pdf", "pages": 5, "oracles": {"pdfium": {"pawls": {"n_oracle": 1058, "matched": 1049, "forgiven": 8, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.991493, "recall_effective": 0.999055, "flagged_pages": [{"page": 1, "n_oracle": 182, "matched": 181, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.994505, "recall_effective": 0.994505, "missing_samples": [{"token": "2024date", "count": 1}]}]}, "content": {"n_oracle": 1058, "matched": 981, "forgiven": 43, "missing_material": 34, "missing_trivial": 0, "recall_strict": 0.927221, "recall_effective": 0.967864, "missing_samples": [{"token": "sixth", "count": 4}, {"token": "seventh", "count": 4}, {"token": "secretary", "count": 4}, {"token": "page", "count": 4}, {"token": "no.", "count": 3}, {"token": "amendment", "count": 3}, {"token": "52004", "count": 3}, {"token": "city", "count": 2}, {"token": "fort", "count": 2}, {"token": "contract", "count": 2}, {"token": "renewal", "count": 1}, {"token": "term.", "count": 1}, {"token": "2024date", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 972, "matched": 907, "forgiven": 65, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.933128, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 972, "matched": 872, "forgiven": 74, "missing_material": 26, "missing_trivial": 0, "recall_strict": 0.897119, "recall_effective": 0.973251, "missing_samples": [{"token": "seventh", "count": 4}, {"token": "secretary", "count": 4}, {"token": "page", "count": 4}, {"token": "of4", "count": 4}, {"token": "52004", "count": 3}, {"token": "contractno.", "count": 3}, {"token": "sixthamendment", "count": 2}, {"token": "fortworthcity", "count": 2}]}}}} +{"doc": "tests/fixtures/contracts/fw_nctcog.pdf", "pages": 5, "oracles": {"pdfium": {"pawls": {"n_oracle": 1303, "matched": 1295, "forgiven": 8, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99386, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1303, "matched": 1260, "forgiven": 15, "missing_material": 28, "missing_trivial": 0, "recall_strict": 0.966999, "recall_effective": 0.978511, "missing_samples": [{"token": "envelope", "count": 3}, {"token": "id:", "count": 3}, {"token": "2439509d-", "count": 3}, {"token": "2b97-", "count": 3}, {"token": "43f2-", "count": 3}, {"token": "926a-", "count": 3}, {"token": "a50a3860f40f", "count": 3}, {"token": "ft.", "count": 1}, {"token": "worth,", "count": 1}, {"token": "tx", "count": 1}, {"token": "349d83294e7946e...", "count": 1}, {"token": "241a3fc61a444c4...", "count": 1}, {"token": "6e2emma52648e..", "count": 1}, {"token": "f453b92f422..", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 1257, "matched": 1221, "forgiven": 33, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.97136, "recall_effective": 0.997613, "flagged_pages": [{"page": 3, "n_oracle": 122, "matched": 117, "forgiven": 3, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.959016, "recall_effective": 0.983607, "missing_samples": [{"token": "qfoq!n", "count": 1}, {"token": "yqa0", "count": 1}]}, {"page": 4, "n_oracle": 210, "matched": 196, "forgiven": 13, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.933333, "recall_effective": 0.995238, "missing_samples": [{"token": "followingordrdeer:r", "count": 1}]}]}, "content": {"n_oracle": 1257, "matched": 1192, "forgiven": 37, "missing_material": 28, "missing_trivial": 0, "recall_strict": 0.94829, "recall_effective": 0.977725, "missing_samples": [{"token": "envelope", "count": 3}, {"token": "id:", "count": 3}, {"token": "2439509d-", "count": 3}, {"token": "2b97-", "count": 3}, {"token": "43f2-", "count": 3}, {"token": "926a-a50a3860f40f", "count": 3}, {"token": "ft.", "count": 1}, {"token": "worth,", "count": 1}, {"token": "tx", "count": 1}, {"token": "349d83294e7946e...", "count": 1}, {"token": "241a3fc61a444c4...", "count": 1}, {"token": "qfoq!n", "count": 1}, {"token": "yqa0", "count": 1}, {"token": "f453b92f422..", "count": 1}, {"token": "6e2emma52648e..", "count": 1}, {"token": "followingordrdeer:r", "count": 1}]}}}} +{"doc": "tests/fixtures/legal100/fw_granicus.pdf", "pages": 5, "oracles": {"pdfium": {"pawls": {"n_oracle": 907, "matched": 891, "forgiven": 15, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.982359, "recall_effective": 0.998897, "flagged_pages": [{"page": 1, "n_oracle": 188, "matched": 180, "forgiven": 7, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.957447, "recall_effective": 0.994681, "missing_samples": [{"token": "sarah14,", "count": 1}]}]}, "content": {"n_oracle": 907, "matched": 810, "forgiven": 49, "missing_material": 46, "missing_trivial": 2, "recall_strict": 0.893054, "recall_effective": 0.947078, "missing_samples": [{"token": "page", "count": 5}, {"token": "envelope", "count": 4}, {"token": "id:", "count": 4}, {"token": "a9ce2100-", "count": 4}, {"token": "9500-", "count": 4}, {"token": "4775-", "count": 4}, {"token": "861b-", "count": 4}, {"token": "90f389fe6a3f", "count": 4}, {"token": "docusign", "count": 3}, {"token": "prepared:", "count": 3}, {"token": "2025", "count": 2}, {"token": "24", "count": 2}, {"token": "renewal", "count": 1}, {"token": "date:", "count": 1}, {"token": "sarah14,", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 868, "matched": 831, "forgiven": 37, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.957373, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 868, "matched": 766, "forgiven": 61, "missing_material": 39, "missing_trivial": 2, "recall_strict": 0.882488, "recall_effective": 0.952765, "missing_samples": [{"token": "page", "count": 5}, {"token": "envelope", "count": 4}, {"token": "id:", "count": 4}, {"token": "a9ce2100-", "count": 4}, {"token": "90f389fe6a3f", "count": 4}, {"token": "docusign", "count": 3}, {"token": "prepared:", "count": 3}, {"token": "9500-4775-861b-", "count": 2}, {"token": "2025", "count": 2}, {"token": "24", "count": 2}, {"token": "renewal", "count": 1}, {"token": "date:", "count": 1}, {"token": "9500-", "count": 1}, {"token": "4775-", "count": 1}, {"token": "861b-", "count": 1}, {"token": "q-4261", "count": 1}]}}}} +{"doc": "files/pdf/EcoScienceSolutionsInc_20180406_8-K_EX-10.1_11135398_EX-10.1_Sponsorship_Agreement.pdf", "pages": 3, "oracles": {"pdfium": {"pawls": {"n_oracle": 1235, "matched": 1235, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1235, "matched": 1194, "forgiven": 8, "missing_material": 16, "missing_trivial": 17, "recall_strict": 0.966802, "recall_effective": 0.973279, "missing_samples": [{"token": "solutions,", "count": 3}, {"token": "source:", "count": 3}, {"token": "8-k,", "count": 3}, {"token": "4/6/2018", "count": 3}, {"token": "science", "count": 2}, {"token": "inc.,", "count": 2}]}}, "plumber": {"pawls": {"n_oracle": 1235, "matched": 1235, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1235, "matched": 1194, "forgiven": 8, "missing_material": 16, "missing_trivial": 17, "recall_strict": 0.966802, "recall_effective": 0.973279, "missing_samples": [{"token": "solutions,", "count": 3}, {"token": "source:", "count": 3}, {"token": "8-k,", "count": 3}, {"token": "4/6/2018", "count": 3}, {"token": "science", "count": 2}, {"token": "inc.,", "count": 2}]}}}} +{"doc": "tests/fixtures/legal100/fw_knight_office.pdf", "pages": 3, "oracles": {"pdfium": {"pawls": {"n_oracle": 657, "matched": 633, "forgiven": 24, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.96347, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 657, "matched": 620, "forgiven": 32, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.943683, "recall_effective": 0.99239, "missing_samples": [{"token": "tx", "count": 2}, {"token": "worth,", "count": 2}, {"token": "fmance/purchasing_.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 625, "matched": 584, "forgiven": 39, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.9344, "recall_effective": 0.9968, "flagged_pages": [{"page": 1, "n_oracle": 168, "matched": 152, "forgiven": 15, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.904762, "recall_effective": 0.994048, "missing_samples": [{"token": "c10ap=", "count": 1}]}, {"page": 2, "n_oracle": 219, "matched": 208, "forgiven": 10, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.949772, "recall_effective": 0.995434, "missing_samples": [{"token": "followingordrdeer:r", "count": 1}]}]}, "content": {"n_oracle": 625, "matched": 573, "forgiven": 45, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.9168, "recall_effective": 0.9888, "missing_samples": [{"token": "tx", "count": 2}, {"token": "worth,", "count": 2}, {"token": "v/departments/fmance/purchasing_.", "count": 1}, {"token": "c10ap=", "count": 1}, {"token": "followingordrdeer:r", "count": 1}]}}}} +{"doc": "files/pdf/8k.pdf", "pages": 5, "oracles": {"pdfium": {"pawls": {"n_oracle": 798, "matched": 796, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997494, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 798, "matched": 791, "forgiven": 2, "missing_material": 1, "missing_trivial": 4, "recall_strict": 0.991228, "recall_effective": 0.993734, "missing_samples": [{"token": "description", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 797, "matched": 797, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 797, "matched": 792, "forgiven": 0, "missing_material": 1, "missing_trivial": 4, "recall_strict": 0.993726, "recall_effective": 0.993726, "missing_samples": [{"token": "description", "count": 1}]}}}} +{"doc": "tests/fixtures/legal100/fw_coast_biomedical.pdf", "pages": 3, "oracles": {"pdfium": {"pawls": {"n_oracle": 657, "matched": 649, "forgiven": 8, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.987823, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 657, "matched": 625, "forgiven": 24, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.951294, "recall_effective": 0.987823, "missing_samples": [{"token": "csc", "count": 2}, {"token": "r2", "count": 2}, {"token": "renewal", "count": 2}, {"token": "page", "count": 1}, {"token": "f4rt", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 624, "matched": 572, "forgiven": 52, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.916667, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 624, "matched": 556, "forgiven": 60, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.891026, "recall_effective": 0.987179, "missing_samples": [{"token": "r2", "count": 2}, {"token": "csc", "count": 2}, {"token": "of2", "count": 2}, {"token": "renewal", "count": 1}, {"token": "f4rt", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/spacex_s1__exhibit41sx1_331523.pdf", "pages": 3, "oracles": {"pdfium": {"pawls": {"n_oracle": 532, "matched": 532, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 532, "matched": 530, "forgiven": 0, "missing_material": 1, "missing_trivial": 1, "recall_strict": 0.996241, "recall_effective": 0.996241, "missing_samples": [{"token": "received,_______________________________________hereby", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 532, "matched": 532, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 532, "matched": 530, "forgiven": 0, "missing_material": 1, "missing_trivial": 1, "recall_strict": 0.996241, "recall_effective": 0.996241, "missing_samples": [{"token": "received,_______________________________________hereby", "count": 1}]}}}} +{"doc": "tests/fixtures/legal100/fw_southern_computer.pdf", "pages": 3, "oracles": {"pdfium": {"pawls": {"n_oracle": 681, "matched": 662, "forgiven": 16, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.9721, "recall_effective": 0.995595, "flagged_pages": [{"page": 1, "n_oracle": 180, "matched": 169, "forgiven": 9, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.938889, "recall_effective": 0.988889, "missing_samples": [{"token": "4.p44bun", "count": 1}, {"token": "i\u00b0o", "count": 1}]}, {"page": 2, "n_oracle": 255, "matched": 247, "forgiven": 7, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.968627, "recall_effective": 0.996078, "missing_samples": [{"token": "order:", "count": 1}]}]}, "content": {"n_oracle": 681, "matched": 651, "forgiven": 24, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.955947, "recall_effective": 0.991189, "missing_samples": [{"token": "page", "count": 2}, {"token": "renewal", "count": 1}, {"token": "4.p44bun", "count": 1}, {"token": "i\u00b0o", "count": 1}, {"token": "order:", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 645, "matched": 608, "forgiven": 36, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.942636, "recall_effective": 0.99845, "flagged_pages": [{"page": 1, "n_oracle": 167, "matched": 153, "forgiven": 13, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.916168, "recall_effective": 0.994012, "missing_samples": [{"token": "22cczz", "count": 1}]}]}, "content": {"n_oracle": 645, "matched": 604, "forgiven": 39, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.936434, "recall_effective": 0.996899, "missing_samples": [{"token": "renewal", "count": 1}, {"token": "22cczz", "count": 1}]}}}} +{"doc": "tests/fixtures/contracts/fw_wert_bookbinding.pdf", "pages": 2, "oracles": {"pdfium": {"pawls": {"n_oracle": 372, "matched": 368, "forgiven": 4, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.989247, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 372, "matched": 343, "forgiven": 13, "missing_material": 15, "missing_trivial": 1, "recall_strict": 0.922043, "recall_effective": 0.956989, "missing_samples": [{"token": "r5", "count": 2}, {"token": "renewal", "count": 2}, {"token": "ft.", "count": 2}, {"token": "worth,", "count": 2}, {"token": "tx", "count": 2}, {"token": "csc-", "count": 2}, {"token": "page", "count": 2}, {"token": "55011-", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 347, "matched": 325, "forgiven": 22, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.936599, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 347, "matched": 304, "forgiven": 27, "missing_material": 15, "missing_trivial": 1, "recall_strict": 0.876081, "recall_effective": 0.95389, "missing_samples": [{"token": "r5", "count": 2}, {"token": "renewal", "count": 2}, {"token": "ft.", "count": 2}, {"token": "worth,", "count": 2}, {"token": "tx", "count": 2}, {"token": "csc-", "count": 2}, {"token": "page", "count": 2}, {"token": "55011-", "count": 1}]}}}} +{"doc": "tests/fixtures/legal100/fw_mentalix.pdf", "pages": 3, "oracles": {"pdfium": {"pawls": {"n_oracle": 602, "matched": 582, "forgiven": 20, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.966777, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 602, "matched": 566, "forgiven": 29, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.940199, "recall_effective": 0.988372, "missing_samples": [{"token": "worth,", "count": 2}, {"token": "ft.", "count": 2}, {"token": "page", "count": 2}, {"token": "renewal", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 547, "matched": 502, "forgiven": 44, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.917733, "recall_effective": 0.998172, "flagged_pages": [{"page": 1, "n_oracle": 176, "matched": 160, "forgiven": 15, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.909091, "recall_effective": 0.994318, "missing_samples": [{"token": "kevinande9rs.", "count": 1}]}]}, "content": {"n_oracle": 547, "matched": 486, "forgiven": 53, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.888483, "recall_effective": 0.985375, "missing_samples": [{"token": "worth,", "count": 2}, {"token": "ft.", "count": 2}, {"token": "page", "count": 2}, {"token": "renewal", "count": 1}, {"token": "kevinande9rs.", "count": 1}]}}}} +{"doc": "files/pdf/SoupmanInc_20150814_8-K_EX-10.1_9230148_EX-10.1_Franchise Agreement2.pdf", "pages": 2, "oracles": {"pdfium": {"pawls": {"n_oracle": 811, "matched": 811, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 811, "matched": 795, "forgiven": 2, "missing_material": 12, "missing_trivial": 2, "recall_strict": 0.980271, "recall_effective": 0.982737, "missing_samples": [{"token": "source:", "count": 2}, {"token": "soupman,", "count": 2}, {"token": "inc.,", "count": 2}, {"token": "8-k,", "count": 2}, {"token": "8/14/2015", "count": 2}, {"token": "__n/a____________________", "count": 1}, {"token": "20____", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 811, "matched": 811, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 811, "matched": 795, "forgiven": 2, "missing_material": 12, "missing_trivial": 2, "recall_strict": 0.980271, "recall_effective": 0.982737, "missing_samples": [{"token": "source:", "count": 2}, {"token": "soupman,", "count": 2}, {"token": "inc.,", "count": 2}, {"token": "8-k,", "count": 2}, {"token": "8/14/2015", "count": 2}, {"token": "__n/a____________________", "count": 1}, {"token": "20____", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/0f0aa1b108dccac413e3264bd5bb3eae5cac68d0aa1c5cae4188770f0f2c8a92.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 222, "matched": 200, "forgiven": 22, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.900901, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 222, "matched": 200, "forgiven": 21, "missing_material": 0, "missing_trivial": 1, "recall_strict": 0.900901, "recall_effective": 0.995495, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 246, "matched": 246, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 246, "matched": 245, "forgiven": 0, "missing_material": 0, "missing_trivial": 1, "recall_strict": 0.995935, "recall_effective": 0.995935, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/1634690602_page2.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 264, "matched": 264, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 264, "matched": 264, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 264, "matched": 264, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 264, "matched": 264, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/1653739079_page34.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 715, "matched": 707, "forgiven": 4, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.988811, "recall_effective": 0.994406, "flagged_pages": [{"page": 0, "n_oracle": 715, "matched": 707, "forgiven": 4, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.988811, "recall_effective": 0.994406, "missing_samples": [{"token": "e:\\fr\\fm\\27myr2.sgm", "count": 1}, {"token": "jspears", "count": 1}, {"token": "dsk121tn23prod", "count": 1}, {"token": "rules2", "count": 1}]}]}, "content": {"n_oracle": 715, "matched": 652, "forgiven": 4, "missing_material": 4, "missing_trivial": 55, "recall_strict": 0.911888, "recall_effective": 0.917483, "missing_samples": [{"token": "e:\\fr\\fm\\27myr2.sgm", "count": 1}, {"token": "jspears", "count": 1}, {"token": "dsk121tn23prod", "count": 1}, {"token": "rules2", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 707, "matched": 697, "forgiven": 5, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.985856, "recall_effective": 0.992928, "flagged_pages": [{"page": 0, "n_oracle": 707, "matched": 697, "forgiven": 5, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.985856, "recall_effective": 0.992928, "missing_samples": [{"token": "e:\\fr\\fm\\27myr2.sgm", "count": 1}, {"token": "2selur", "count": 1}, {"token": "htiw", "count": 1}, {"token": "dorp32nt121ksd", "count": 1}, {"token": "sraepsj", "count": 1}]}]}, "content": {"n_oracle": 707, "matched": 644, "forgiven": 3, "missing_material": 7, "missing_trivial": 53, "recall_strict": 0.910891, "recall_effective": 0.915134, "missing_samples": [{"token": "from/to............................................", "count": 2}, {"token": "e:\\fr\\fm\\27myr2.sgm", "count": 1}, {"token": "2selur", "count": 1}, {"token": "htiw", "count": 1}, {"token": "dorp32nt121ksd", "count": 1}, {"token": "sraepsj", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/1653739079_page36.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 959, "matched": 949, "forgiven": 6, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.989572, "recall_effective": 0.995829, "flagged_pages": [{"page": 0, "n_oracle": 959, "matched": 949, "forgiven": 6, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.989572, "recall_effective": 0.995829, "missing_samples": [{"token": "e:\\fr\\fm\\27myr2.sgm", "count": 1}, {"token": "jspears", "count": 1}, {"token": "dsk121tn23prod", "count": 1}, {"token": "rules2", "count": 1}]}]}, "content": {"n_oracle": 959, "matched": 938, "forgiven": 6, "missing_material": 4, "missing_trivial": 11, "recall_strict": 0.978102, "recall_effective": 0.984359, "missing_samples": [{"token": "e:\\fr\\fm\\27myr2.sgm", "count": 1}, {"token": "jspears", "count": 1}, {"token": "dsk121tn23prod", "count": 1}, {"token": "rules2", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 952, "matched": 942, "forgiven": 5, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.989496, "recall_effective": 0.994748, "flagged_pages": [{"page": 0, "n_oracle": 952, "matched": 942, "forgiven": 5, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.989496, "recall_effective": 0.994748, "missing_samples": [{"token": "e:\\fr\\fm\\27myr2.sgm", "count": 1}, {"token": "2selur", "count": 1}, {"token": "htiw", "count": 1}, {"token": "dorp32nt121ksd", "count": 1}, {"token": "sraepsj", "count": 1}]}]}, "content": {"n_oracle": 952, "matched": 931, "forgiven": 5, "missing_material": 5, "missing_trivial": 11, "recall_strict": 0.977941, "recall_effective": 0.983193, "missing_samples": [{"token": "e:\\fr\\fm\\27myr2.sgm", "count": 1}, {"token": "2selur", "count": 1}, {"token": "htiw", "count": 1}, {"token": "dorp32nt121ksd", "count": 1}, {"token": "sraepsj", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/1eee511b689c66082949cfbf0eb877a04842ec3b4a0427525ef21f4df6081f10.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 116, "matched": 116, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 116, "matched": 116, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 111, "matched": 105, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.945946, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 111, "matched": 105, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.945946, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/0000027_page1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 988, "matched": 986, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.997976, "recall_effective": 0.997976, "flagged_pages": [{"page": 0, "n_oracle": 988, "matched": 986, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.997976, "recall_effective": 0.997976, "missing_samples": [{"token": "5.0075.00", "count": 1}, {"token": "2.5052.50", "count": 1}]}]}, "content": {"n_oracle": 988, "matched": 983, "forgiven": 0, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.994939, "recall_effective": 0.994939, "missing_samples": [{"token": "134", "count": 1}, {"token": "135", "count": 1}, {"token": "s.p.a.", "count": 1}, {"token": "5.0075.00", "count": 1}, {"token": "2.5052.50", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 2203, "matched": 710, "forgiven": 1493, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.322288, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 2203, "matched": 709, "forgiven": 1492, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.321834, "recall_effective": 0.999092, "missing_samples": [{"token": "134", "count": 1}, {"token": "135", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/1f981b8374d8287941b74f82608bad578f0f65c2f258f5dc3b74c559e24f23bd.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 127, "matched": 122, "forgiven": 5, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.96063, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 127, "matched": 116, "forgiven": 7, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.913386, "recall_effective": 0.968504, "missing_samples": [{"token": "33", "count": 1}, {"token": "total", "count": 1}, {"token": "salary", "count": 1}, {"token": "employee", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 125, "matched": 120, "forgiven": 5, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.96, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 125, "matched": 114, "forgiven": 7, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.912, "recall_effective": 0.968, "missing_samples": [{"token": "total", "count": 1}, {"token": "salary", "count": 1}, {"token": "employee", "count": 1}, {"token": "33", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/1724234475631-marico-annual-report-fy24-pdf_p4.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 672, "matched": 672, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 672, "matched": 646, "forgiven": 0, "missing_material": 26, "missing_trivial": 0, "recall_strict": 0.96131, "recall_effective": 0.96131, "missing_samples": [{"token": "210..", "count": 1}, {"token": "222..", "count": 1}, {"token": "272..", "count": 1}, {"token": "334..", "count": 1}, {"token": "421..", "count": 1}, {"token": "07....", "count": 1}, {"token": "24....", "count": 1}, {"token": "26....", "count": 1}, {"token": "30....", "count": 1}, {"token": "34....", "count": 1}, {"token": "36....", "count": 1}, {"token": "44....", "count": 1}, {"token": "52....", "count": 1}, {"token": "54....", "count": 1}, {"token": "66....", "count": 1}, {"token": "70....", "count": 1}, {"token": "72....", "count": 1}, {"token": "74....", "count": 1}, {"token": "78....", "count": 1}, {"token": "82....", "count": 1}, {"token": "112..", "count": 1}, {"token": "128..", "count": 1}, {"token": "140..", "count": 1}, {"token": "168..", "count": 1}, {"token": "206..", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 646, "matched": 620, "forgiven": 26, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.959752, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 646, "matched": 620, "forgiven": 0, "missing_material": 26, "missing_trivial": 0, "recall_strict": 0.959752, "recall_effective": 0.959752, "missing_samples": [{"token": "07....about", "count": 1}, {"token": "72....leadership", "count": 1}, {"token": "210..management", "count": 1}, {"token": "24....global", "count": 1}, {"token": "74....stakeholder", "count": 1}, {"token": "222..business", "count": 1}, {"token": "26....chairman's", "count": 1}, {"token": "78....investors", "count": 1}, {"token": "30....md", "count": 1}, {"token": "82....consumers", "count": 1}, {"token": "272..board's", "count": 1}, {"token": "34....key", "count": 1}, {"token": "112..people", "count": 1}, {"token": "36....product", "count": 1}, {"token": "128..value-chain", "count": 1}, {"token": "44....materiality", "count": 1}, {"token": "140..communities", "count": 1}, {"token": "52....business", "count": 1}, {"token": "168..environment", "count": 1}, {"token": "334..consolidated", "count": 1}, {"token": "54....risk", "count": 1}, {"token": "206..corporate", "count": 1}, {"token": "421..standalone", "count": 1}, {"token": "66....strategy", "count": 1}, {"token": "208..awards", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/20240402_072920_TS0U_5JAN8NIL1XY3DOFS.1_p11.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 241, "matched": 241, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 241, "matched": 239, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.991701, "recall_effective": 0.995851, "missing_samples": [{"token": "singapore", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 241, "matched": 241, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 241, "matched": 239, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.991701, "recall_effective": 0.995851, "missing_samples": [{"token": "singapore", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/20250320-annual-report-2024-incl-sustainability-report-and-corporate-governance-report-copy-of-the-official-ESEF-format_p23.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 749, "matched": 723, "forgiven": 15, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.965287, "recall_effective": 0.985314, "flagged_pages": [{"page": 0, "n_oracle": 749, "matched": 723, "forgiven": 15, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.965287, "recall_effective": 0.985314, "missing_samples": [{"token": "low-carbon", "count": 3}, {"token": "air-treatment", "count": 2}, {"token": "energy-efficient", "count": 2}, {"token": "oil-free", "count": 1}, {"token": "high-speed", "count": 1}, {"token": "life-cycle", "count": 1}, {"token": "data-driven", "count": 1}]}]}, "content": {"n_oracle": 749, "matched": 723, "forgiven": 15, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.965287, "recall_effective": 0.985314, "missing_samples": [{"token": "low-carbon", "count": 3}, {"token": "air-treatment", "count": 2}, {"token": "energy-efficient", "count": 2}, {"token": "oil-free", "count": 1}, {"token": "high-speed", "count": 1}, {"token": "life-cycle", "count": 1}, {"token": "data-driven", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 751, "matched": 723, "forgiven": 17, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.962716, "recall_effective": 0.985353, "flagged_pages": [{"page": 0, "n_oracle": 751, "matched": 723, "forgiven": 17, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.962716, "recall_effective": 0.985353, "missing_samples": [{"token": "lowcarbon", "count": 3}, {"token": "airtreatment", "count": 2}, {"token": "energyefficient", "count": 2}, {"token": "lifec", "count": 1}, {"token": "oilfree", "count": 1}, {"token": "datadriven", "count": 1}, {"token": "highspeed", "count": 1}]}]}, "content": {"n_oracle": 751, "matched": 723, "forgiven": 17, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.962716, "recall_effective": 0.985353, "missing_samples": [{"token": "lowcarbon", "count": 3}, {"token": "airtreatment", "count": 2}, {"token": "energyefficient", "count": 2}, {"token": "lifec", "count": 1}, {"token": "oilfree", "count": 1}, {"token": "datadriven", "count": 1}, {"token": "highspeed", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/20250320-annual-report-2024-incl-sustainability-report-and-corporate-governance-report-copy-of-the-official-ESEF-format_p22.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 819, "matched": 802, "forgiven": 13, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.979243, "recall_effective": 0.995116, "flagged_pages": [{"page": 0, "n_oracle": 819, "matched": 802, "forgiven": 13, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.979243, "recall_effective": 0.995116, "missing_samples": [{"token": "medium-sized", "count": 1}, {"token": "energy-efficient", "count": 1}, {"token": "application-driven", "count": 1}, {"token": "druckluft-technik-nord", "count": 1}]}]}, "content": {"n_oracle": 819, "matched": 801, "forgiven": 14, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.978022, "recall_effective": 0.995116, "missing_samples": [{"token": "medium-sized", "count": 1}, {"token": "energy-efficient", "count": 1}, {"token": "application-driven", "count": 1}, {"token": "druckluft-technik-nord", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 819, "matched": 802, "forgiven": 13, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.979243, "recall_effective": 0.995116, "flagged_pages": [{"page": 0, "n_oracle": 819, "matched": 802, "forgiven": 13, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.979243, "recall_effective": 0.995116, "missing_samples": [{"token": "drucklufttechniknord", "count": 1}, {"token": "energyefficient", "count": 1}, {"token": "mediumsized", "count": 1}, {"token": "applicationdriven", "count": 1}]}]}, "content": {"n_oracle": 819, "matched": 801, "forgiven": 14, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.978022, "recall_effective": 0.995116, "missing_samples": [{"token": "drucklufttechniknord", "count": 1}, {"token": "energyefficient", "count": 1}, {"token": "mediumsized", "count": 1}, {"token": "applicationdriven", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/2239ffa88e1bd767d12dce472c96d1b4299ae27c3b53a3ef2948e3553705583b.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 658, "matched": 658, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 658, "matched": 658, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 389, "matched": 327, "forgiven": 62, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.840617, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 389, "matched": 327, "forgiven": 62, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.840617, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/2503.12267v1_p3.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 598, "matched": 594, "forgiven": 4, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.993311, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 598, "matched": 594, "forgiven": 4, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.993311, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 506, "matched": 486, "forgiven": 20, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.960474, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 506, "matched": 486, "forgiven": 20, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.960474, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/2503.12267v1_p5.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 699, "matched": 690, "forgiven": 9, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.987124, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 699, "matched": 689, "forgiven": 9, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.985694, "recall_effective": 0.998569, "missing_samples": [{"token": "[20].", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 601, "matched": 572, "forgiven": 29, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.951747, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 601, "matched": 571, "forgiven": 29, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.950083, "recall_effective": 0.998336, "missing_samples": [{"token": "[20].", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/2a438f8678ee8466f981c94b296224a9de2d6840df9e400a6c4e7f7f1a85923b.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 264, "matched": 252, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.954545, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 264, "matched": 249, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.943182, "recall_effective": 0.992424, "missing_samples": [{"token": "*job", "count": 1}, {"token": "*content,", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 166, "matched": 145, "forgiven": 21, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.873494, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 166, "matched": 142, "forgiven": 22, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.855422, "recall_effective": 0.987952, "missing_samples": [{"token": "*job", "count": 1}, {"token": "*content,", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/2271fdca260e8f64f1e6a2101a4e2795a24ccbba3366933e665e8319253614bf.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 361, "matched": 220, "forgiven": 119, "missing_material": 22, "missing_trivial": 0, "recall_strict": 0.609418, "recall_effective": 0.939058, "flagged_pages": [{"page": 0, "n_oracle": 361, "matched": 220, "forgiven": 119, "missing_material": 22, "missing_trivial": 0, "recall_strict": 0.609418, "recall_effective": 0.939058, "missing_samples": [{"token": "1.9", "count": 3}, {"token": "215.4", "count": 1}, {"token": "237.9", "count": 1}, {"token": "239.6", "count": 1}, {"token": "125.5", "count": 1}, {"token": "221.0", "count": 1}, {"token": "1.53", "count": 1}, {"token": "1.33", "count": 1}, {"token": "1.44", "count": 1}, {"token": "0.73", "count": 1}, {"token": "1.26", "count": 1}, {"token": "388.2", "count": 1}, {"token": "336.4", "count": 1}, {"token": "325.7", "count": 1}, {"token": "218.6", "count": 1}, {"token": "266.4", "count": 1}, {"token": "2.1", "count": 1}, {"token": "3.9", "count": 1}, {"token": "4.2", "count": 1}, {"token": "1.7", "count": 1}]}]}, "content": {"n_oracle": 361, "matched": 219, "forgiven": 102, "missing_material": 40, "missing_trivial": 0, "recall_strict": 0.606648, "recall_effective": 0.889197, "missing_samples": [{"token": "1.9", "count": 3}, {"token": "of", "count": 2}, {"token": "meeting", "count": 2}, {"token": "215.4", "count": 1}, {"token": "237.9", "count": 1}, {"token": "239.6", "count": 1}, {"token": "125.5", "count": 1}, {"token": "221.0", "count": 1}, {"token": "1.53", "count": 1}, {"token": "1.33", "count": 1}, {"token": "1.44", "count": 1}, {"token": "0.73", "count": 1}, {"token": "1.26", "count": 1}, {"token": "388.2", "count": 1}, {"token": "336.4", "count": 1}, {"token": "325.7", "count": 1}, {"token": "218.6", "count": 1}, {"token": "266.4", "count": 1}, {"token": "3.9", "count": 1}, {"token": "4.2", "count": 1}, {"token": "1.7", "count": 1}, {"token": "services,", "count": 1}, {"token": "inc.", "count": 1}, {"token": "notice", "count": 1}, {"token": "annual", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 403, "matched": 222, "forgiven": 155, "missing_material": 26, "missing_trivial": 0, "recall_strict": 0.550868, "recall_effective": 0.935484, "flagged_pages": [{"page": 0, "n_oracle": 403, "matched": 222, "forgiven": 155, "missing_material": 26, "missing_trivial": 0, "recall_strict": 0.550868, "recall_effective": 0.935484, "missing_samples": [{"token": "9.1", "count": 2}, {"token": "9.3", "count": 1}, {"token": "3.2", "count": 1}, {"token": "4.2", "count": 1}, {"token": "5.2", "count": 1}, {"token": "7.2", "count": 1}, {"token": "4.662", "count": 1}, {"token": "7.1", "count": 1}, {"token": "6.812", "count": 1}, {"token": "7.523", "count": 1}, {"token": "4.633", "count": 1}, {"token": "2.883", "count": 1}, {"token": "0.122", "count": 1}, {"token": "62.1", "count": 1}, {"token": "37.0", "count": 1}, {"token": "6.932", "count": 1}, {"token": "44.1", "count": 1}, {"token": "4.512", "count": 1}, {"token": "33.1", "count": 1}, {"token": "9.732", "count": 1}, {"token": "35.1", "count": 1}, {"token": "services,", "count": 1}, {"token": "inc.", "count": 1}, {"token": "annual", "count": 1}, {"token": "report", "count": 1}]}]}, "content": {"n_oracle": 403, "matched": 221, "forgiven": 117, "missing_material": 65, "missing_trivial": 0, "recall_strict": 0.548387, "recall_effective": 0.83871, "missing_samples": [{"token": "of", "count": 3}, {"token": "services,", "count": 3}, {"token": "inc.", "count": 3}, {"token": "annual", "count": 3}, {"token": "and", "count": 2}, {"token": "9.1", "count": 2}, {"token": "republic", "count": 2}, {"token": "meeting", "count": 2}, {"token": "ritm,", "count": 2}, {"token": "llc", "count": 2}, {"token": "9.3", "count": 1}, {"token": "3.2", "count": 1}, {"token": "4.2", "count": 1}, {"token": "5.2", "count": 1}, {"token": "7.2", "count": 1}, {"token": "4.662", "count": 1}, {"token": "7.1", "count": 1}, {"token": "6.812", "count": 1}, {"token": "7.523", "count": 1}, {"token": "4.633", "count": 1}, {"token": "2.883", "count": 1}, {"token": "0.122", "count": 1}, {"token": "62.1", "count": 1}, {"token": "37.0", "count": 1}, {"token": "6.932", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/47c5f627ce9b5085b64cecf3090ed4d1fea9ad562ea97ff342487760ac772051.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 432, "matched": 414, "forgiven": 18, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.958333, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 432, "matched": 403, "forgiven": 5, "missing_material": 2, "missing_trivial": 22, "recall_strict": 0.93287, "recall_effective": 0.944444, "missing_samples": [{"token": "losses..................", "count": 2}]}}, "plumber": {"pawls": {"n_oracle": 432, "matched": 416, "forgiven": 16, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.962963, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 432, "matched": 405, "forgiven": 3, "missing_material": 2, "missing_trivial": 22, "recall_strict": 0.9375, "recall_effective": 0.944444, "missing_samples": [{"token": "losses..................", "count": 2}]}}}} +{"doc": "tests/fixtures/hetero100/pages/3aaafdfd27141d74c4f211f24c1b97af7434459f20052f11d199f290f5e74767.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 460, "matched": 460, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 460, "matched": 458, "forgiven": 0, "missing_material": 1, "missing_trivial": 1, "recall_strict": 0.995652, "recall_effective": 0.995652, "missing_samples": [{"token": "96", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 460, "matched": 460, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 460, "matched": 458, "forgiven": 0, "missing_material": 1, "missing_trivial": 1, "recall_strict": 0.995652, "recall_effective": 0.995652, "missing_samples": [{"token": "96", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/6482543f4e6b569aaeaef5547650d67ee38ebc3a8fb4359f3c200833a0cbbf21.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 345, "matched": 339, "forgiven": 2, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.982609, "recall_effective": 0.988406, "flagged_pages": [{"page": 0, "n_oracle": 345, "matched": 339, "forgiven": 2, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.982609, "recall_effective": 0.988406, "missing_samples": [{"token": "4.90+0.22\u22120.27", "count": 1}, {"token": "0.88+0.14\u22120.13", "count": 1}, {"token": "4.20+0.32\u22120.03", "count": 1}, {"token": "0.89+0.15\u22120.10", "count": 1}]}]}, "content": {"n_oracle": 345, "matched": 338, "forgiven": 3, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.97971, "recall_effective": 0.988406, "missing_samples": [{"token": "4.90+0.22\u22120.27", "count": 1}, {"token": "0.88+0.14\u22120.13", "count": 1}, {"token": "4.20+0.32\u22120.03", "count": 1}, {"token": "0.89+0.15\u22120.10", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 299, "matched": 252, "forgiven": 47, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.842809, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 299, "matched": 251, "forgiven": 48, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.839465, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/6f6fed09430ede7da068b3ae12a6301de48123c812af4433d91857af88f86f6a.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 313, "matched": 253, "forgiven": 43, "missing_material": 17, "missing_trivial": 0, "recall_strict": 0.808307, "recall_effective": 0.945687, "flagged_pages": [{"page": 0, "n_oracle": 313, "matched": 253, "forgiven": 43, "missing_material": 17, "missing_trivial": 0, "recall_strict": 0.808307, "recall_effective": 0.945687, "missing_samples": [{"token": "(tf", "count": 4}, {"token": "t0)", "count": 2}, {"token": "\u03b35q", "count": 2}, {"token": "3x", "count": 1}, {"token": "~x\u2208v", "count": 1}, {"token": "h0|a", "count": 1}, {"token": "(t0)", "count": 1}, {"token": "|0i,", "count": 1}, {"token": "\u2212bq", "count": 1}, {"token": "moi", "count": 1}, {"token": "t0)e", "count": 1}, {"token": "\u2212t0)/2", "count": 1}]}]}, "content": {"n_oracle": 313, "matched": 251, "forgiven": 44, "missing_material": 18, "missing_trivial": 0, "recall_strict": 0.801917, "recall_effective": 0.942492, "missing_samples": [{"token": "(tf", "count": 4}, {"token": "t0)", "count": 2}, {"token": "\u03b35q", "count": 2}, {"token": "3x", "count": 1}, {"token": "~x\u2208v", "count": 1}, {"token": "h0|a", "count": 1}, {"token": "(t0)", "count": 1}, {"token": "|0i,", "count": 1}, {"token": "\u2212bq", "count": 1}, {"token": "moi", "count": 1}, {"token": "t0)e", "count": 1}, {"token": "\u2212t0)/2", "count": 1}, {"token": "22", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 307, "matched": 246, "forgiven": 59, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.801303, "recall_effective": 0.993485, "flagged_pages": [{"page": 0, "n_oracle": 307, "matched": 246, "forgiven": 59, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.801303, "recall_effective": 0.993485, "missing_samples": [{"token": "~x\u2208v", "count": 1}, {"token": "(tf\u2212t0)/2", "count": 1}]}]}, "content": {"n_oracle": 307, "matched": 244, "forgiven": 60, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.794788, "recall_effective": 0.990228, "missing_samples": [{"token": "~x\u2208v", "count": 1}, {"token": "(tf\u2212t0)/2", "count": 1}, {"token": "22", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/637951191e7b35ae07dbd4c76a2e6690c370_pg2_pg1_page1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 561, "matched": 561, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 561, "matched": 561, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 561, "matched": 561, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 561, "matched": 561, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/8bde5f82f0be403249a5d85119be711481b5cd9134b2ca74debe975d8b650a21.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 719, "matched": 713, "forgiven": 4, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.991655, "recall_effective": 0.997218, "flagged_pages": [{"page": 0, "n_oracle": 719, "matched": 713, "forgiven": 4, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.991655, "recall_effective": 0.997218, "missing_samples": [{"token": "management's", "count": 1}, {"token": "discussion", "count": 1}]}]}, "content": {"n_oracle": 719, "matched": 713, "forgiven": 4, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.991655, "recall_effective": 0.997218, "missing_samples": [{"token": "management's", "count": 1}, {"token": "discussion", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 474, "matched": 414, "forgiven": 58, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.873418, "recall_effective": 0.995781, "flagged_pages": [{"page": 0, "n_oracle": 474, "matched": 414, "forgiven": 58, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.873418, "recall_effective": 0.995781, "missing_samples": [{"token": "management's", "count": 1}, {"token": "discussion", "count": 1}]}]}, "content": {"n_oracle": 474, "matched": 414, "forgiven": 58, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.873418, "recall_effective": 0.995781, "missing_samples": [{"token": "management's", "count": 1}, {"token": "discussion", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/96a8b3c14d41f3ff00586d39256d24c6f5adccb2d09ef235d14a23ad73849dab.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 511, "matched": 500, "forgiven": 11, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.978474, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 511, "matched": 499, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.976517, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 510, "matched": 501, "forgiven": 9, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.982353, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 510, "matched": 500, "forgiven": 10, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.980392, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/AXP_2023_2024_ESG_Report_p46.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 406, "matched": 389, "forgiven": 11, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.958128, "recall_effective": 0.985222, "flagged_pages": [{"page": 0, "n_oracle": 406, "matched": 389, "forgiven": 11, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.958128, "recall_effective": 0.985222, "missing_samples": [{"token": "globally", "count": 2}, {"token": "black/african", "count": 1}, {"token": "white", "count": 1}, {"token": "women", "count": 1}, {"token": "other2,", "count": 1}]}]}, "content": {"n_oracle": 406, "matched": 388, "forgiven": 11, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.955665, "recall_effective": 0.982759, "missing_samples": [{"token": "globally", "count": 2}, {"token": "black/african", "count": 1}, {"token": "white", "count": 1}, {"token": "women", "count": 1}, {"token": "46", "count": 1}, {"token": "other2,", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 402, "matched": 385, "forgiven": 6, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.957711, "recall_effective": 0.972637, "flagged_pages": [{"page": 0, "n_oracle": 402, "matched": 385, "forgiven": 6, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.957711, "recall_effective": 0.972637, "missing_samples": [{"token": "yllabolg", "count": 2}, {"token": "nemow", "count": 1}, {"token": "nem", "count": 1}, {"token": "2naisa", "count": 1}, {"token": "nacirfa/kcalb", "count": 1}, {"token": "2nacirema", "count": 1}, {"token": "cinapsih", "count": 1}, {"token": "2xnital", "count": 1}, {"token": ",2rehto", "count": 1}, {"token": "2etihw", "count": 1}]}]}, "content": {"n_oracle": 402, "matched": 384, "forgiven": 6, "missing_material": 12, "missing_trivial": 0, "recall_strict": 0.955224, "recall_effective": 0.970149, "missing_samples": [{"token": "yllabolg", "count": 2}, {"token": "46", "count": 1}, {"token": "nemow", "count": 1}, {"token": "nem", "count": 1}, {"token": "2naisa", "count": 1}, {"token": "nacirfa/kcalb", "count": 1}, {"token": "2nacirema", "count": 1}, {"token": "cinapsih", "count": 1}, {"token": "2xnital", "count": 1}, {"token": ",2rehto", "count": 1}, {"token": "2etihw", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/911cc28d2b0eb55a14845e92314c30f40c5b96cde70ee6bab0abcd86061b392b.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 610, "matched": 602, "forgiven": 3, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.986885, "recall_effective": 0.991803, "flagged_pages": [{"page": 0, "n_oracle": 610, "matched": 602, "forgiven": 3, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.986885, "recall_effective": 0.991803, "missing_samples": [{"token": "net", "count": 2}, {"token": "income", "count": 2}, {"token": "operating", "count": 1}]}]}, "content": {"n_oracle": 610, "matched": 602, "forgiven": 3, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.986885, "recall_effective": 0.991803, "missing_samples": [{"token": "net", "count": 2}, {"token": "income", "count": 2}, {"token": "operating", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 611, "matched": 604, "forgiven": 1, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.988543, "recall_effective": 0.99018, "flagged_pages": [{"page": 0, "n_oracle": 611, "matched": 604, "forgiven": 1, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.988543, "recall_effective": 0.99018, "missing_samples": [{"token": "ten", "count": 2}, {"token": "emocni", "count": 2}, {"token": "eunever", "count": 1}, {"token": "gnitarepo", "count": 1}]}]}, "content": {"n_oracle": 611, "matched": 604, "forgiven": 1, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.988543, "recall_effective": 0.99018, "missing_samples": [{"token": "ten", "count": 2}, {"token": "emocni", "count": 2}, {"token": "eunever", "count": 1}, {"token": "gnitarepo", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/Annual-Report-FY-2023-24_p59.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 196, "matched": 196, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 196, "matched": 196, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 196, "matched": 196, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 196, "matched": 196, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/Annual-Report-FY-2023-24_p107.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 257, "matched": 255, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.992218, "recall_effective": 0.992218, "flagged_pages": [{"page": 0, "n_oracle": 257, "matched": 255, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.992218, "recall_effective": 0.992218, "missing_samples": [{"token": "co2", "count": 2}]}]}, "content": {"n_oracle": 257, "matched": 251, "forgiven": 0, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.976654, "recall_effective": 0.976654, "missing_samples": [{"token": "co2", "count": 2}, {"token": "mahindra", "count": 1}, {"token": "community", "count": 1}, {"token": "lifespaces", "count": 1}, {"token": "107", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 265, "matched": 265, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 265, "matched": 261, "forgiven": 0, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.984906, "recall_effective": 0.984906, "missing_samples": [{"token": "mahindra", "count": 1}, {"token": "community", "count": 1}, {"token": "lifespaces", "count": 1}, {"token": "107", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/AXP_2023_2024_ESG_Report_p8.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 677, "matched": 675, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997046, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 677, "matched": 651, "forgiven": 8, "missing_material": 18, "missing_trivial": 0, "recall_strict": 0.961595, "recall_effective": 0.973412, "missing_samples": [{"token": "sustainability", "count": 1}, {"token": "8", "count": 1}, {"token": "maintain", "count": 1}, {"token": "with", "count": 1}, {"token": "including", "count": 1}, {"token": "low-carbon", "count": 1}, {"token": "commit", "count": 1}, {"token": "net-zero", "count": 1}, {"token": "emissions", "count": 1}, {"token": "alignment", "count": 1}, {"token": "across", "count": 1}, {"token": "pilot", "count": 1}, {"token": "innovations,", "count": 1}, {"token": "engage", "count": 1}, {"token": "colleagues", "count": 1}, {"token": "100%", "count": 1}, {"token": "pay", "count": 1}, {"token": "genders", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 677, "matched": 677, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 677, "matched": 653, "forgiven": 6, "missing_material": 18, "missing_trivial": 0, "recall_strict": 0.964549, "recall_effective": 0.973412, "missing_samples": [{"token": "sustainability", "count": 1}, {"token": "8", "count": 1}, {"token": "maintain", "count": 1}, {"token": "including", "count": 1}, {"token": "with", "count": 1}, {"token": "low-carbon", "count": 1}, {"token": "commit", "count": 1}, {"token": "net-zero", "count": 1}, {"token": "emissions", "count": 1}, {"token": "alignment", "count": 1}, {"token": "pilot", "count": 1}, {"token": "innovations,", "count": 1}, {"token": "across", "count": 1}, {"token": "engage", "count": 1}, {"token": "colleagues", "count": 1}, {"token": "100%", "count": 1}, {"token": "pay", "count": 1}, {"token": "genders", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/Apple_Environmental_Progress_Report_2025_p16.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 432, "matched": 432, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 432, "matched": 432, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 432, "matched": 432, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 432, "matched": 432, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/Annual_Report_FY2022_p16.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 723, "matched": 723, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 723, "matched": 718, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.993084, "recall_effective": 0.994467, "missing_samples": [{"token": "*gender", "count": 2}, {"token": "exchange(s)", "count": 1}, {"token": "where", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 726, "matched": 724, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997245, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 726, "matched": 719, "forgiven": 3, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.990358, "recall_effective": 0.99449, "missing_samples": [{"token": "*gender", "count": 2}, {"token": "exchange(s)", "count": 1}, {"token": "where", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/Apple_Environmental_Progress_Report_2024_p97.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 821, "matched": 820, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998782, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 821, "matched": 817, "forgiven": 3, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.995128, "recall_effective": 0.998782, "missing_samples": [{"token": "renewable", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 1001, "matched": 786, "forgiven": 215, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.785215, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1001, "matched": 784, "forgiven": 216, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.783217, "recall_effective": 0.999001, "missing_samples": [{"token": "renewable", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/CEJ_Lincoln_Max_Income_illustration_att1_p20.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 305, "matched": 301, "forgiven": 0, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.986885, "recall_effective": 0.986885, "flagged_pages": [{"page": 0, "n_oracle": 305, "matched": 301, "forgiven": 0, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.986885, "recall_effective": 0.986885, "missing_samples": [{"token": "lincoln", "count": 1}, {"token": "wealthaccumulate", "count": 1}, {"token": "iul", "count": 1}, {"token": "(2020)", "count": 1}]}]}, "content": {"n_oracle": 305, "matched": 299, "forgiven": 2, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.980328, "recall_effective": 0.986885, "missing_samples": [{"token": "lincoln", "count": 1}, {"token": "wealthaccumulate", "count": 1}, {"token": "iul", "count": 1}, {"token": "(2020)", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 303, "matched": 298, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.983498, "recall_effective": 0.986799, "flagged_pages": [{"page": 0, "n_oracle": 303, "matched": 298, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.983498, "recall_effective": 0.986799, "missing_samples": [{"token": ")0202(", "count": 1}, {"token": "lui", "count": 1}, {"token": "\u00aeetalumuccahtlaew", "count": 1}, {"token": "nlocnil", "count": 1}]}]}, "content": {"n_oracle": 303, "matched": 296, "forgiven": 3, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.976898, "recall_effective": 0.986799, "missing_samples": [{"token": ")0202(", "count": 1}, {"token": "lui", "count": 1}, {"token": "\u00aeetalumuccahtlaew", "count": 1}, {"token": "nlocnil", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/CEJ_Lincoln_Max_Income_illustration_att1_p28.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 445, "matched": 440, "forgiven": 2, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.988764, "recall_effective": 0.993258, "flagged_pages": [{"page": 0, "n_oracle": 445, "matched": 440, "forgiven": 2, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.988764, "recall_effective": 0.993258, "missing_samples": [{"token": "wealthaccumulate", "count": 1}, {"token": "iul", "count": 1}, {"token": "(2020)", "count": 1}]}]}, "content": {"n_oracle": 445, "matched": 438, "forgiven": 4, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.98427, "recall_effective": 0.993258, "missing_samples": [{"token": "wealthaccumulate", "count": 1}, {"token": "iul", "count": 1}, {"token": "(2020)", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 443, "matched": 438, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.988713, "recall_effective": 0.990971, "flagged_pages": [{"page": 0, "n_oracle": 443, "matched": 438, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.988713, "recall_effective": 0.990971, "missing_samples": [{"token": ")0202(", "count": 1}, {"token": "lui", "count": 1}, {"token": "\u00aeetalumuccahtlaew", "count": 1}, {"token": "nlocnil", "count": 1}]}]}, "content": {"n_oracle": 443, "matched": 436, "forgiven": 3, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.984199, "recall_effective": 0.990971, "missing_samples": [{"token": ")0202(", "count": 1}, {"token": "lui", "count": 1}, {"token": "\u00aeetalumuccahtlaew", "count": 1}, {"token": "nlocnil", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/CEJ_Lincoln_Max_Income_illustration_att1_p60.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 539, "matched": 535, "forgiven": 0, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.992579, "recall_effective": 0.992579, "flagged_pages": [{"page": 0, "n_oracle": 539, "matched": 535, "forgiven": 0, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.992579, "recall_effective": 0.992579, "missing_samples": [{"token": "lincoln", "count": 1}, {"token": "wealthaccumulate", "count": 1}, {"token": "iul", "count": 1}, {"token": "(2020)", "count": 1}]}]}, "content": {"n_oracle": 539, "matched": 534, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.990724, "recall_effective": 0.992579, "missing_samples": [{"token": "lincoln", "count": 1}, {"token": "wealthaccumulate", "count": 1}, {"token": "iul", "count": 1}, {"token": "(2020)", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 537, "matched": 532, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.990689, "recall_effective": 0.992551, "flagged_pages": [{"page": 0, "n_oracle": 537, "matched": 532, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.990689, "recall_effective": 0.992551, "missing_samples": [{"token": ")0202(", "count": 1}, {"token": "lui", "count": 1}, {"token": "\u00aeetalumuccahtlaew", "count": 1}, {"token": "nlocnil", "count": 1}]}]}, "content": {"n_oracle": 537, "matched": 531, "forgiven": 2, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.988827, "recall_effective": 0.992551, "missing_samples": [{"token": ")0202(", "count": 1}, {"token": "lui", "count": 1}, {"token": "\u00aeetalumuccahtlaew", "count": 1}, {"token": "nlocnil", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/Cost_Estimating_Guideline_1__page3.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 290, "matched": 289, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996552, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 290, "matched": 289, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.996552, "recall_effective": 0.996552, "missing_samples": [{"token": "10", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 290, "matched": 289, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996552, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 290, "matched": 289, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.996552, "recall_effective": 0.996552, "missing_samples": [{"token": "10", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/GSK_PLC_ORD_GBP0_p5.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 433, "matched": 433, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 433, "matched": 428, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.988453, "recall_effective": 0.990762, "missing_samples": [{"token": "...steered", "count": 1}, {"token": "priorities...", "count": 1}, {"token": "for...", "count": 1}, {"token": "...and", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 433, "matched": 433, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 433, "matched": 428, "forgiven": 1, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.988453, "recall_effective": 0.990762, "missing_samples": [{"token": "...steered", "count": 1}, {"token": "priorities...", "count": 1}, {"token": "for...", "count": 1}, {"token": "...and", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/GBFR22-value-creation-model_p1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 901, "matched": 735, "forgiven": 60, "missing_material": 105, "missing_trivial": 1, "recall_strict": 0.81576, "recall_effective": 0.882353, "flagged_pages": [{"page": 0, "n_oracle": 901, "matched": 735, "forgiven": 60, "missing_material": 105, "missing_trivial": 1, "recall_strict": 0.81576, "recall_effective": 0.882353, "missing_samples": [{"token": "capital:", "count": 12}, {"token": "and", "count": 10}, {"token": "community", "count": 4}, {"token": "these", "count": 4}, {"token": "strategy", "count": 4}, {"token": "human", "count": 3}, {"token": "inputs", "count": 3}, {"token": "resources", "count": 3}, {"token": "social", "count": 2}, {"token": "has", "count": 2}, {"token": "activities:", "count": 2}, {"token": "employees,", "count": 2}, {"token": "intellectual", "count": 2}, {"token": "business", "count": 2}, {"token": "utilizes", "count": 2}, {"token": "execute", "count": 2}, {"token": "execution,", "count": 2}, {"token": "transformed", "count": 2}, {"token": "yield", "count": 2}, {"token": "results", "count": 2}, {"token": "values", "count": 2}, {"token": "stakeholders.", "count": 2}, {"token": "value:", "count": 2}, {"token": "technology:", "count": 2}, {"token": "experience:", "count": 2}]}]}, "content": {"n_oracle": 901, "matched": 735, "forgiven": 55, "missing_material": 110, "missing_trivial": 1, "recall_strict": 0.81576, "recall_effective": 0.876804, "missing_samples": [{"token": "capital:", "count": 12}, {"token": "and", "count": 10}, {"token": "community", "count": 4}, {"token": "these", "count": 4}, {"token": "strategy", "count": 4}, {"token": "human", "count": 3}, {"token": "inputs", "count": 3}, {"token": "resources", "count": 3}, {"token": "social", "count": 2}, {"token": "has", "count": 2}, {"token": "activities:", "count": 2}, {"token": "employees,", "count": 2}, {"token": "intellectual", "count": 2}, {"token": "business", "count": 2}, {"token": "utilizes", "count": 2}, {"token": "execute", "count": 2}, {"token": "execution,", "count": 2}, {"token": "transformed", "count": 2}, {"token": "yield", "count": 2}, {"token": "results", "count": 2}, {"token": "values", "count": 2}, {"token": "stakeholders.", "count": 2}, {"token": "value:", "count": 2}, {"token": "technology:", "count": 2}, {"token": "experience:", "count": 2}]}}, "plumber": {"pawls": {"n_oracle": 818, "matched": 818, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 818, "matched": 812, "forgiven": 0, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.992665, "recall_effective": 0.992665, "missing_samples": [{"token": "ssttaakkeehhoollddeerrss..", "count": 1}, {"token": "11..", "count": 1}, {"token": "22..", "count": 1}, {"token": "33..", "count": 1}, {"token": "44..", "count": 1}, {"token": "55..", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/Human-Development-Indices-and-Indicators_p35.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1011, "matched": 1011, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1011, "matched": 970, "forgiven": 0, "missing_material": 0, "missing_trivial": 41, "recall_strict": 0.959446, "recall_effective": 0.959446, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 1239, "matched": 953, "forgiven": 286, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.769169, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1239, "matched": 912, "forgiven": 286, "missing_material": 0, "missing_trivial": 41, "recall_strict": 0.736077, "recall_effective": 0.966909, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/Lancashire-Annual-Report-and-Accounts-2024_p32.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 586, "matched": 585, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.998294, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 586, "matched": 584, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.996587, "recall_effective": 0.998294, "missing_samples": [{"token": "30", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 586, "matched": 586, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 586, "matched": 585, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.998294, "recall_effective": 0.998294, "missing_samples": [{"token": "30", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/Low-Back-Guideline_1__page2.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 372, "matched": 372, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 372, "matched": 364, "forgiven": 0, "missing_material": 7, "missing_trivial": 1, "recall_strict": 0.978495, "recall_effective": 0.978495, "missing_samples": [{"token": "page", "count": 1}, {"token": "13", "count": 1}, {"token": "medical", "count": 1}, {"token": "history", "count": 1}, {"token": "physical", "count": 1}, {"token": "examination/diagnostic", "count": 1}, {"token": "testing", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 375, "matched": 363, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.968, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 375, "matched": 356, "forgiven": 12, "missing_material": 6, "missing_trivial": 1, "recall_strict": 0.949333, "recall_effective": 0.981333, "missing_samples": [{"token": "medical", "count": 1}, {"token": "history", "count": 1}, {"token": "examination/diagnostic", "count": 1}, {"token": "testing", "count": 1}, {"token": "page", "count": 1}, {"token": "13", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/Micron-2025-Sustainability-Report_p80.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 257, "matched": 256, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996109, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 257, "matched": 256, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996109, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 257, "matched": 256, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996109, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 257, "matched": 256, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996109, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/Human-Development-Indices-and-Indicators_p43.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1381, "matched": 1377, "forgiven": 4, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997104, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1381, "matched": 1286, "forgiven": 4, "missing_material": 0, "missing_trivial": 91, "recall_strict": 0.931209, "recall_effective": 0.934106, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 2312, "matched": 1197, "forgiven": 1115, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.517734, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 2312, "matched": 1106, "forgiven": 1115, "missing_material": 0, "missing_trivial": 91, "recall_strict": 0.478374, "recall_effective": 0.96064, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/OUEREIT_AR2023_p24.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 428, "matched": 428, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 428, "matched": 427, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997664, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 428, "matched": 428, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 428, "matched": 427, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997664, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/PEC-Tomorrow-comes-today-Trends-shaping-the-future-of-the-Creative-Industries-August-2023_p9.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 324, "matched": 324, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 324, "matched": 311, "forgiven": 5, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.959877, "recall_effective": 0.975309, "missing_samples": [{"token": "future", "count": 1}, {"token": "of", "count": 1}, {"token": "creative", "count": 1}, {"token": "intelligence,", "count": 1}, {"token": "climate", "count": 1}, {"token": "its", "count": 1}, {"token": "socioeconomic", "count": 1}, {"token": "inequality", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 324, "matched": 324, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 324, "matched": 311, "forgiven": 5, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.959877, "recall_effective": 0.975309, "missing_samples": [{"token": "future", "count": 1}, {"token": "of", "count": 1}, {"token": "creative", "count": 1}, {"token": "inequality", "count": 1}, {"token": "climate", "count": 1}, {"token": "its", "count": 1}, {"token": "socioeconomic", "count": 1}, {"token": "intelligence,", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/OUEREIT_AR2023_p25.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 497, "matched": 497, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 497, "matched": 488, "forgiven": 2, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.981891, "recall_effective": 0.985915, "missing_samples": [{"token": "ltd.", "count": 1}, {"token": "xihe", "count": 1}, {"token": "capital", "count": 1}, {"token": "(pte.)", "count": 1}, {"token": "scholarship", "count": 1}, {"token": "force", "count": 1}, {"token": "(1979)", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 497, "matched": 497, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 497, "matched": 488, "forgiven": 2, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.981891, "recall_effective": 0.985915, "missing_samples": [{"token": "ltd.", "count": 1}, {"token": "xihe", "count": 1}, {"token": "capital", "count": 1}, {"token": "(pte.)", "count": 1}, {"token": "scholarship", "count": 1}, {"token": "force", "count": 1}, {"token": "(1979)", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/OTC_TATLY_2023_p9.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 446, "matched": 446, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 446, "matched": 446, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 446, "matched": 446, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 446, "matched": 446, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/PhilipsFullAnnualReport2023-English_p27.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 604, "matched": 599, "forgiven": 3, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.991722, "recall_effective": 0.996689, "flagged_pages": [{"page": 0, "n_oracle": 604, "matched": 599, "forgiven": 3, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.991722, "recall_effective": 0.996689, "missing_samples": [{"token": "1)(1.2)%", "count": 1}, {"token": "basis*),", "count": 1}]}]}, "content": {"n_oracle": 604, "matched": 598, "forgiven": 4, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.990066, "recall_effective": 0.996689, "missing_samples": [{"token": "1)(1.2)%", "count": 1}, {"token": "basis*),", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 592, "matched": 580, "forgiven": 8, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.97973, "recall_effective": 0.993243, "flagged_pages": [{"page": 0, "n_oracle": 592, "matched": 580, "forgiven": 8, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.97973, "recall_effective": 0.993243, "missing_samples": [{"token": "basis*),", "count": 1}, {"token": "growth1)", "count": 1}, {"token": "ebita*)amounted", "count": 1}, {"token": "ebita*)margin,", "count": 1}]}]}, "content": {"n_oracle": 592, "matched": 579, "forgiven": 9, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.978041, "recall_effective": 0.993243, "missing_samples": [{"token": "basis*),", "count": 1}, {"token": "growth1)", "count": 1}, {"token": "ebita*)amounted", "count": 1}, {"token": "ebita*)margin,", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/a115a4c1424fce2abc68c3077a3e79a98d44e30170d417337b440a871632bf82.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 374, "matched": 373, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997326, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 374, "matched": 373, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997326, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 374, "matched": 374, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 374, "matched": 374, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/WU.2015.page_161.pdf_68095_page1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1173, "matched": 1162, "forgiven": 11, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.990622, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1173, "matched": 1158, "forgiven": 13, "missing_material": 1, "missing_trivial": 1, "recall_strict": 0.987212, "recall_effective": 0.998295, "missing_samples": [{"token": "k", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 1157, "matched": 1147, "forgiven": 10, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.991357, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1157, "matched": 1145, "forgiven": 10, "missing_material": 1, "missing_trivial": 1, "recall_strict": 0.989628, "recall_effective": 0.998271, "missing_samples": [{"token": "10-k", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/a6a468dd5ca52b8f3bc078e9f255659368e59993d288a71c249bb54e18c6ae7d.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 499, "matched": 473, "forgiven": 26, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.947896, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 499, "matched": 469, "forgiven": 4, "missing_material": 4, "missing_trivial": 22, "recall_strict": 0.93988, "recall_effective": 0.947896, "missing_samples": [{"token": "...................................\u00a518,775.3", "count": 1}, {"token": ".....................................\u00a574,263.3", "count": 1}, {"token": "...................................\u00a518,017.3", "count": 1}, {"token": ".....................................\u00a565,112.3", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 511, "matched": 487, "forgiven": 24, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.953033, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 511, "matched": 485, "forgiven": 0, "missing_material": 8, "missing_trivial": 18, "recall_strict": 0.949119, "recall_effective": 0.949119, "missing_samples": [{"token": "...................................\u00a518,775.3", "count": 1}, {"token": ".....................................\u00a574,263.3", "count": 1}, {"token": "...................................\u00a518,017.3", "count": 1}, {"token": ".....................................\u00a565,112.3", "count": 1}, {"token": "..................................\u00a523,232.7", "count": 1}, {"token": ".....................................\u00a524,589.4", "count": 1}, {"token": "..................................\u00a517,929.1", "count": 1}, {"token": ".....................................\u00a519,698.8", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/annual-report-2020_p65.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 434, "matched": 434, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 434, "matched": 431, "forgiven": 0, "missing_material": 0, "missing_trivial": 3, "recall_strict": 0.993088, "recall_effective": 0.993088, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 437, "matched": 431, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.98627, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 437, "matched": 428, "forgiven": 6, "missing_material": 0, "missing_trivial": 3, "recall_strict": 0.979405, "recall_effective": 0.993135, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/annual-report_fy2024_06_en_p14.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 440, "matched": 439, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997727, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 440, "matched": 434, "forgiven": 5, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.986364, "recall_effective": 0.997727, "missing_samples": [{"token": "84", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 440, "matched": 440, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 440, "matched": 435, "forgiven": 4, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.988636, "recall_effective": 0.997727, "missing_samples": [{"token": "84", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/annual-report_fy2024_en_p20.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 617, "matched": 606, "forgiven": 4, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.982172, "recall_effective": 0.988655, "flagged_pages": [{"page": 0, "n_oracle": 617, "matched": 606, "forgiven": 4, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.982172, "recall_effective": 0.988655, "missing_samples": [{"token": "\u00a51,127tn", "count": 1}, {"token": "\u00a52,141tn", "count": 1}, {"token": "\u00a514.1tn", "count": 1}, {"token": "\u00a55.1tn", "count": 1}, {"token": "11.0%", "count": 1}, {"token": "\u00a527.8tn", "count": 1}, {"token": "\u00a54.7tn", "count": 1}]}]}, "content": {"n_oracle": 617, "matched": 603, "forgiven": 7, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.97731, "recall_effective": 0.988655, "missing_samples": [{"token": "\u00a51,127tn", "count": 1}, {"token": "\u00a52,141tn", "count": 1}, {"token": "\u00a514.1tn", "count": 1}, {"token": "\u00a55.1tn", "count": 1}, {"token": "11.0%", "count": 1}, {"token": "\u00a527.8tn", "count": 1}, {"token": "\u00a54.7tn", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 636, "matched": 618, "forgiven": 17, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.971698, "recall_effective": 0.998428, "flagged_pages": [{"page": 0, "n_oracle": 636, "matched": 618, "forgiven": 17, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.971698, "recall_effective": 0.998428, "missing_samples": [{"token": "\u00a51,127tn", "count": 1}]}]}, "content": {"n_oracle": 636, "matched": 615, "forgiven": 20, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.966981, "recall_effective": 0.998428, "missing_samples": [{"token": "\u00a51,127tn", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/b173789c57a86e786d6b0f05d35c1962ed13d99ddf633fa3749acc3af6aadbc1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 152, "matched": 148, "forgiven": 4, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.973684, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 152, "matched": 147, "forgiven": 4, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.967105, "recall_effective": 0.993421, "missing_samples": [{"token": "11", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 126, "matched": 102, "forgiven": 24, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.809524, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 126, "matched": 101, "forgiven": 24, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.801587, "recall_effective": 0.992063, "missing_samples": [{"token": "11", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/a7c671cb50fe6dae860f2bafb4ab7db58717efa7c99c11091d2d8dc7148a2a5d.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 228, "matched": 228, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 228, "matched": 226, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.991228, "recall_effective": 0.995614, "missing_samples": [{"token": "s-1", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 228, "matched": 228, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 228, "matched": 226, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.991228, "recall_effective": 0.995614, "missing_samples": [{"token": "s-1", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/ar2024e_p49.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 514, "matched": 505, "forgiven": 5, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.98249, "recall_effective": 0.992218, "flagged_pages": [{"page": 0, "n_oracle": 514, "matched": 505, "forgiven": 5, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.98249, "recall_effective": 0.992218, "missing_samples": [{"token": "51.07%", "count": 1}, {"token": "0.01%", "count": 1}, {"token": "14.52%", "count": 1}, {"token": "1.59%", "count": 1}]}]}, "content": {"n_oracle": 514, "matched": 503, "forgiven": 7, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.978599, "recall_effective": 0.992218, "missing_samples": [{"token": "51.07%", "count": 1}, {"token": "0.01%", "count": 1}, {"token": "14.52%", "count": 1}, {"token": "1.59%", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 515, "matched": 504, "forgiven": 7, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.978641, "recall_effective": 0.992233, "flagged_pages": [{"page": 0, "n_oracle": 515, "matched": 504, "forgiven": 7, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.978641, "recall_effective": 0.992233, "missing_samples": [{"token": "14.52%", "count": 1}, {"token": "0.01%", "count": 1}, {"token": "1.59%", "count": 1}, {"token": "51.07%", "count": 1}]}]}, "content": {"n_oracle": 515, "matched": 503, "forgiven": 8, "missing_material": 4, "missing_trivial": 0, "recall_strict": 0.976699, "recall_effective": 0.992233, "missing_samples": [{"token": "14.52%", "count": 1}, {"token": "0.01%", "count": 1}, {"token": "1.59%", "count": 1}, {"token": "51.07%", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/bd-sustainability-report-2024_p82.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 614, "matched": 614, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 614, "matched": 614, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 614, "matched": 614, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 614, "matched": 614, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/c826c4873f48af66abbefe7ed5b11e90e4a48357b44230a407e43e7ced28b159.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 771, "matched": 758, "forgiven": 12, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.983139, "recall_effective": 0.998703, "flagged_pages": [{"page": 0, "n_oracle": 771, "matched": 758, "forgiven": 12, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.983139, "recall_effective": 0.998703, "missing_samples": [{"token": "kh2po4", "count": 1}]}]}, "content": {"n_oracle": 771, "matched": 755, "forgiven": 13, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.979248, "recall_effective": 0.996109, "missing_samples": [{"token": "aug.", "count": 1}, {"token": "6,", "count": 1}, {"token": "kh2po4", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 772, "matched": 764, "forgiven": 8, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.989637, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 772, "matched": 761, "forgiven": 9, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.985751, "recall_effective": 0.997409, "missing_samples": [{"token": "aug.", "count": 1}, {"token": "6,", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/d875f8f9efde74972bd5911e30afb8b651fa0f009b0066b702e80db4b587d770.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 543, "matched": 531, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.977901, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 543, "matched": 528, "forgiven": 15, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.972376, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 546, "matched": 536, "forgiven": 10, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.981685, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 546, "matched": 533, "forgiven": 13, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.97619, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/dec974701437d8ecfe8fd1045b2e4c5ac2c5e723f566cd1cc873c5baf5bb8d51.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 237, "matched": 235, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.991561, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 237, "matched": 234, "forgiven": 2, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.987342, "recall_effective": 0.995781, "missing_samples": [{"token": "23", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 241, "matched": 237, "forgiven": 4, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.983402, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 241, "matched": 236, "forgiven": 4, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.979253, "recall_effective": 0.995851, "missing_samples": [{"token": "23", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/docu-robeco-integrated-annual-report-2024_p25.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 367, "matched": 337, "forgiven": 17, "missing_material": 13, "missing_trivial": 0, "recall_strict": 0.918256, "recall_effective": 0.964578, "flagged_pages": [{"page": 0, "n_oracle": 367, "matched": 337, "forgiven": 17, "missing_material": 13, "missing_trivial": 0, "recall_strict": 0.918256, "recall_effective": 0.964578, "missing_samples": [{"token": "sustainability", "count": 1}, {"token": "dialogue", "count": 1}, {"token": "survey", "count": 1}, {"token": "co2e", "count": 1}, {"token": "sustainable", "count": 1}, {"token": "emerging", "count": 1}, {"token": "markets", "count": 1}, {"token": "credits", "count": 1}, {"token": "trends", "count": 1}, {"token": "centric", "count": 1}, {"token": "inn", "count": 1}, {"token": "ovative", "count": 1}, {"token": "connecting", "count": 1}]}]}, "content": {"n_oracle": 367, "matched": 337, "forgiven": 17, "missing_material": 13, "missing_trivial": 0, "recall_strict": 0.918256, "recall_effective": 0.964578, "missing_samples": [{"token": "sustainability", "count": 1}, {"token": "dialogue", "count": 1}, {"token": "survey", "count": 1}, {"token": "co2e", "count": 1}, {"token": "sustainable", "count": 1}, {"token": "emerging", "count": 1}, {"token": "markets", "count": 1}, {"token": "credits", "count": 1}, {"token": "trends", "count": 1}, {"token": "centric", "count": 1}, {"token": "inn", "count": 1}, {"token": "ovative", "count": 1}, {"token": "connecting", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 554, "matched": 392, "forgiven": 154, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.707581, "recall_effective": 0.98556, "flagged_pages": [{"page": 0, "n_oracle": 554, "matched": 392, "forgiven": 154, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.707581, "recall_effective": 0.98556, "missing_samples": [{"token": "2255", "count": 1}, {"token": "rroobbeeccoo", "count": 1}, {"token": "iinntteeggrraatteedd", "count": 1}, {"token": "aannnnuuaall", "count": 1}, {"token": "rreeppoorrtt", "count": 1}, {"token": "22002243", "count": 1}, {"token": "dialogue", "count": 1}, {"token": "survey", "count": 1}]}]}, "content": {"n_oracle": 554, "matched": 392, "forgiven": 154, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.707581, "recall_effective": 0.98556, "missing_samples": [{"token": "2255", "count": 1}, {"token": "rroobbeeccoo", "count": 1}, {"token": "iinntteeggrraatteedd", "count": 1}, {"token": "aannnnuuaall", "count": 1}, {"token": "rreeppoorrtt", "count": 1}, {"token": "22002243", "count": 1}, {"token": "dialogue", "count": 1}, {"token": "survey", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/docu-robeco-integrated-annual-report-2024_p45.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 805, "matched": 805, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 805, "matched": 785, "forgiven": 7, "missing_material": 13, "missing_trivial": 0, "recall_strict": 0.975155, "recall_effective": 0.983851, "missing_samples": [{"token": "of", "count": 2}, {"token": "by", "count": 1}, {"token": "misconduct", "count": 1}, {"token": "serious", "count": 1}, {"token": "error", "count": 1}, {"token": "judgement", "count": 1}, {"token": "evidence", "count": 1}, {"token": "fundamental", "count": 1}, {"token": "misconduct,", "count": 1}, {"token": "errors", "count": 1}, {"token": "integrity", "count": 1}, {"token": "issues", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 805, "matched": 805, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 805, "matched": 785, "forgiven": 7, "missing_material": 13, "missing_trivial": 0, "recall_strict": 0.975155, "recall_effective": 0.983851, "missing_samples": [{"token": "of", "count": 2}, {"token": "by", "count": 1}, {"token": "misconduct", "count": 1}, {"token": "serious", "count": 1}, {"token": "error", "count": 1}, {"token": "judgement", "count": 1}, {"token": "evidence", "count": 1}, {"token": "fundamental", "count": 1}, {"token": "misconduct,", "count": 1}, {"token": "errors", "count": 1}, {"token": "integrity", "count": 1}, {"token": "issues", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/f792e252d22b6dd24797fe0eeba24bfc6b2b9d4d05eaa8bbc1c04d7a4fb5a7b9.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 94, "matched": 94, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 94, "matched": 94, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 93, "matched": 91, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.978495, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 93, "matched": 91, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.978495, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/f106fc6aa8028e5cbe474bce8597c10db70cf3b7ec55b41957addc8a2f7155d4.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 448, "matched": 448, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 448, "matched": 446, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.995536, "recall_effective": 0.995536, "missing_samples": [{"token": "(bgbl.", "count": 2}]}}, "plumber": {"pawls": {"n_oracle": 448, "matched": 448, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 448, "matched": 446, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.995536, "recall_effective": 0.995536, "missing_samples": [{"token": "(bgbl.", "count": 2}]}}}} +{"doc": "tests/fixtures/hetero100/pages/chapter_10_p2.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 303, "matched": 197, "forgiven": 34, "missing_material": 72, "missing_trivial": 0, "recall_strict": 0.650165, "recall_effective": 0.762376, "flagged_pages": [{"page": 0, "n_oracle": 303, "matched": 197, "forgiven": 34, "missing_material": 72, "missing_trivial": 0, "recall_strict": 0.650165, "recall_effective": 0.762376, "missing_samples": [{"token": "handwriting", "count": 3}, {"token": "other", "count": 2}, {"token": "questioned", "count": 2}, {"token": "forgery", "count": 2}, {"token": "detect", "count": 1}, {"token": "vocabulary", "count": 1}, {"token": "counterfeiting", "count": 1}, {"token": "production", "count": 1}, {"token": "imitation", "count": 1}, {"token": "art,", "count": 1}, {"token": "documents,", "count": 1}, {"token": "brand", "count": 1}, {"token": "look-alikes", "count": 1}, {"token": "purpose", "count": 1}, {"token": "documents", "count": 1}, {"token": "examination", "count": 1}, {"token": "known", "count": 1}, {"token": "material", "count": 1}, {"token": "variety", "count": 1}, {"token": "analyses,", "count": 1}, {"token": "such", "count": 1}, {"token": "authentic", "count": 1}, {"token": "ity,", "count": 1}, {"token": "alterations,", "count": 1}, {"token": "erasures,", "count": 1}]}]}, "content": {"n_oracle": 303, "matched": 196, "forgiven": 34, "missing_material": 73, "missing_trivial": 0, "recall_strict": 0.646865, "recall_effective": 0.759076, "missing_samples": [{"token": "handwriting", "count": 3}, {"token": "other", "count": 2}, {"token": "questioned", "count": 2}, {"token": "forgery", "count": 2}, {"token": "277", "count": 1}, {"token": "detect", "count": 1}, {"token": "vocabulary", "count": 1}, {"token": "counterfeiting", "count": 1}, {"token": "production", "count": 1}, {"token": "imitation", "count": 1}, {"token": "art,", "count": 1}, {"token": "documents,", "count": 1}, {"token": "brand", "count": 1}, {"token": "look-alikes", "count": 1}, {"token": "purpose", "count": 1}, {"token": "documents", "count": 1}, {"token": "examination", "count": 1}, {"token": "known", "count": 1}, {"token": "material", "count": 1}, {"token": "variety", "count": 1}, {"token": "analyses,", "count": 1}, {"token": "such", "count": 1}, {"token": "authentic", "count": 1}, {"token": "ity,", "count": 1}, {"token": "alterations,", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 1385, "matched": 151, "forgiven": 1224, "missing_material": 10, "missing_trivial": 0, "recall_strict": 0.109025, "recall_effective": 0.99278, "flagged_pages": [{"page": 0, "n_oracle": 1385, "matched": 151, "forgiven": 1224, "missing_material": 10, "missing_trivial": 0, "recall_strict": 0.109025, "recall_effective": 0.99278, "missing_samples": [{"token": "gery.", "count": 1}, {"token": "vocabulary", "count": 1}, {"token": ".noitazirohtua", "count": 1}, {"token": "sserpxe", "count": 1}, {"token": "tuohtiw", "count": 1}, {"token": "dewolla", "count": 1}, {"token": "noitubirtsid", "count": 1}, {"token": ".devreser", "count": 1}, {"token": "sthgir", "count": 1}, {"token": "lla", "count": 1}]}]}, "content": {"n_oracle": 1385, "matched": 143, "forgiven": 1224, "missing_material": 18, "missing_trivial": 0, "recall_strict": 0.103249, "recall_effective": 0.987004, "missing_samples": [{"token": "gery.", "count": 1}, {"token": "vocabulary", "count": 1}, {"token": "277", "count": 1}, {"token": "9944003344__1100__cc1100__pp227766--330077..iinndddd", "count": 1}, {"token": "227777", "count": 1}, {"token": "11//1177//1111", "count": 1}, {"token": "77::2211::1199", "count": 1}, {"token": "ppmm", "count": 1}, {"token": ".noitazirohtua", "count": 1}, {"token": "sserpxe", "count": 1}, {"token": "tuohtiw", "count": 1}, {"token": "dewolla", "count": 1}, {"token": "noitubirtsid", "count": 1}, {"token": ".devreser", "count": 1}, {"token": "sthgir", "count": 1}, {"token": "lla", "count": 1}, {"token": ".gninrael", "count": 1}, {"token": "egagnec", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/fh_2025_004e_p7.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 442, "matched": 442, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 442, "matched": 442, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 521, "matched": 411, "forgiven": 110, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.788868, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 521, "matched": 411, "forgiven": 110, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.788868, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/f7a6a9baac6e7074c192565f35f4d198c67f499e53f85644b9eda945ddf97efd.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 488, "matched": 488, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 488, "matched": 488, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 495, "matched": 494, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99798, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 495, "matched": 494, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99798, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/f83a4f2fce3f3da60b857fcbbabf3f9c5b8882a229e641688d7b37699330722f.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 328, "matched": 326, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.993902, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 328, "matched": 321, "forgiven": 2, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.978659, "recall_effective": 0.984756, "missing_samples": [{"token": "8.9", "count": 1}, {"token": "*4-wall", "count": 1}, {"token": "**inventory", "count": 1}, {"token": "***see", "count": 1}, {"token": "nordstrom.com/companyreview", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 344, "matched": 344, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 344, "matched": 330, "forgiven": 2, "missing_material": 12, "missing_trivial": 0, "recall_strict": 0.959302, "recall_effective": 0.965116, "missing_samples": [{"token": "2014", "count": 1}, {"token": "8.9", "count": 1}, {"token": "*4-wall", "count": 1}, {"token": "**inventory", "count": 1}, {"token": "annual", "count": 1}, {"token": "***see", "count": 1}, {"token": "nordstrom.com/companyreview", "count": 1}, {"token": "21008", "count": 1}, {"token": "037404a", "count": 1}, {"token": "report", "count": 1}, {"token": "pg", "count": 1}, {"token": "ifc-ibc", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/giv-2024-integrated-report_0_p5.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 575, "matched": 575, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 575, "matched": 555, "forgiven": 12, "missing_material": 5, "missing_trivial": 3, "recall_strict": 0.965217, "recall_effective": 0.986087, "missing_samples": [{"token": "disclosure", "count": 1}, {"token": "nature-related", "count": 1}, {"token": "(tnfd)", "count": 1}, {"token": "from", "count": 1}, {"token": "print", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 575, "matched": 575, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 575, "matched": 555, "forgiven": 12, "missing_material": 5, "missing_trivial": 3, "recall_strict": 0.965217, "recall_effective": 0.986087, "missing_samples": [{"token": "disclosure", "count": 1}, {"token": "from", "count": 1}, {"token": "nature-related", "count": 1}, {"token": "(tnfd)", "count": 1}, {"token": "print", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/hdr_2016_statistical_annex_p55.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1369, "matched": 1368, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99927, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1369, "matched": 1216, "forgiven": 1, "missing_material": 0, "missing_trivial": 152, "recall_strict": 0.88824, "recall_effective": 0.88897, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 1873, "matched": 1255, "forgiven": 618, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.670048, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1873, "matched": 1103, "forgiven": 618, "missing_material": 0, "missing_trivial": 152, "recall_strict": 0.588895, "recall_effective": 0.918847, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/huhtamaki-annual-report-2024-printer-friendly_p32.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 432, "matched": 348, "forgiven": 84, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.805556, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 432, "matched": 340, "forgiven": 85, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.787037, "recall_effective": 0.983796, "missing_samples": [{"token": "water", "count": 2}, {"token": "million", "count": 1}, {"token": "water:", "count": 1}, {"token": "discharge", "count": 1}, {"token": "7.2", "count": 1}, {"token": "m3;", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 433, "matched": 350, "forgiven": 83, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.808314, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 433, "matched": 342, "forgiven": 84, "missing_material": 7, "missing_trivial": 0, "recall_strict": 0.789838, "recall_effective": 0.983834, "missing_samples": [{"token": "water", "count": 2}, {"token": "million", "count": 1}, {"token": "water:", "count": 1}, {"token": "discharge", "count": 1}, {"token": "7.2", "count": 1}, {"token": "m3;", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/integrated-annual-report-fy2024-25_p4.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 892, "matched": 883, "forgiven": 6, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.98991, "recall_effective": 0.996637, "flagged_pages": [{"page": 0, "n_oracle": 892, "matched": 883, "forgiven": 6, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.98991, "recall_effective": 0.996637, "missing_samples": [{"token": "integrated", "count": 1}, {"token": "statutory", "count": 1}, {"token": "reports", "count": 1}]}]}, "content": {"n_oracle": 892, "matched": 881, "forgiven": 6, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.987668, "recall_effective": 0.994395, "missing_samples": [{"token": "integrated", "count": 1}, {"token": "statutory", "count": 1}, {"token": "04", "count": 1}, {"token": "05", "count": 1}, {"token": "reports", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 892, "matched": 886, "forgiven": 3, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.993274, "recall_effective": 0.996637, "flagged_pages": [{"page": 0, "n_oracle": 892, "matched": 886, "forgiven": 3, "missing_material": 3, "missing_trivial": 0, "recall_strict": 0.993274, "recall_effective": 0.996637, "missing_samples": [{"token": "integrated", "count": 1}, {"token": "statutory", "count": 1}, {"token": "reports", "count": 1}]}]}, "content": {"n_oracle": 892, "matched": 884, "forgiven": 3, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.991031, "recall_effective": 0.994395, "missing_samples": [{"token": "integrated", "count": 1}, {"token": "statutory", "count": 1}, {"token": "04", "count": 1}, {"token": "05", "count": 1}, {"token": "reports", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/integrated-report-consolidated_p64.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 458, "matched": 457, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997817, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 458, "matched": 457, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997817, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 458, "matched": 458, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 458, "matched": 458, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/iul_age_50_example_p12.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 242, "matched": 242, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 242, "matched": 241, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.995868, "recall_effective": 0.995868, "missing_samples": [{"token": "*this", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 242, "matched": 242, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 242, "matched": 241, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.995868, "recall_effective": 0.995868, "missing_samples": [{"token": "*this", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/iul_age_50_example_p6.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 228, "matched": 228, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 228, "matched": 227, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.995614, "recall_effective": 0.995614, "missing_samples": [{"token": "*this", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 228, "matched": 228, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 228, "matched": 227, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.995614, "recall_effective": 0.995614, "missing_samples": [{"token": "*this", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/ir2025e_all_p20.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1263, "matched": 1242, "forgiven": 20, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.983373, "recall_effective": 0.999208, "flagged_pages": [{"page": 0, "n_oracle": 1263, "matched": 1242, "forgiven": 20, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.983373, "recall_effective": 0.999208, "missing_samples": [{"token": "(3\u6708\u671f)", "count": 1}]}]}, "content": {"n_oracle": 1263, "matched": 1230, "forgiven": 24, "missing_material": 9, "missing_trivial": 0, "recall_strict": 0.973872, "recall_effective": 0.992874, "missing_samples": [{"token": "(3\u6708\u671f)", "count": 1}, {"token": "https://www.mazda.com/en/sustainability/social/human-capital/", "count": 1}, {"token": "completion", "count": 1}, {"token": "december", "count": 1}, {"token": "2026)", "count": 1}, {"token": "integrated", "count": 1}, {"token": "report", "count": 1}, {"token": "38", "count": 1}, {"token": "01", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 1260, "matched": 1255, "forgiven": 5, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996032, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1260, "matched": 1243, "forgiven": 9, "missing_material": 8, "missing_trivial": 0, "recall_strict": 0.986508, "recall_effective": 0.993651, "missing_samples": [{"token": "01", "count": 1}, {"token": "https://www.mazda.com/en/sustainability/social/human-capital/", "count": 1}, {"token": "completion", "count": 1}, {"token": "december", "count": 1}, {"token": "2026)", "count": 1}, {"token": "integrated", "count": 1}, {"token": "report", "count": 1}, {"token": "38", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/lm555_p1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 262, "matched": 262, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 262, "matched": 234, "forgiven": 8, "missing_material": 20, "missing_trivial": 0, "recall_strict": 0.89313, "recall_effective": 0.923664, "missing_samples": [{"token": "important", "count": 2}, {"token": "the", "count": 1}, {"token": "and", "count": 1}, {"token": "notice", "count": 1}, {"token": "this", "count": 1}, {"token": "addresses", "count": 1}, {"token": "availability,", "count": 1}, {"token": "warranty,", "count": 1}, {"token": "changes,", "count": 1}, {"token": "use", "count": 1}, {"token": "safety-critical", "count": 1}, {"token": "applications,", "count": 1}, {"token": "intellectual", "count": 1}, {"token": "property", "count": 1}, {"token": "matters", "count": 1}, {"token": "other", "count": 1}, {"token": "disclaimers.", "count": 1}, {"token": "production", "count": 1}, {"token": "data.", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 171, "matched": 141, "forgiven": 30, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.824561, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 171, "matched": 126, "forgiven": 34, "missing_material": 11, "missing_trivial": 0, "recall_strict": 0.736842, "recall_effective": 0.935673, "missing_samples": [{"token": "important", "count": 1}, {"token": "notice", "count": 1}, {"token": "this", "count": 1}, {"token": "addresses", "count": 1}, {"token": "availability,", "count": 1}, {"token": "warranty,", "count": 1}, {"token": "changes,", "count": 1}, {"token": "use", "count": 1}, {"token": "safety-critical", "count": 1}, {"token": "applications,", "count": 1}, {"token": "intellectualpropertymattersandotherimportantdisclaimers.productiondata.", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/ir2025e_all_p43.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1182, "matched": 1167, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.98731, "recall_effective": 0.998308, "flagged_pages": [{"page": 0, "n_oracle": 1182, "matched": 1167, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.98731, "recall_effective": 0.998308, "missing_samples": [{"token": "t-co2e", "count": 1}, {"token": "low-co2-emissions", "count": 1}]}]}, "content": {"n_oracle": 1182, "matched": 1146, "forgiven": 27, "missing_material": 9, "missing_trivial": 0, "recall_strict": 0.969543, "recall_effective": 0.992386, "missing_samples": [{"token": "hiroshima", "count": 2}, {"token": "t-co2e", "count": 1}, {"token": "including", "count": 1}, {"token": "mazda's", "count": 1}, {"token": "head", "count": 1}, {"token": "office", "count": 1}, {"token": "(aki-gun", "count": 1}, {"token": "low-co2-emissions", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 1233, "matched": 1169, "forgiven": 62, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.948094, "recall_effective": 0.998378, "flagged_pages": [{"page": 0, "n_oracle": 1233, "matched": 1169, "forgiven": 62, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.948094, "recall_effective": 0.998378, "missing_samples": [{"token": "low-co2-emissions", "count": 1}, {"token": "t-co2e", "count": 1}]}]}, "content": {"n_oracle": 1233, "matched": 1148, "forgiven": 76, "missing_material": 9, "missing_trivial": 0, "recall_strict": 0.931062, "recall_effective": 0.992701, "missing_samples": [{"token": "hiroshima", "count": 2}, {"token": "mazda's", "count": 1}, {"token": "low-co2-emissions", "count": 1}, {"token": "including", "count": 1}, {"token": "head", "count": 1}, {"token": "office", "count": 1}, {"token": "(aki-gun", "count": 1}, {"token": "t-co2e", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/maersk-annual-report-2024_p22.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1157, "matched": 1124, "forgiven": 33, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.971478, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1157, "matched": 1093, "forgiven": 42, "missing_material": 22, "missing_trivial": 0, "recall_strict": 0.944685, "recall_effective": 0.980985, "missing_samples": [{"token": "chief", "count": 6}, {"token": "officer", "count": 2}, {"token": "1", "count": 1}, {"token": "technology", "count": 1}, {"token": "cso", "count": 1}, {"token": "executive", "count": 1}, {"token": "officercoo", "count": 1}, {"token": "operating", "count": 1}, {"token": "officerccao", "count": 1}, {"token": "corporate", "count": 1}, {"token": "affairs", "count": 1}, {"token": "officercto", "count": 1}, {"token": "officercpo", "count": 1}, {"token": "22", "count": 1}, {"token": "a.p.", "count": 1}, {"token": "moller", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 958, "matched": 946, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.987474, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 958, "matched": 909, "forgiven": 25, "missing_material": 24, "missing_trivial": 0, "recall_strict": 0.948852, "recall_effective": 0.974948, "missing_samples": [{"token": "chief", "count": 6}, {"token": "officer", "count": 6}, {"token": "1", "count": 1}, {"token": "technology", "count": 1}, {"token": "ccao", "count": 1}, {"token": "cso", "count": 1}, {"token": "executive", "count": 1}, {"token": "coo", "count": 1}, {"token": "corporate", "count": 1}, {"token": "22", "count": 1}, {"token": "a.p.", "count": 1}, {"token": "moller", "count": 1}, {"token": "operating", "count": 1}, {"token": "affairs", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/maersk-annual-report-2024_p8.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 518, "matched": 499, "forgiven": 18, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.96332, "recall_effective": 0.998069, "flagged_pages": [{"page": 0, "n_oracle": 518, "matched": 499, "forgiven": 18, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.96332, "recall_effective": 0.998069, "missing_samples": [{"token": "5.1bn", "count": 1}]}]}, "content": {"n_oracle": 518, "matched": 499, "forgiven": 18, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.96332, "recall_effective": 0.998069, "missing_samples": [{"token": "5.1bn", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 522, "matched": 514, "forgiven": 8, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.984674, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 522, "matched": 514, "forgiven": 8, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.984674, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/novartis-integrated-report-2021_p2.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 707, "matched": 705, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997171, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 707, "matched": 705, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997171, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 708, "matched": 706, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997175, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 708, "matched": 706, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.997175, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/novartis-integrated-report-2024_p51.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 257, "matched": 250, "forgiven": 7, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.972763, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 257, "matched": 246, "forgiven": 11, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.957198, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 265, "matched": 265, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 265, "matched": 257, "forgiven": 8, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.969811, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/orbia_impact_report_2024_p94.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 328, "matched": 328, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 328, "matched": 328, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 328, "matched": 328, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 328, "matched": 328, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/novartis-integrated-report-2021_p8.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 967, "matched": 952, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.984488, "recall_effective": 0.997932, "flagged_pages": [{"page": 0, "n_oracle": 967, "matched": 952, "forgiven": 13, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.984488, "recall_effective": 0.997932, "missing_samples": [{"token": "impact", "count": 1}, {"token": "external", "count": 1}]}]}, "content": {"n_oracle": 967, "matched": 946, "forgiven": 16, "missing_material": 5, "missing_trivial": 0, "recall_strict": 0.978283, "recall_effective": 0.994829, "missing_samples": [{"token": "impact", "count": 1}, {"token": "external", "count": 1}, {"token": "partners", "count": 1}, {"token": "policymakers", "count": 1}, {"token": "regulators", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 971, "matched": 964, "forgiven": 1, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.992791, "recall_effective": 0.993821, "flagged_pages": [{"page": 0, "n_oracle": 971, "matched": 964, "forgiven": 1, "missing_material": 6, "missing_trivial": 0, "recall_strict": 0.992791, "recall_effective": 0.993821, "missing_samples": [{"token": "tcapmi", "count": 2}, {"token": "hgih", "count": 1}, {"token": "wol", "count": 1}, {"token": "sredlohekats", "count": 1}, {"token": "lanretxe", "count": 1}]}]}, "content": {"n_oracle": 971, "matched": 958, "forgiven": 4, "missing_material": 9, "missing_trivial": 0, "recall_strict": 0.986612, "recall_effective": 0.990731, "missing_samples": [{"token": "tcapmi", "count": 2}, {"token": "hgih", "count": 1}, {"token": "wol", "count": 1}, {"token": "sredlohekats", "count": 1}, {"token": "lanretxe", "count": 1}, {"token": "partners", "count": 1}, {"token": "policymakers", "count": 1}, {"token": "regulators", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/pdf_211245533966_p5.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 416, "matched": 410, "forgiven": 5, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.985577, "recall_effective": 0.997596, "flagged_pages": [{"page": 0, "n_oracle": 416, "matched": 410, "forgiven": 5, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.985577, "recall_effective": 0.997596, "missing_samples": [{"token": "2025.8", "count": 1}]}]}, "content": {"n_oracle": 416, "matched": 410, "forgiven": 5, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.985577, "recall_effective": 0.997596, "missing_samples": [{"token": "2025.8", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 415, "matched": 414, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.99759, "recall_effective": 0.99759, "flagged_pages": [{"page": 0, "n_oracle": 415, "matched": 414, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.99759, "recall_effective": 0.99759, "missing_samples": [{"token": "2025.8", "count": 1}]}]}, "content": {"n_oracle": 415, "matched": 414, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.99759, "recall_effective": 0.99759, "missing_samples": [{"token": "2025.8", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/pdf_211245533966_p15.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 761, "matched": 761, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 761, "matched": 761, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 761, "matched": 761, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 761, "matched": 761, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/pdf_29f6bce2b33c_p19.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 633, "matched": 631, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.99684, "recall_effective": 0.99684, "flagged_pages": [{"page": 0, "n_oracle": 633, "matched": 631, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.99684, "recall_effective": 0.99684, "missing_samples": [{"token": "strategic", "count": 1}, {"token": "report", "count": 1}]}]}, "content": {"n_oracle": 633, "matched": 631, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.99684, "recall_effective": 0.99684, "missing_samples": [{"token": "strategic", "count": 1}, {"token": "report", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 633, "matched": 631, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.99684, "recall_effective": 0.99684, "flagged_pages": [{"page": 0, "n_oracle": 633, "matched": 631, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.99684, "recall_effective": 0.99684, "missing_samples": [{"token": "strategic", "count": 1}, {"token": "report", "count": 1}]}]}, "content": {"n_oracle": 633, "matched": 631, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.99684, "recall_effective": 0.99684, "missing_samples": [{"token": "strategic", "count": 1}, {"token": "report", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/pdf_cfd715c9c8f3_p94.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1505, "matched": 1500, "forgiven": 5, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.996678, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1505, "matched": 1475, "forgiven": 17, "missing_material": 13, "missing_trivial": 0, "recall_strict": 0.980066, "recall_effective": 0.991362, "missing_samples": [{"token": "directors'", "count": 1}, {"token": "attendance", "count": 1}, {"token": "100%", "count": 1}, {"token": "(13/13)", "count": 1}, {"token": "outside,", "count": 1}, {"token": "emi", "count": 1}, {"token": "osono", "count": 1}, {"token": "female", "count": 1}, {"token": "58", "count": 1}, {"token": "figures", "count": 1}, {"token": "fiscal", "count": 1}, {"token": "2024.", "count": 1}, {"token": "follows..", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 1695, "matched": 1472, "forgiven": 223, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.868437, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1695, "matched": 1452, "forgiven": 231, "missing_material": 12, "missing_trivial": 0, "recall_strict": 0.856637, "recall_effective": 0.99292, "missing_samples": [{"token": "follows..", "count": 1}, {"token": "fiscal", "count": 1}, {"token": "directors'", "count": 1}, {"token": "2024.", "count": 1}, {"token": "attendance", "count": 1}, {"token": "100%", "count": 1}, {"token": "(13/13)", "count": 1}, {"token": "emi", "count": 1}, {"token": "osono", "count": 1}, {"token": "female", "count": 1}, {"token": "58", "count": 1}, {"token": "figures", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/pdf_d47bf4ce95f6_p71.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 616, "matched": 610, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99026, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 616, "matched": 610, "forgiven": 6, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.99026, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 613, "matched": 613, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 613, "matched": 613, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/pdf_cfd715c9c8f3_p95.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1654, "matched": 1647, "forgiven": 7, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.995768, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1654, "matched": 1645, "forgiven": 9, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.994559, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 3357, "matched": 1259, "forgiven": 2098, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.375037, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3357, "matched": 1257, "forgiven": 2099, "missing_material": 0, "missing_trivial": 1, "recall_strict": 0.374441, "recall_effective": 0.999702, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/pdf_dfe73b598dd5_p25.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 525, "matched": 525, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 525, "matched": 523, "forgiven": 1, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.99619, "recall_effective": 0.998095, "missing_samples": [{"token": "25", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 516, "matched": 507, "forgiven": 9, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.982558, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 516, "matched": 505, "forgiven": 10, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.978682, "recall_effective": 0.998062, "missing_samples": [{"token": "25", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/p29_page1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 3321, "matched": 3321, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3321, "matched": 3321, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 3435, "matched": 3287, "forgiven": 148, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.956914, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 3435, "matched": 3287, "forgiven": 148, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.956914, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/pdf_fe474dd12f60_p12.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 242, "matched": 242, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 242, "matched": 241, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.995868, "recall_effective": 0.995868, "missing_samples": [{"token": "*this", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 242, "matched": 242, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 242, "matched": 241, "forgiven": 0, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.995868, "recall_effective": 0.995868, "missing_samples": [{"token": "*this", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/sbkk_integrated_report_2024_en_p2.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 536, "matched": 533, "forgiven": 3, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.994403, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 536, "matched": 531, "forgiven": 4, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.990672, "recall_effective": 0.998134, "missing_samples": [{"token": "p00", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 542, "matched": 530, "forgiven": 12, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.97786, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 542, "matched": 528, "forgiven": 13, "missing_material": 1, "missing_trivial": 0, "recall_strict": 0.97417, "recall_effective": 0.998155, "missing_samples": [{"token": "p00", "count": 1}]}}}} +{"doc": "tests/fixtures/hetero100/pages/sbkk_integrated_report_2020_section4_en_p6.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 1152, "matched": 1144, "forgiven": 8, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.993056, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1152, "matched": 1143, "forgiven": 8, "missing_material": 0, "missing_trivial": 1, "recall_strict": 0.992188, "recall_effective": 0.999132, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 1193, "matched": 1138, "forgiven": 55, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.953898, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 1193, "matched": 1137, "forgiven": 55, "missing_material": 0, "missing_trivial": 1, "recall_strict": 0.95306, "recall_effective": 0.999162, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/scas-annual-report-2023_eng_klickbar_p56.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 544, "matched": 541, "forgiven": 3, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.994485, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 544, "matched": 541, "forgiven": 3, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.994485, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 544, "matched": 544, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 544, "matched": 544, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/scas-annual-report-2023_eng_klickbar_p76.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 737, "matched": 737, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 737, "matched": 737, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 737, "matched": 737, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 737, "matched": 737, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/solvay-2023-ENG_p27.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 662, "matched": 659, "forgiven": 3, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.995468, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 662, "matched": 659, "forgiven": 3, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.995468, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 662, "matched": 662, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 662, "matched": 662, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/solvay-fre73e8a389231629a930fff0000028e79_p59.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 701, "matched": 680, "forgiven": 21, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.970043, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 701, "matched": 678, "forgiven": 23, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.96719, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 707, "matched": 685, "forgiven": 22, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.968883, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 707, "matched": 683, "forgiven": 24, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.966054, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/cerebras_s1__exhibit231sx1_d5d6f3.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 73, "matched": 73, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 73, "matched": 73, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 73, "matched": 73, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 73, "matched": 73, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/eikon_s1__ex231_6fa7f0.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 69, "matched": 67, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.971014, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 69, "matched": 67, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.971014, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 72, "matched": 71, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.986111, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 72, "matched": 71, "forgiven": 1, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.986111, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/exyn_s1__ex211_812970.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 20, "matched": 20, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 20, "matched": 20, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 20, "matched": 20, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 20, "matched": 20, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/fervo_s1__exhibit211sx1_0c768b.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 34, "matched": 34, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 34, "matched": 34, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 34, "matched": 34, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 34, "matched": 34, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/forbright_s1__exhibit211sx1p_2eae0d.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 17, "matched": 17, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 17, "matched": 17, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 17, "matched": 17, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 17, "matched": 17, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/generate_bio_s1__ex211_84dda0.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 14, "matched": 14, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 14, "matched": 14, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 14, "matched": 14, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 14, "matched": 14, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/hawkeye360_s1__exhibit211sx1_eed44c.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 23, "matched": 23, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 23, "matched": 23, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 23, "matched": 23, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 23, "matched": 23, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/infleqtion_s1__ex211_b9070a.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 25, "matched": 25, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 25, "matched": 25, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 25, "matched": 25, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 25, "matched": 25, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/innio_s1__ex231_490e47.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 57, "matched": 57, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 57, "matched": 57, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 57, "matched": 57, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 57, "matched": 57, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/kardigan_s1__ex211_6d4418.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 18, "matched": 18, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 18, "matched": 18, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 18, "matched": 18, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 18, "matched": 18, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/hetero100/pages/sample_page_16_page1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 2585, "matched": 2585, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 2585, "matched": 2585, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 2703, "matched": 2557, "forgiven": 146, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.945986, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 2703, "matched": 2557, "forgiven": 146, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.945986, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/liftoff_s1__ex231_c133b1.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 64, "matched": 64, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 64, "matched": 64, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 64, "matched": 64, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 64, "matched": 64, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/parabilis_s1__ex211_f13ef7.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 20, "matched": 20, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 20, "matched": 20, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 20, "matched": 20, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 20, "matched": 20, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/neutron_lime_s1__exhibit231sx1_96d831.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 72, "matched": 72, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 72, "matched": 72, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 72, "matched": 72, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 72, "matched": 72, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/pershing_square_s1__ex231_22654c.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 67, "matched": 67, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 67, "matched": 67, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 67, "matched": 67, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 67, "matched": 67, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/solv_energy_s1__ex231_98c758.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 83, "matched": 81, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.975904, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 83, "matched": 81, "forgiven": 2, "missing_material": 0, "missing_trivial": 0, "recall_strict": 0.975904, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 85, "matched": 85, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 85, "matched": 85, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/spacex_s1__exhibit211sx1_3ecd38.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 27, "matched": 27, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 27, "matched": 25, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.925926, "recall_effective": 0.925926, "missing_samples": [{"token": "corp.", "count": 1}, {"token": "jurisdiction", "count": 1}]}}, "plumber": {"pawls": {"n_oracle": 27, "matched": 27, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 27, "matched": 25, "forgiven": 0, "missing_material": 2, "missing_trivial": 0, "recall_strict": 0.925926, "recall_effective": 0.925926, "missing_samples": [{"token": "corp.", "count": 1}, {"token": "jurisdiction", "count": 1}]}}}} +{"doc": "tests/fixtures/s1/quantinuum_s1__exhibit997sx1_bead0b.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 132, "matched": 132, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 132, "matched": 132, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 132, "matched": 132, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 132, "matched": 132, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/swarmer_s1__ex211_b96bf3.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 12, "matched": 12, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 12, "matched": 12, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 12, "matched": 12, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 12, "matched": 12, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/xenergy_s1__ex232_ebd77e.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 64, "matched": 64, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 64, "matched": 64, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 64, "matched": 64, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 64, "matched": 64, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/s1/yesway_s1__ex231_598e22.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 70, "matched": 70, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 70, "matched": 70, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 70, "matched": 70, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 70, "matched": 70, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/with_images.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 66, "matched": 66, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 66, "matched": 66, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 66, "matched": 66, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 66, "matched": 66, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} +{"doc": "tests/fixtures/needs_ocr.pdf", "pages": 1, "oracles": {"pdfium": {"pawls": {"n_oracle": 0, "matched": 0, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 0, "matched": 0, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}, "plumber": {"pawls": {"n_oracle": 0, "matched": 0, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "flagged_pages": []}, "content": {"n_oracle": 0, "matched": 0, "forgiven": 0, "missing_material": 0, "missing_trivial": 0, "recall_strict": 1.0, "recall_effective": 1.0, "missing_samples": []}}}} diff --git a/tests/test_text_faithfulness.py b/tests/test_text_faithfulness.py new file mode 100644 index 0000000..f327bf2 --- /dev/null +++ b/tests/test_text_faithfulness.py @@ -0,0 +1,57 @@ +"""Text-faithfulness regression: Warp must never LOSE text. + +Locks in the result of the exhaustive corpus evaluation +(``docs/text_faithfulness_eval.md``): on the OpenContracts PAWLS token surface +(the export's designed no-loss layer), effective token recall against the +independent PDFium oracle stays >= 0.999 on the canonical text fixtures. + +"Effective" recall forgives re-tokenization (different word segmentation of +the same characters) but counts real absences — see +``scripts/text_faithfulness_eval.py`` for the scoring definition. +""" + +import importlib.util +import os +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[1] +FIX_DIR = REPO / "tests" / "fixtures" + +_spec = importlib.util.spec_from_file_location( + "text_faithfulness_eval", REPO / "scripts" / "text_faithfulness_eval.py" +) +tfe = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(tfe) + +TEXT_DOCS = [ + ("USC Title 1 - CHAPTER 1.pdf", 0.999), + ( + "EtonPharmaceuticalsInc_20191114_10-Q_EX-10.1_11893941_" + "EX-10.1_Development_Agreement_ZrZJLLv.pdf", + 0.999, + ), +] + + +@pytest.mark.parametrize("filename,floor", TEXT_DOCS) +def test_pawls_effective_recall_vs_pdfium(filename, floor): + rec = tfe.eval_doc(str(FIX_DIR / filename), ["pdfium"]) + assert "error" not in rec, rec.get("error") + pawls = rec["oracles"]["pdfium"]["pawls"] + assert pawls["n_oracle"] > 1000, "oracle extracted implausibly little text" + assert pawls["recall_effective"] >= floor, ( + f"{filename}: PAWLS effective recall {pawls['recall_effective']} " + f"< {floor}; missing pages: {pawls['flagged_pages'][:3]}" + ) + + +def test_scanned_doc_does_not_regress_text_layer(): + """needs_ocr.pdf has no embedded text layer, so the oracle sees ~nothing; + the eval must degrade to a trivial pass rather than crash, and Warp's OCR + should still produce a substantial token stream.""" + rec = tfe.eval_doc(str(FIX_DIR / "needs_ocr.pdf"), ["pdfium"]) + assert "error" not in rec, rec.get("error") + pawls = rec["oracles"]["pdfium"]["pawls"] + assert pawls["recall_effective"] >= 0.999