From 76b0023b6bee2031ea20acf25899c86306c8d99e Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Tue, 16 Jun 2026 21:52:16 +0530 Subject: [PATCH 1/2] Add alpha layout confidence policy Signed-off-by: docushell-admin --- crates/ethos-layout/src/lib.rs | 66 ++++++++++++-- docs/execution-status.md | 8 +- fixtures/evaluate_layout_alpha.py | 120 +++++++++++++++++++++++++ fixtures/test_evaluate_layout_alpha.py | 58 +++++++++++- 4 files changed, 238 insertions(+), 14 deletions(-) diff --git a/crates/ethos-layout/src/lib.rs b/crates/ethos-layout/src/lib.rs index 1f7e30d8..013a4305 100644 --- a/crates/ethos-layout/src/lib.rs +++ b/crates/ethos-layout/src/lib.rs @@ -21,10 +21,11 @@ //! emitted as grounded headings, and multi-column text is ordered by column before //! vertical position. +use ethos_core::codes::WarningCode; use ethos_core::error::EthosError; use ethos_core::geom::QRect; -use ethos_core::ids::element_id; -use ethos_core::model::{Element, ElementType, Span}; +use ethos_core::ids::{element_id, warning_id}; +use ethos_core::model::{Element, ElementType, Span, Warning}; use ethos_core::traits::{Extraction, LayoutEngine, LayoutOutput}; /// Deterministic paragraph-level layout engine. @@ -37,12 +38,16 @@ const HEADING_MAX_WORDS: usize = 12; const HEADING_CONFIDENCE_LARGER_AND_BOLD: u16 = 900; const HEADING_CONFIDENCE_LARGER: u16 = 850; const HEADING_CONFIDENCE_BOLD_SAME_SIZE: u16 = 720; +const LAYOUT_CONFIDENCE_WARNING_THRESHOLD: u16 = 800; +const LOW_CONFIDENCE_READING_ORDER_MESSAGE: &str = "layout confidence below alpha threshold"; const MAX_HEADING_LEVEL: u8 = 3; impl LayoutEngine for BasicLayoutEngine { fn layout(&self, extraction: &Extraction) -> Result { let mut elements = Vec::new(); + let mut warnings = Vec::new(); let mut next_element = 1u32; + let mut next_warning = 1u32; for page in &extraction.pages { let page_spans: Vec> = extraction @@ -70,15 +75,14 @@ impl LayoutEngine for BasicLayoutEngine { &heading_sizes, &page.id, &mut next_element, + &mut next_warning, &mut elements, + &mut warnings, )?; } } - Ok(LayoutOutput { - elements, - warnings: Vec::new(), - }) + Ok(LayoutOutput { elements, warnings }) } } @@ -271,7 +275,9 @@ fn layout_column_lines<'a>( heading_sizes: &[i64], page_id: &str, next_element: &mut u32, + next_warning: &mut u32, elements: &mut Vec, + warnings: &mut Vec, ) -> Result<(), EthosError> { lines.sort_by(line_order); let mut text_lines = Vec::new(); @@ -316,14 +322,16 @@ fn layout_column_lines<'a>( heading_lines.push(next_line); } let heading_spans = lines_spans(&heading_lines); - elements.push(build_element( + let mut element = build_element( *next_element, page_id, &heading_spans, ElementType::Heading, Some(level), Some(confidence), - )?); + )?; + apply_confidence_policy(&mut element, next_warning, warnings)?; + elements.push(element); *next_element += 1; } else { text_lines.push(line); @@ -357,6 +365,31 @@ fn flush_text_lines<'a>( Ok(()) } +fn apply_confidence_policy( + element: &mut Element, + next_warning: &mut u32, + warnings: &mut Vec, +) -> Result<(), EthosError> { + if element + .confidence + .is_some_and(|confidence| confidence < LAYOUT_CONFIDENCE_WARNING_THRESHOLD) + { + let id = warning_id(*next_warning)?; + *next_warning += 1; + element.warning_refs.push(id.clone()); + warnings.push(Warning { + id, + code: WarningCode::LowConfidenceReadingOrder, + message: LOW_CONFIDENCE_READING_ORDER_MESSAGE.to_string(), + page: Some(element.page.clone()), + element_ref: Some(element.id.clone()), + span_ref: None, + region_ref: None, + }); + } + Ok(()) +} + fn is_list_item_line(line: &Line<'_>) -> bool { let text = line_text(line); let trimmed = text.trim_start(); @@ -865,10 +898,12 @@ mod tests { assert_eq!(output.elements[0].element_type, ElementType::Heading); assert_eq!(output.elements[0].heading_level, Some(1)); assert_eq!(output.elements[0].confidence, Some(850)); + assert!(output.elements[0].warning_refs.is_empty()); assert_eq!(output.elements[0].text.as_deref(), Some("Overview")); assert_eq!(output.elements[0].span_refs, vec!["s000001".to_string()]); assert_eq!(output.elements[1].element_type, ElementType::TextBlock); assert_eq!(output.elements[1].text.as_deref(), Some("Body text")); + assert!(output.warnings.is_empty()); } #[test] @@ -902,8 +937,23 @@ mod tests { assert_eq!(output.elements[0].element_type, ElementType::Heading); assert_eq!(output.elements[0].heading_level, Some(1)); assert_eq!(output.elements[0].confidence, Some(720)); + assert_eq!(output.elements[0].warning_refs, vec!["w0001".to_string()]); assert_eq!(output.elements[0].text.as_deref(), Some("Background")); assert_eq!(output.elements[1].text.as_deref(), Some("Plain body")); + assert_eq!(output.warnings.len(), 1); + assert_eq!(output.warnings[0].id, "w0001"); + assert_eq!( + output.warnings[0].code, + WarningCode::LowConfidenceReadingOrder + ); + assert_eq!( + output.warnings[0].message, + LOW_CONFIDENCE_READING_ORDER_MESSAGE + ); + assert_eq!(output.warnings[0].page.as_deref(), Some("p0001")); + assert_eq!(output.warnings[0].element_ref.as_deref(), Some("e000001")); + assert_eq!(output.warnings[0].span_ref, None); + assert_eq!(output.warnings[0].region_ref, None); } #[test] diff --git a/docs/execution-status.md b/docs/execution-status.md index 52ead91b..cf470dcf 100644 --- a/docs/execution-status.md +++ b/docs/execution-status.md @@ -15,8 +15,8 @@ The committed implementation now includes: - Runtime checks that reject missing or mismatched PDFium versions, release artifacts, and extracted libraries with stable errors before dynamic loading. - The determinism workflow includes a Windows x64 preflight lane for core c14n/profile/fingerprint contract tests, while PDFium-backed corpus work remains explicitly skipped unless the pinned runtime is configured on that runner. A static workflow test guards that matrix wiring. - `ethos doc parse` / `ethos fingerprint` PDF execution through a worker process with `max_parse_ms` timeout enforcement, stable error-envelope relay, diagnostics-gated worker stderr, and page-range validation/filtering. -- Quantized page/span extraction at the backend boundary, plus a basic deterministic layout pass that assembles paragraph `text_block` elements, fixture-backed alpha heading and flat list-item elements, and simple column reading order for the current born-digital fixtures. Fixture validation binds selected `fixture.json` expectations to committed extraction/layout goldens and binds current alpha text/Markdown exports to committed layout output so current read-order, element-type, heading-export, list-item, and export cases fail closed on drift. -- An internal layout evaluator scaffold exists at `fixtures/evaluate_layout_alpha.py` and `make layout-evaluator-alpha`. It reads committed `fixture.json` and `layout.json` files, summarizes alpha element-type and subset coverage, and fails closed on missing layout expectations or drift in fixture-backed reading order / heading / list-item cases. +- Quantized page/span extraction at the backend boundary, plus a basic deterministic layout pass that assembles paragraph `text_block` elements, fixture-backed alpha heading and flat list-item elements, and simple column reading order for the current born-digital fixtures. Current alpha layout confidence is explicit for heading signals, and below-threshold layout confidence emits deterministic `low_confidence_reading_order` diagnostics instead of staying silent. Fixture validation binds selected `fixture.json` expectations to committed extraction/layout goldens and binds current alpha text/Markdown exports to committed layout output so current read-order, element-type, heading-export, list-item, and export cases fail closed on drift. +- An internal layout evaluator scaffold exists at `fixtures/evaluate_layout_alpha.py` and `make layout-evaluator-alpha`. It reads committed `fixture.json` and `layout.json` files, summarizes alpha element-type and subset coverage, and fails closed on missing layout expectations, confidence-policy drift, or drift in fixture-backed reading order / heading / list-item cases. - Schema/example/profile validation is green through `schemas/validate_examples.py` using `jsonschema` draft 2020-12 validation, including the crop descriptor artifact contract plus referential-integrity and bbox sanity checks outside JSON Schema. - `ethos verify` now produces non-empty quote, value, presence, and table-cell verification checks over native Ethos document JSON and synthetic OpenDataLoader-style JSON through `--grounding opendataloader-json`; it also verifies quote/value/presence citations over pinned real OpenDataLoader 2.4.7 JSON, including grounded and ungrounded cases. Citation/config inputs are rejected when they drift outside the closed schemas. The public demo harness covers grounded, ungrounded, split-quote, not-found, stale-fingerprint, unsupported non-v1 claim, capability-limited, malformed-citation, malformed OpenDataLoader-style input, and summary-format reject paths. - Verification semantics are now trust-honest at alpha scope: quote containment is explicitly labeled, value/table-cell checks require normalized equality, fingerprint-pinned citations fail closed when source fingerprints are unavailable, and structured capability limits explain why a run is downgraded. @@ -52,8 +52,8 @@ Milestone A has an accepted internal Gate Zero decision for roadmap control, so | PDFium Phase 1 profile | Landed: pinned profile, V8/XFA-disabled state, platform hashes, runtime library hashes, and provenance are recorded | Phase 2 project-maintained builds still block Public Beta | | PDFium loader/runtime checks | Landed: missing/mismatched version, artifact, and runtime library hashes fail deterministically | Release packaging and operator setup path still need hardening | | Real PDF backend | Landed for simple born-digital PDFs: page count, quantized spans, worker execution, timeout, page filtering, and fingerprint path exist | Wider corpus coverage, failure fixtures, memory-limit behavior, quirk log, and Gate Zero run are still missing | -| Layout groundwork | Landed: basic paragraph text blocks, fixture-backed alpha heading and flat list-item elements, simple column reading order over quantized spans, fixture metadata checks against committed extraction/layout goldens for current read-order and element-type expectations, and alpha text/Markdown export goldens derived from committed layout output | Tables, nested/richer list and heading semantics, rotation/quirk handling, and confidence policy remain future work | -| Layout evaluator scaffold | Landed: deterministic internal evaluator over committed layout fixture expectations, with heading/list/reading-order coverage checks, expectation drift diagnostics, report JSON, Make target, and unit coverage | Broader evaluator dimensions and CI matrix integration remain future work | +| Layout groundwork | Landed: basic paragraph text blocks, fixture-backed alpha heading and flat list-item elements, simple column reading order over quantized spans, explicit alpha heading-confidence values, deterministic below-threshold confidence diagnostics, fixture metadata checks against committed extraction/layout goldens for current read-order and element-type expectations, and alpha text/Markdown export goldens derived from committed layout output | Tables, nested/richer list and heading semantics, rotation/quirk handling, and broader confidence dimensions remain future work | +| Layout evaluator scaffold | Landed: deterministic internal evaluator over committed layout fixture expectations, with heading/list/reading-order coverage checks, confidence-policy checks, expectation drift diagnostics, report JSON, Make target, and unit coverage | Broader evaluator dimensions and CI matrix integration remain future work | | Python surface scaffold | Landed: internal stdlib wrapper over a caller-provided local `ethos doc parse` command, with explicit JSON/Markdown/text methods, page selection passthrough, diagnostics passthrough, timeout handling, command failure reporting, and mocked-command unit coverage | Native binding work, broader API design, and public setup path remain future work | | Font policy groundwork | Partially landed: substitution table and profile policy are present; fixture output uses deterministic substitution IDs | Bundled fallback asset hashing and broader font/CID validation remain open | | Schema/example validation | Landed: schemas, examples, deterministic profile, referential integrity, and bbox sanity pass the `jsonschema` validation gate | Contract changes still require explicit versioning and compatibility review | diff --git a/fixtures/evaluate_layout_alpha.py b/fixtures/evaluate_layout_alpha.py index dc824d06..00af91ba 100644 --- a/fixtures/evaluate_layout_alpha.py +++ b/fixtures/evaluate_layout_alpha.py @@ -34,6 +34,8 @@ ROOT = Path(__file__).resolve().parent REQUIRED_EXPECTATION_FIELDS = ("expected_text", "expected_element_types") +ALPHA_LAYOUT_CONFIDENCE_WARNING_THRESHOLD = 800 +LOW_CONFIDENCE_READING_ORDER_CODE = "low_confidence_reading_order" COVERAGE_GATES = { "heading_fixture": { "subset": "headings", @@ -226,6 +228,17 @@ def evaluate_fixture( ) ) return None + warnings = layout.get("warnings", []) + if not isinstance(warnings, list): + diagnostics.append( + diagnostic( + "invalid_layout", + fixture_id, + "layout.json warnings must be an array", + f"{fixture_rel}/layout.json", + ) + ) + warnings = [] element_text = [] element_types = [] @@ -281,6 +294,13 @@ def evaluate_fixture( len(elements), diagnostics, ) + confidence_policy_status = compare_confidence_policy( + fixture_id, + fixture_rel, + elements, + warnings, + diagnostics, + ) subset_status = compare_subset_expectations( fixture_id, fixture_rel, @@ -299,6 +319,7 @@ def evaluate_fixture( "expected_text": expected_text_status, "expected_element_types": expected_element_types_status, "expected_elements": expected_elements_status, + "confidence_policy": confidence_policy_status, "subset_expectations": subset_status, } @@ -417,6 +438,105 @@ def compare_expected_elements( return "pass" +def compare_confidence_policy( + fixture_id: str, + fixture_rel: str, + elements: List[Any], + warnings: List[Any], + diagnostics: List[Dict[str, Any]], +) -> str: + invalid = False + checked = 0 + warning_by_id: Dict[str, Dict[str, Any]] = {} + + for warning_index, warning in enumerate(warnings): + if not isinstance(warning, dict): + diagnostics.append( + diagnostic( + "invalid_layout", + fixture_id, + f"layout warning {warning_index} must be an object", + f"{fixture_rel}/layout.json", + ) + ) + invalid = True + continue + warning_id = warning.get("id") + if isinstance(warning_id, str): + warning_by_id[warning_id] = warning + + for element_index, element in enumerate(elements): + if not isinstance(element, dict): + continue + confidence = element.get("confidence") + if confidence is None: + continue + checked += 1 + if not isinstance(confidence, int) or not 0 <= confidence <= 1000: + diagnostics.append( + diagnostic( + "invalid_layout", + fixture_id, + f"layout element {element_index} confidence must be an integer 0..1000", + f"{fixture_rel}/layout.json", + ) + ) + invalid = True + continue + if confidence >= ALPHA_LAYOUT_CONFIDENCE_WARNING_THRESHOLD: + continue + + warning_refs = element.get("warning_refs", []) + if not isinstance(warning_refs, list) or not all( + isinstance(item, str) for item in warning_refs + ): + diagnostics.append( + diagnostic( + "invalid_layout", + fixture_id, + f"layout element {element_index} warning_refs must be a string array", + f"{fixture_rel}/layout.json", + ) + ) + invalid = True + continue + + element_id = element.get("id") + matched_warning = None + for warning_ref in warning_refs: + warning = warning_by_id.get(warning_ref) + if not isinstance(warning, dict): + continue + if ( + warning.get("code") == LOW_CONFIDENCE_READING_ORDER_CODE + and warning.get("element_ref") == element_id + ): + matched_warning = warning + break + if matched_warning is None: + diagnostics.append( + diagnostic( + "confidence_policy_mismatch", + fixture_id, + "layout element below alpha confidence threshold must reference " + "a matching low_confidence_reading_order warning", + f"{fixture_rel}/layout.json", + expected={ + "code": LOW_CONFIDENCE_READING_ORDER_CODE, + "element_ref": element_id, + }, + actual={ + "confidence": confidence, + "warning_refs": warning_refs, + }, + ) + ) + + if invalid: + return "invalid" + return "pass" if checked else "not_applicable" + + def compare_subset_expectations( fixture_id: str, fixture_rel: str, diff --git a/fixtures/test_evaluate_layout_alpha.py b/fixtures/test_evaluate_layout_alpha.py index bbb657de..49796c8e 100644 --- a/fixtures/test_evaluate_layout_alpha.py +++ b/fixtures/test_evaluate_layout_alpha.py @@ -52,6 +52,10 @@ def test_passing_fixture_set_reports_counts_and_coverage(self) -> None: "multi_column_reading_order_fixture": ["column-case"], }, ) + heading_check = next( + check for check in report["checks"] if check["fixture_id"] == "heading-case" + ) + self.assertEqual(heading_check["confidence_policy"], "pass") self.assertEqual(report["diagnostics"], []) def test_missing_expected_text_fails_closed(self) -> None: @@ -121,6 +125,37 @@ def test_multi_column_fixture_requires_multi_element_expected_text(self) -> None self.assertDiagnostic(report, "subset_expectation_mismatch", "column-case") self.assertDiagnostic(report, "missing_coverage", None) + def test_low_confidence_element_requires_matching_warning(self) -> None: + self.write_required_alpha_fixture_set() + layout_path = self.root / "synthetic/heading-case/layout.json" + layout = json.loads(layout_path.read_text(encoding="utf-8")) + layout["elements"][0].pop("warning_refs") + layout["warnings"] = [] + self.write_json(layout_path, layout) + + report = evaluate_layout_alpha(self.root) + + self.assertEqual(report["status"], "fail") + diagnostic = self.onlyDiagnostic( + report, + "confidence_policy_mismatch", + "heading-case", + ) + self.assertEqual( + diagnostic["expected"], + { + "code": "low_confidence_reading_order", + "element_ref": "e000001", + }, + ) + self.assertEqual( + diagnostic["actual"], + { + "confidence": 720, + "warning_refs": [], + }, + ) + def test_missing_layout_file_reports_missing_file(self) -> None: self.write_required_alpha_fixture_set() (self.root / "synthetic/list-case/layout.json").unlink() @@ -176,9 +211,24 @@ def write_required_alpha_fixture_set(self) -> None: expected_text=["Alpha Heading", "Body text"], expected_element_types=["heading", "text_block"], elements=[ - {"id": "e000001", "type": "heading", "text": "Alpha Heading"}, + { + "id": "e000001", + "type": "heading", + "text": "Alpha Heading", + "confidence": 720, + "warning_refs": ["w0001"], + }, {"id": "e000002", "type": "text_block", "text": "Body text"}, ], + warnings=[ + { + "id": "w0001", + "code": "low_confidence_reading_order", + "message": "layout confidence below alpha threshold", + "page": "p0001", + "element_ref": "e000001", + } + ], ), self.write_fixture( fixture_id="list-case", @@ -227,6 +277,7 @@ def write_fixture( expected_text, expected_element_types: list[str], elements: list[dict], + warnings: list[dict] | None = None, ): fixture_dir = (self.root / fixture_path).parent fixture_dir.mkdir(parents=True, exist_ok=True) @@ -240,7 +291,10 @@ def write_fixture( "expected_elements": len(elements), }, ) - self.write_json(fixture_dir / "layout.json", {"elements": elements, "warnings": []}) + self.write_json( + fixture_dir / "layout.json", + {"elements": elements, "warnings": warnings or []}, + ) return { "id": fixture_id, "file": fixture_path, From 6384fa7474c2b46c65384ecf0e80708c83d0110e Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Tue, 16 Jun 2026 21:57:08 +0530 Subject: [PATCH 2/2] Fix layout confidence clippy lint Signed-off-by: docushell-admin --- crates/ethos-layout/src/lib.rs | 68 +++++++++++++++++----------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/crates/ethos-layout/src/lib.rs b/crates/ethos-layout/src/lib.rs index 013a4305..40c1d2a2 100644 --- a/crates/ethos-layout/src/lib.rs +++ b/crates/ethos-layout/src/lib.rs @@ -69,16 +69,14 @@ impl LayoutEngine for BasicLayoutEngine { let heading_sizes = heading_size_levels(&columns, body_font_size_q); for column in columns { - layout_column_lines( - column.lines, - body_font_size_q, - &heading_sizes, - &page.id, - &mut next_element, - &mut next_warning, - &mut elements, - &mut warnings, - )?; + let mut sink = LayoutColumnSink { + page_id: &page.id, + next_element: &mut next_element, + next_warning: &mut next_warning, + elements: &mut elements, + warnings: &mut warnings, + }; + layout_column_lines(column.lines, body_font_size_q, &heading_sizes, &mut sink)?; } } @@ -110,6 +108,14 @@ struct Paragraph<'a> { bbox: QRect, } +struct LayoutColumnSink<'a> { + page_id: &'a str, + next_element: &'a mut u32, + next_warning: &'a mut u32, + elements: &'a mut Vec, + warnings: &'a mut Vec, +} + #[derive(Clone, Copy)] struct HeadingSignal { size_q: i64, @@ -273,11 +279,7 @@ fn layout_column_lines<'a>( mut lines: Vec>, body_font_size_q: Option, heading_sizes: &[i64], - page_id: &str, - next_element: &mut u32, - next_warning: &mut u32, - elements: &mut Vec, - warnings: &mut Vec, + sink: &mut LayoutColumnSink<'_>, ) -> Result<(), EthosError> { lines.sort_by(line_order); let mut text_lines = Vec::new(); @@ -285,19 +287,19 @@ fn layout_column_lines<'a>( while let Some(line) = line_iter.next() { if is_list_item_line(&line) { - flush_text_lines(&mut text_lines, page_id, next_element, elements)?; + flush_text_lines(&mut text_lines, sink)?; let list_item_spans = line_spans(&line); - elements.push(build_element( - *next_element, - page_id, + sink.elements.push(build_element( + *sink.next_element, + sink.page_id, &list_item_spans, ElementType::ListItem, None, None, )?); - *next_element += 1; + *sink.next_element += 1; } else if let Some(signal) = heading_signal(&line, body_font_size_q) { - flush_text_lines(&mut text_lines, page_id, next_element, elements)?; + flush_text_lines(&mut text_lines, sink)?; let level = heading_level(signal.size_q, heading_sizes); let mut confidence = signal.confidence; let mut heading_lines = vec![line]; @@ -323,44 +325,42 @@ fn layout_column_lines<'a>( } let heading_spans = lines_spans(&heading_lines); let mut element = build_element( - *next_element, - page_id, + *sink.next_element, + sink.page_id, &heading_spans, ElementType::Heading, Some(level), Some(confidence), )?; - apply_confidence_policy(&mut element, next_warning, warnings)?; - elements.push(element); - *next_element += 1; + apply_confidence_policy(&mut element, sink.next_warning, sink.warnings)?; + sink.elements.push(element); + *sink.next_element += 1; } else { text_lines.push(line); } } - flush_text_lines(&mut text_lines, page_id, next_element, elements) + flush_text_lines(&mut text_lines, sink) } fn flush_text_lines<'a>( text_lines: &mut Vec>, - page_id: &str, - next_element: &mut u32, - elements: &mut Vec, + sink: &mut LayoutColumnSink<'_>, ) -> Result<(), EthosError> { if text_lines.is_empty() { return Ok(()); } for paragraph in group_paragraphs(std::mem::take(text_lines)) { let paragraph_spans = paragraph_spans(¶graph); - elements.push(build_element( - *next_element, - page_id, + sink.elements.push(build_element( + *sink.next_element, + sink.page_id, ¶graph_spans, ElementType::TextBlock, None, None, )?); - *next_element += 1; + *sink.next_element += 1; } Ok(()) }