From ea8ae6a30b4792aa32a6163e971733426330bab3 Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Tue, 21 Jul 2026 12:59:33 +0530 Subject: [PATCH 01/30] release: reconcile v0.4.0 and scope v0.5.0 Record the published v0.4.0 tag, assets, hashes, and package baseline; close its release preparation and correct the changelog boundary. Accept ADR-0015 and establish the bounded v0.5.0 implementation, consumer, determinism, performance, and freeze gates. Signed-off-by: docushell-admin --- .github/scripts/test_execution_status.py | 16 ++--- CHANGELOG.md | 5 ++ ...DR-0015-opt-in-bundled-pdfium-artifacts.md | 8 +-- docs/execution-status.md | 7 +- docs/public-release-checklist.md | 4 +- docs/release-state.json | 44 ++++++------ docs/releases/v0.4.0.md | 12 ++-- docs/v0-4-0-release-prep.md | 12 ++-- docs/v0-5-0-release-prep.md | 67 +++++++++++++++++++ .../v0-4-0-release-closeout-summary.md | 22 ++++++ 10 files changed, 146 insertions(+), 51 deletions(-) create mode 100644 docs/v0-5-0-release-prep.md create mode 100644 docs/validation/v0-4-0-release-closeout-summary.md diff --git a/.github/scripts/test_execution_status.py b/.github/scripts/test_execution_status.py index f697ca1a..26c92ca8 100644 --- a/.github/scripts/test_execution_status.py +++ b/.github/scripts/test_execution_status.py @@ -34,18 +34,18 @@ def test_status_is_scoped_to_internal_continuation(self) -> None: text = status_text() self.assertIn( - "Status: v0.3.0 Rust library crates `ethos-doc-core`, `ethos-verify`, and `ethos-pdf` " + "Status: v0.4.0 Rust library crates `ethos-doc-core`, `ethos-verify`, and `ethos-pdf` " "are live on crates.io, and the Python `ethos-pdf` wheel is live on PyPI.", text, ) self.assertIn( - "The exact v0.3.0 public install wording packet is approved and closed out", + "The exact v0.4.0 public install wording packet is approved and closed out", text, ) self.assertIn("docs/validation/v0-3-0-public-install-wording-approval-decision-validation-2026-07-02.md", text) self.assertIn("docs/validation/v0-3-0-public-install-wording-closeout-validation-2026-07-02.md", text) self.assertIn("GitHub Release `v0.3.0`", text) - self.assertIn("npm `@docushell/ethos-pdf@0.3.0` is live on npm", text) + self.assertIn("npm `@docushell/ethos-pdf@0.4.0` is live on npm", text) self.assertIn("v0.3.0 npm publication closeout", text) self.assertIn("DocuShell integration remain blocked", text) self.assertIn("Internal Milestone D source-only closeout remains complete", text) @@ -106,15 +106,15 @@ def test_public_posture_boundary_remains_explicit(self) -> None: text, ) self.assertIn( - "v0.3.0 Rust library crates `ethos-doc-core`, `ethos-verify`, and `ethos-pdf` are live on crates.io", + "v0.4.0 Rust library crates `ethos-doc-core`, `ethos-verify`, and `ethos-pdf` are live on crates.io", text, ) self.assertIn("the Python `ethos-pdf` wheel is live on PyPI", text) - self.assertIn("npm `@docushell/ethos-pdf@0.3.0` is live on npm", text) - self.assertIn("The exact v0.3.0 public install wording packet is approved and closed out", text) - self.assertIn("GitHub Release `v0.3.0`", text) + self.assertIn("npm `@docushell/ethos-pdf@0.4.0` is live on npm", text) + self.assertIn("The exact v0.4.0 public install wording packet is approved and closed out", text) + self.assertIn("GitHub Release `v0.4.0`", text) self.assertIn("macOS arm64/Linux x64 CLI artifacts", text) - self.assertIn("`@docushell/ethos-pdf@0.3.0`", text) + self.assertIn("`@docushell/ethos-pdf@0.4.0`", text) self.assertIn("docs/validation/v0-3-0-publication-closeout-validation-2026-07-01.md", text) self.assertIn("ethos-doc-core", text) self.assertIn("ethos-verify", text) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e8abf94..31927c8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- release: reconcile the published v0.4.0 baseline, accept the bounded v0.5.0 + `ethos-full` exception, and establish the v0.5.0 four-deliverable release scope. + +## 0.4.0 - 2026-07-21 + - boundary-exception: refresh the v0.4.0 npm vendor payload from the final stable, byte-identical macOS arm64/Linux x64 CLI archives after removing the manifest checksum self-reference. - release: remove the CLI's compile-time npm vendor-manifest inclusion so the final CLI artifact diff --git a/docs/decisions/ADR-0015-opt-in-bundled-pdfium-artifacts.md b/docs/decisions/ADR-0015-opt-in-bundled-pdfium-artifacts.md index 73904226..23247052 100644 --- a/docs/decisions/ADR-0015-opt-in-bundled-pdfium-artifacts.md +++ b/docs/decisions/ADR-0015-opt-in-bundled-pdfium-artifacts.md @@ -1,6 +1,6 @@ # ADR-0015: Opt-In Bundled PDFium Artifacts -- Status: Proposed +- Status: Accepted - Date: 2026-07-20 - Decider: Product / Gate Zero decider - Governs: optional `ethos-full` macOS arm64 and Linux x64 archives; ADR-0002 and ADR-0013 @@ -24,9 +24,9 @@ Phase 1 runtime with its complete license material. This proposal recommends the standalone `ethos-full` class only. It does not make that recommendation active while this ADR is Proposed. -## Proposed Decision +## Decision -If accepted, this ADR amends ADR-0013 only as follows: +This accepted ADR amends ADR-0013 only as follows: 1. Add optional `ethos-full--macos-arm64.tar.gz` and `ethos-full--linux-x64.tar.gz` standalone archives. Base crates, wheels, npm packages, @@ -46,7 +46,7 @@ If accepted, this ADR amends ADR-0013 only as follows: 5. Require target-platform smoke before publication: `ethos --version`, `ethos doctor --require-pdfium`, and two byte-identical parses of the pinned license-clean fixture. A build on one platform is not evidence for the other. -6. Treat every candidate as non-publishable until this ADR is Accepted and the v0.4.0 release +6. Treat every candidate as non-publishable until this ADR is Accepted and the v0.5.0 release lane explicitly includes the artifact. Preparing candidates, checksums, and evidence does not authorize uploads, tags, release edits, or public wording. 7. Revisit Phase 2 independently. Acceptance of this narrow Phase 1 redistribution exception diff --git a/docs/execution-status.md b/docs/execution-status.md index 92146443..17e29459 100644 --- a/docs/execution-status.md +++ b/docs/execution-status.md @@ -3,9 +3,9 @@ Date: 2026-07-20 Owner: product / decider -Status: v0.3.0 Rust library crates `ethos-doc-core`, `ethos-verify`, and `ethos-pdf` are live on crates.io, and the Python `ethos-pdf` wheel is live on PyPI. Its released version is `0.3.0`. GitHub Release `v0.3.0` is marked as the repository's latest release and contains closed-out macOS arm64/Linux x64 CLI artifacts for evaluation with caller-provided PDFium through `ETHOS_PDFIUM_LIBRARY_PATH`. npm `@docushell/ethos-pdf@0.3.0` is live on npm. The exact v0.3.0 public install wording packet is approved and closed out. Package-tag creation for `ethos-package-ethos-doc-core-0.3.0`, `ethos-package-ethos-verify-0.3.0`, and `ethos-package-ethos-pdf-0.3.0` is closed out, and the existing release tag is closed out. +Status: v0.4.0 Rust library crates `ethos-doc-core`, `ethos-verify`, and `ethos-pdf` are live on crates.io, and the Python `ethos-pdf` wheel is live on PyPI. Its released version is `0.4.0`. GitHub Release `v0.4.0` is marked as the repository's latest release and contains closed-out macOS arm64/Linux x64 CLI artifacts for evaluation with caller-provided PDFium through `ETHOS_PDFIUM_LIBRARY_PATH`. npm `@docushell/ethos-pdf@0.4.0` is live on npm. The exact v0.4.0 public install wording packet is approved and closed out. Package-tag creation for `ethos-package-ethos-doc-core-0.4.0`, `ethos-package-ethos-verify-0.4.0`, and `ethos-package-ethos-pdf-0.4.0` is closed out, and the existing release tag is closed out. -Current closeout records: [rust python publication](validation/v0-3-0-release-closeout-summary.md); [github release artifacts](validation/v0-3-0-release-closeout-summary.md); [npm publication](validation/v0-3-0-release-closeout-summary.md); [public install wording](validation/v0-3-0-release-closeout-summary.md); [package tags](validation/v0-3-0-release-closeout-summary.md); [release tag](validation/v0-3-0-release-closeout-summary.md); [release metadata](validation/v0-3-0-release-closeout-summary.md). +Current closeout records: [rust python publication](validation/v0-4-0-release-closeout-summary.md); [github release artifacts](validation/v0-4-0-release-closeout-summary.md); [npm publication](validation/v0-4-0-release-closeout-summary.md); [public install wording](validation/v0-4-0-release-closeout-summary.md); [package tags](validation/v0-4-0-release-closeout-summary.md); [release tag](validation/v0-4-0-release-closeout-summary.md); [release metadata](validation/v0-4-0-release-closeout-summary.md). Still blocked: additional release tags or release targets, hosted surfaces, production positioning, Windows packaged artifacts, bundled project-maintained PDFium builds, public benchmark reports and claims, speed, footprint, parser-quality, and table-quality claims, ethos-doc, and ethos-rag. @@ -13,8 +13,7 @@ Still blocked: additional release tags or release targets, hosted surfaces, prod The generated block above is the sole current public-release authority in this historical ledger. Sections below preserve dated milestone and wording records for auditability; version statements inside them describe their recorded point in time and are not the current release state unless the -generated block explicitly repeats them. Candidate source metadata is currently `0.4.0`, while the -published public baseline remains `0.3.0`. +generated block explicitly repeats them. The published public baseline is `0.4.0`. The DocuShell first-consumer integration closeout is accepted in `docs/validation/nip-1-docushell-integration-closeout-2026-07-20.md`. The worker-only public CLI, diff --git a/docs/public-release-checklist.md b/docs/public-release-checklist.md index 92359ba5..4d6b0b82 100644 --- a/docs/public-release-checklist.md +++ b/docs/public-release-checklist.md @@ -6,9 +6,9 @@ or launch announcement. It is intentionally stricter than the day-to-day enginee ## Current Status -Status: v0.3.0 Rust library crates `ethos-doc-core`, `ethos-verify`, and `ethos-pdf` are live on crates.io, and the Python `ethos-pdf` wheel is live on PyPI. Its released version is `0.3.0`. GitHub Release `v0.3.0` is marked as the repository's latest release and contains closed-out macOS arm64/Linux x64 CLI artifacts for evaluation with caller-provided PDFium through `ETHOS_PDFIUM_LIBRARY_PATH`. npm `@docushell/ethos-pdf@0.3.0` is live on npm. The exact v0.3.0 public install wording packet is approved and closed out. Package-tag creation for `ethos-package-ethos-doc-core-0.3.0`, `ethos-package-ethos-verify-0.3.0`, and `ethos-package-ethos-pdf-0.3.0` is closed out, and the existing release tag is closed out. +Status: v0.4.0 Rust library crates `ethos-doc-core`, `ethos-verify`, and `ethos-pdf` are live on crates.io, and the Python `ethos-pdf` wheel is live on PyPI. Its released version is `0.4.0`. GitHub Release `v0.4.0` is marked as the repository's latest release and contains closed-out macOS arm64/Linux x64 CLI artifacts for evaluation with caller-provided PDFium through `ETHOS_PDFIUM_LIBRARY_PATH`. npm `@docushell/ethos-pdf@0.4.0` is live on npm. The exact v0.4.0 public install wording packet is approved and closed out. Package-tag creation for `ethos-package-ethos-doc-core-0.4.0`, `ethos-package-ethos-verify-0.4.0`, and `ethos-package-ethos-pdf-0.4.0` is closed out, and the existing release tag is closed out. -Current closeout records: [rust python publication](validation/v0-3-0-release-closeout-summary.md); [github release artifacts](validation/v0-3-0-release-closeout-summary.md); [npm publication](validation/v0-3-0-release-closeout-summary.md); [public install wording](validation/v0-3-0-release-closeout-summary.md); [package tags](validation/v0-3-0-release-closeout-summary.md); [release tag](validation/v0-3-0-release-closeout-summary.md); [release metadata](validation/v0-3-0-release-closeout-summary.md). +Current closeout records: [rust python publication](validation/v0-4-0-release-closeout-summary.md); [github release artifacts](validation/v0-4-0-release-closeout-summary.md); [npm publication](validation/v0-4-0-release-closeout-summary.md); [public install wording](validation/v0-4-0-release-closeout-summary.md); [package tags](validation/v0-4-0-release-closeout-summary.md); [release tag](validation/v0-4-0-release-closeout-summary.md); [release metadata](validation/v0-4-0-release-closeout-summary.md). Still blocked: additional release tags or release targets, hosted surfaces, production positioning, Windows packaged artifacts, bundled project-maintained PDFium builds, public benchmark reports and claims, speed, footprint, parser-quality, and table-quality claims, ethos-doc, and ethos-rag. diff --git a/docs/release-state.json b/docs/release-state.json index 3bb63a53..9e23eefb 100644 --- a/docs/release-state.json +++ b/docs/release-state.json @@ -1,8 +1,8 @@ { "schema_version": 2, - "as_of": "2026-07-03", + "as_of": "2026-07-21", "release": { - "version": "0.3.0", + "version": "0.4.0", "rust_crates": [ "ethos-doc-core", "ethos-verify", @@ -10,16 +10,16 @@ ], "python_package": { "name": "ethos-pdf", - "version": "0.3.0" + "version": "0.4.0" }, "npm_package": { "name": "@docushell/ethos-pdf", - "version": "0.3.0" + "version": "0.4.0" }, "github_release": { - "tag": "v0.3.0", - "version": "0.3.0", - "name": "Release v0.3.0", + "tag": "v0.4.0", + "version": "0.4.0", + "name": "Release v0.4.0", "latest": true, "notes": "docs/releases/v0.4.0.md", "platforms": [ @@ -27,31 +27,31 @@ "Linux x64" ], "assets": [ - "ethos-macos-arm64.tar.gz", - "ethos-macos-arm64.tar.gz.sha256", - "ethos-macos-arm64.inventory.json", - "ethos-macos-arm64.smoke.json", "ethos-linux-x64.tar.gz", "ethos-linux-x64.tar.gz.sha256", "ethos-linux-x64.inventory.json", - "ethos-linux-x64.smoke.json" + "ethos-linux-x64.smoke.json", + "ethos-macos-arm64.tar.gz", + "ethos-macos-arm64.tar.gz.sha256", + "ethos-macos-arm64.inventory.json", + "ethos-macos-arm64.smoke.json" ] }, "package_tags": [ - "ethos-package-ethos-doc-core-0.3.0", - "ethos-package-ethos-verify-0.3.0", - "ethos-package-ethos-pdf-0.3.0" + "ethos-package-ethos-doc-core-0.4.0", + "ethos-package-ethos-verify-0.4.0", + "ethos-package-ethos-pdf-0.4.0" ], "pdfium_environment": "ETHOS_PDFIUM_LIBRARY_PATH" }, "closed_lanes": { - "rust_python_publication": "docs/validation/v0-3-0-release-closeout-summary.md", - "github_release_artifacts": "docs/validation/v0-3-0-release-closeout-summary.md", - "npm_publication": "docs/validation/v0-3-0-release-closeout-summary.md", - "public_install_wording": "docs/validation/v0-3-0-release-closeout-summary.md", - "package_tags": "docs/validation/v0-3-0-release-closeout-summary.md", - "release_tag": "docs/validation/v0-3-0-release-closeout-summary.md", - "release_metadata": "docs/validation/v0-3-0-release-closeout-summary.md" + "rust_python_publication": "docs/validation/v0-4-0-release-closeout-summary.md", + "github_release_artifacts": "docs/validation/v0-4-0-release-closeout-summary.md", + "npm_publication": "docs/validation/v0-4-0-release-closeout-summary.md", + "public_install_wording": "docs/validation/v0-4-0-release-closeout-summary.md", + "package_tags": "docs/validation/v0-4-0-release-closeout-summary.md", + "release_tag": "docs/validation/v0-4-0-release-closeout-summary.md", + "release_metadata": "docs/validation/v0-4-0-release-closeout-summary.md" }, "blocked_lanes": [ "additional release tags or release targets", diff --git a/docs/releases/v0.4.0.md b/docs/releases/v0.4.0.md index 8c753087..c6718e58 100644 --- a/docs/releases/v0.4.0.md +++ b/docs/releases/v0.4.0.md @@ -1,8 +1,6 @@ # Ethos v0.4.0 -Public release wording approved by the decider on 2026-07-20. Publication remains pending, and -the final archive hashes must be added from a release workflow run bound to the final merged -`main` commit. +Public release wording was approved by the decider on 2026-07-20 and published on 2026-07-21. Ethos v0.4.0 is a public-beta evaluation release for deterministic document evidence grounding and citation checking across native Ethos JSON and supported foreign parser outputs. @@ -34,8 +32,12 @@ answer-release decisions. PDFium-backed commands require caller-provided PDFium through `ETHOS_PDFIUM_LIBRARY_PATH`. This release does not bundle a project-maintained PDFium build. -Archive SHA256 values will be inserted here from the final main-bound release workflow before -publication. +Published archive SHA256 values: + +- `ethos-macos-arm64.tar.gz`: + `719f092101bb1e4cfeebc9507155b15739642f0b891664bd0d9f8a05d9688ba5` +- `ethos-linux-x64.tar.gz`: + `616be562306d64a293554ca4695f19deb6e135dd328e88598a80e76f6f8fb3cd` Outside the v0.4.0 release boundary: Windows packaged artifacts, bundled/project-maintained PDFium builds, `ethos-mcp`, hosted surfaces, the WASM playground, native OCR or scanned-document diff --git a/docs/v0-4-0-release-prep.md b/docs/v0-4-0-release-prep.md index 04a0deff..745d9960 100644 --- a/docs/v0-4-0-release-prep.md +++ b/docs/v0-4-0-release-prep.md @@ -1,11 +1,11 @@ # Ethos v0.4.0 Release Preparation -Status: **merged release candidate — final publication preparation in progress.** Source/package -metadata is activated at `0.4.0`; PR #223 is merged at main commit `525f50a`, and post-merge CI -run `29729605736` passed. Final main-bound release artifacts, registry publication, tag/GitHub -Release actions, final artifact hashes, and closeout remain pending. The v0.4.0 release-note -wording is approved, while public installation wording remains on the actually published -`0.3.0` surfaces until publication. +Status: **closed.** GitHub Release `v0.4.0` was published on 2026-07-21 from source commit +`e73477e427d2384bdb3b6b913578411325d3d107` (annotated tag object +`6041b2ed5617f923e5d226f1f69bb184d70d5fce`). The final macOS arm64 and Linux x64 caller-provided +PDFium archives and their checksum, inventory, and smoke sidecars are recorded in +`docs/validation/v0-4-0-release-closeout-summary.md`. Windows and `ethos-full` were skipped; +neither is a published v0.4.0 surface. The release scope may be amended before tagging only through an explicitly scoped, reviewed PR. Any implementation change invalidates earlier final-artifact hashes and requires fresh full CI, diff --git a/docs/v0-5-0-release-prep.md b/docs/v0-5-0-release-prep.md new file mode 100644 index 00000000..3c6d1b71 --- /dev/null +++ b/docs/v0-5-0-release-prep.md @@ -0,0 +1,67 @@ +# Ethos v0.5.0 Release Preparation + +Status: **accepted implementation scope.** This document is the scoped decider request for the +four v0.5.0 deliverables. It authorizes implementation only; publication remains subject to the +freeze, target-smoke, claims, licence, determinism, and operator gates below. + +## Included scope + +1. Optional `ethos-full-0.5.0-macos-arm64.tar.gz` and + `ethos-full-0.5.0-linux-x64.tar.gz` under accepted ADR-0015. Base archives and all packages + remain caller-PDFium. +2. `ethos verify-batch` for 1–1,024 canonical citation requests against one validated source, + with atomic canonical NDJSON output and documented aggregate exit semantics. +3. `ethos report html`, a deterministic self-contained, cropless-by-default proof view of a + supported verification report. +4. Evidence Handle Bridge v1 trusted contexts and v2 structured model citations, including + deterministic hydration and evidence-state projection. Citation-emission v1 remains frozen. + +## Explicit exclusions + +No search, indexing, embeddings, model extraction, answer-prose citation parsing, multi-document +contexts, multiple handles per claim, locator repair, XLSX adapter, artifact bundling, Windows +public artifacts, Windows-with-PDFium, `ethos-mcp`, hosted services, benchmark/production claims, +or DocuShell type-mirror retirement is in scope. Windows verify-only CI remains maintained and +unpublished. + +## Deliverable boundaries and acceptance + +`ethos-full` is separately named, has a relative launcher, includes only the allowlisted binary, +runtime, licence/notice material and canonical manifest, verifies profile-pinned upstream/archive +and runtime hashes, and is double-built byte-identically. It requires macOS arm64 and Linux x64 +smoke, including `doctor --require-pdfium` and two identical pinned-fixture parses. + +Batch validates all input before processing or writing; blank, malformed, empty, oversized, crop, +or more than 1,024 requests fail atomically. Each output line must byte-equal the corresponding +single `verify` report with only its final newline removed. + +HTML has no JavaScript, network resources, timestamps, random IDs, external fonts, or +environment-dependent metadata. It escapes controlled content, preserves report/check order, +states the citation-grounding boundary, and permits crop links only with a safe relative prefix. + +Evidence handles use one document fingerprint, 1–1,024 unique opaque IDs, exactly one trusted +primary locator per entry, and 1–256 model claims each with exactly one handle. Hydration and +projection fail closed on invalid contexts, version/schema errors, mismatches, stale fingerprints, +or internally inconsistent reports; display and excerpt have no proof authority. + +## Consumer and release gates + +DocuShell worker integration, package/version/vendor migration, and its acceptance harness are +cross-repository work. Ethos will retain its platform-neutral type mirror in v0.5 and will not +claim that external migration complete until its separately scoped change is supplied and passes. + +Every deliverable adds focused tests and an Unreleased changelog entry. Required gates include +locked workspace build/tests, schemas/examples, Python API/package tests, npm generated-type and +consumer compilation, claims gates, `cargo deny` licence/source/ban checks, release-state and +changelog validation, relevant DocuShell acceptance, and clean generated-artifact validation. +All new artifacts require double-run byte identity. Before freeze, record the 30-sample cold +single-request comparison against v0.4.0 (no more than 10% regression) and the 32-request batch +comparison (batch median at most 50% of individual-process median). + +## Freeze and publication + +Freeze core commit A after core validation; build all candidates and record target smoke/hashes. +Refresh npm payload/types from A and freeze npm commit B. Rebuild unaffected candidates for byte +identity, run binding performance and consumer acceptance against A/B bytes, then publish without +rebuilding. Record a closeout only after live registries and GitHub assets match accepted hashes; +confirm that no Windows artifact was published. diff --git a/docs/validation/v0-4-0-release-closeout-summary.md b/docs/validation/v0-4-0-release-closeout-summary.md new file mode 100644 index 00000000..4880cff3 --- /dev/null +++ b/docs/validation/v0-4-0-release-closeout-summary.md @@ -0,0 +1,22 @@ +# Ethos v0.4.0 Release Closeout Summary + +Status: closed on 2026-07-21. + +GitHub Release `v0.4.0` was published at 2026-07-21T05:02:19Z and is the latest non-draft, +non-prerelease release. Its annotated tag object is +`6041b2ed5617f923e5d226f1f69bb184d70d5fce`, resolving to source commit +`e73477e427d2384bdb3b6b913578411325d3d107`. + +The published caller-provided-PDFium archives are: + +- `ethos-macos-arm64.tar.gz` — SHA-256 + `719f092101bb1e4cfeebc9507155b15739642f0b891664bd0d9f8a05d9688ba5`; +- `ethos-linux-x64.tar.gz` — SHA-256 + `616be562306d64a293554ca4695f19deb6e135dd328e88598a80e76f6f8fb3cd`. + +Each archive has its published checksum, canonical inventory, and target smoke sidecar. The +Rust crates, Python wheel, and `@docushell/ethos-pdf@0.4.0` package are live at `0.4.0`. +Windows verify-only and `ethos-full` were intentionally skipped and were not published. + +The release does not alter the caller-provided PDFium boundary or authorize hosted, production, +benchmark, speed, footprint, parser-quality, table-quality, semantic-truth, or Windows claims. From aad49a44c22a7eb223b16ab087f4f698177d92c5 Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Tue, 21 Jul 2026 13:02:19 +0530 Subject: [PATCH 02/30] verify: add atomic batch citation verification Add ethos verify-batch for 1-1024 canonical citation requests against one validated native or OpenDataLoader source. Buffer canonical NDJSON reports before atomic output, preserve single-verify bytes per line, reject crop evidence, and retain aggregate exit semantics. Signed-off-by: docushell-admin --- CHANGELOG.md | 2 + crates/ethos-cli/src/cmd/verify.rs | 111 ++++++++++++++++++++++++++++- crates/ethos-cli/src/main.rs | 24 +++++++ 3 files changed, 136 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31927c8b..eae22ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- verify: add buffered, atomic `verify-batch` NDJSON verification against one validated source, + with canonical per-line reports and aggregate ungrounded exit semantics. - release: reconcile the published v0.4.0 baseline, accept the bounded v0.5.0 `ethos-full` exception, and establish the v0.5.0 four-deliverable release scope. diff --git a/crates/ethos-cli/src/cmd/verify.rs b/crates/ethos-cli/src/cmd/verify.rs index 6e889f7d..64d991da 100644 --- a/crates/ethos-cli/src/cmd/verify.rs +++ b/crates/ethos-cli/src/cmd/verify.rs @@ -17,6 +17,7 @@ use std::collections::BTreeMap; use std::collections::HashSet; use std::path::{Path, PathBuf}; +use std::io::Write as _; use ethos_core::crop_element::{CropElementDescriptor, CropElementRendering}; use ethos_core::error::EthosError; @@ -40,7 +41,7 @@ use crate::cmd::crop_artifacts::{ }; use crate::{ default_max_input_bytes, read_document, read_file_limited, write_output, Failure, VerifyArgs, - VerifyOutputFormat, + VerifyBatchArgs, VerifyOutputFormat, }; pub(crate) fn verify(args: VerifyArgs) -> Result<(), Failure> { @@ -123,6 +124,114 @@ pub(crate) fn verify(args: VerifyArgs) -> Result<(), Failure> { write_report(args.out, args.format, report, args.fail_on_ungrounded) } +/// Verify an all-or-nothing NDJSON batch. Source/configuration validation happens once before +/// any report is rendered or output is written. +pub(crate) fn verify_batch(args: VerifyBatchArgs) -> Result<(), Failure> { + let max_input_bytes = default_max_input_bytes(); + let citations = parse_batch_citations(&args.citations_ndjson, max_input_bytes)?; + let config = load_batch_config(args.config.as_deref(), max_input_bytes)?; + if config.evidence.as_ref().is_some_and(|evidence| evidence.include_crops) { + return Err(Failure::Usage("verify-batch does not support crop evidence".to_string())); + } + for citation in &citations { + validate_citation_input(citation, &config)?; + } + let config_value = serde_json::to_value(&config).map_err(|e| EthosError::internal(e.to_string()))?; + let config_sha256 = ethos_core::c14n::sha256_hex(&config_value) + .map_err(|e| EthosError::internal(e.message))?; + + let reports = match args.grounding.as_deref() { + None => { + let document = read_document(&args.input)?; + batch_reports(&document, citations, &config, &config_sha256) + } + Some("opendataloader-json") => { + let bytes = read_file_limited(&args.input, max_input_bytes)?; + let text = String::from_utf8(bytes) + .map_err(|_| Failure::Usage("grounding input is not UTF-8".to_string()))?; + let source = OdlJsonSource::from_json_str(&text) + .map_err(|e| Failure::Usage(format!("opendataloader-json adapter: {e}")))?; + batch_reports(&source, citations, &config, &config_sha256) + } + Some(other) => return Err(Failure::Usage(format!( + "unknown grounding adapter '{other}' (available: opendataloader-json)" + ))), + }; + + let mut output = Vec::new(); + let mut any_ungrounded = false; + for report in reports { + any_ungrounded |= !report.all_evidence_grounded; + let mut line = verification_report_json_bytes(&report)?; + line.pop(); // The ordinary report framing newline becomes NDJSON framing below. + output.extend_from_slice(&line); + output.push(b'\n'); + } + write_batch_output(args.out, &output)?; + if args.fail_on_ungrounded && any_ungrounded { + return Err(Failure::Ungrounded); + } + Ok(()) +} + +fn write_batch_output(out: Option, bytes: &[u8]) -> Result<(), Failure> { + let Some(path) = out else { return write_output(None, bytes); }; + let parent = path.parent().filter(|parent| !parent.as_os_str().is_empty()).unwrap_or(Path::new(".")); + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .map_err(|_| Failure::Usage(format!("cannot write output: {}", path.display())))?; + temporary.write_all(bytes).and_then(|_| temporary.as_file().sync_all()) + .map_err(|_| Failure::Usage(format!("cannot write output: {}", path.display())))?; + temporary.persist(&path) + .map_err(|_| Failure::Usage(format!("cannot write output: {}", path.display())))?; + Ok(()) +} + +fn parse_batch_citations(path: &Path, max_input_bytes: u64) -> Result, Failure> { + let bytes = read_file_limited(path, max_input_bytes)?; + let text = std::str::from_utf8(&bytes) + .map_err(|_| Failure::Usage("citations NDJSON is not UTF-8".to_string()))?; + if text.is_empty() { + return Err(Failure::Usage("citations NDJSON must contain 1 to 1024 requests".to_string())); + } + let mut citations = Vec::new(); + for (index, line) in text.split('\n').enumerate() { + let line = line.strip_suffix('\r').unwrap_or(line); + if line.is_empty() { + if index + 1 == text.split('\n').count() { continue; } + return Err(Failure::Usage("citations NDJSON must not contain blank lines".to_string())); + } + let citation: CitationInput = serde_json::from_str(line).map_err(|_| { + Failure::Usage(format!("citations NDJSON line {} does not match the canonical citation input shape", index + 1)) + })?; + citations.push(citation); + } + if citations.is_empty() || citations.len() > 1024 { + return Err(Failure::Usage("citations NDJSON must contain 1 to 1024 requests".to_string())); + } + Ok(citations) +} + +fn load_batch_config(path: Option<&Path>, max_input_bytes: u64) -> Result { + let config = match path { + Some(path) => serde_json::from_slice(&read_file_limited(path, max_input_bytes)?) + .map_err(|_| Failure::Usage("verification config does not match the schema".to_string()))?, + None => VerificationConfig::default_v1(), + }; + validate_verification_config(&config)?; + Ok(config) +} + +fn batch_reports( + source: &dyn GroundingSource, + citations: Vec, + config: &VerificationConfig, + config_sha256: &str, +) -> Vec { + citations.into_iter().map(|citation| { + ethos_verify::verify_claims(source, citation, config, config_sha256.to_string()) + }).collect() +} + fn write_report( out: Option, format: VerifyOutputFormat, diff --git a/crates/ethos-cli/src/main.rs b/crates/ethos-cli/src/main.rs index ca7232b0..ba955c38 100644 --- a/crates/ethos-cli/src/main.rs +++ b/crates/ethos-cli/src/main.rs @@ -81,6 +81,8 @@ enum Command { }, /// Citation evidence verification (ethos-verify) Verify(VerifyArgs), + /// Verify many citation requests against one loaded grounding source + VerifyBatch(VerifyBatchArgs), /// Recompute and check a document fingerprint Fingerprint(FingerprintArgs), /// Diagnose local Ethos and caller-provided PDFium setup @@ -291,6 +293,27 @@ pub(crate) struct VerifyArgs { pub(crate) fail_on_ungrounded: bool, } +#[derive(Args)] +pub(crate) struct VerifyBatchArgs { + /// Grounding input: canonical Ethos document, or foreign output with --grounding. + pub(crate) input: PathBuf, + /// NDJSON file containing one canonical citation input per non-empty line. + #[arg(long)] + pub(crate) citations_ndjson: PathBuf, + /// Foreign grounding adapter id (e.g. `opendataloader-json`). + #[arg(long)] + pub(crate) grounding: Option, + /// Verification config (JSON); defaults to the pinned `default-v1`. + #[arg(long)] + pub(crate) config: Option, + /// Output path for canonical verification-report NDJSON (default: stdout). + #[arg(long)] + pub(crate) out: Option, + /// Exit 1 after writing reports when any request is not fully grounded. + #[arg(long)] + pub(crate) fail_on_ungrounded: bool, +} + #[derive(Clone, Copy, ValueEnum)] pub(crate) enum VerifyOutputFormat { Json, @@ -393,6 +416,7 @@ fn run(cli: Cli) -> Result<(), Failure> { command: EvidenceCommand::Anchor(args), } => cmd::evidence::evidence_anchor(args), Command::Verify(args) => cmd::verify::verify(args), + Command::VerifyBatch(args) => cmd::verify::verify_batch(args), Command::Fingerprint(args) => cmd::doc::fingerprint(args), Command::Doctor(args) => cmd::doctor::doctor(args), Command::CropElement(args) => cmd::crop::crop_element(args), From 32d7ef9f570ea2c34237f0dc0c73f7758f15ffc6 Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Tue, 21 Jul 2026 13:15:16 +0530 Subject: [PATCH 03/30] report: add deterministic HTML proof reports Add ethos report html for supported verification-report schemas. Render a self-contained escaped proof view with grounding-boundary wording, deterministic output, and fail-closed crop-root validation; cover determinism, injection, unsupported schema, and unsafe paths. Signed-off-by: docushell-admin --- CHANGELOG.md | 2 ++ crates/ethos-cli/src/cmd/mod.rs | 1 + crates/ethos-cli/src/cmd/report.rs | 49 ++++++++++++++++++++++++++++++ crates/ethos-cli/src/main.rs | 24 +++++++++++++++ crates/ethos-cli/tests/verify.rs | 43 ++++++++++++++++++++++++++ 5 files changed, 119 insertions(+) create mode 100644 crates/ethos-cli/src/cmd/report.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index eae22ae5..0a447642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- report: add deterministic, self-contained HTML proof reports with supported-schema checks, + escaped report content, and fail-closed crop-root validation. - verify: add buffered, atomic `verify-batch` NDJSON verification against one validated source, with canonical per-line reports and aggregate ungrounded exit semantics. - release: reconcile the published v0.4.0 baseline, accept the bounded v0.5.0 diff --git a/crates/ethos-cli/src/cmd/mod.rs b/crates/ethos-cli/src/cmd/mod.rs index 9f7c3141..0250df7d 100644 --- a/crates/ethos-cli/src/cmd/mod.rs +++ b/crates/ethos-cli/src/cmd/mod.rs @@ -20,5 +20,6 @@ pub(crate) mod doc; pub(crate) mod doctor; pub(crate) mod evidence; pub(crate) mod rag; +pub(crate) mod report; pub(crate) mod security; pub(crate) mod verify; diff --git a/crates/ethos-cli/src/cmd/report.rs b/crates/ethos-cli/src/cmd/report.rs new file mode 100644 index 00000000..5f02c622 --- /dev/null +++ b/crates/ethos-cli/src/cmd/report.rs @@ -0,0 +1,49 @@ +use ethos_core::verify_types::{VerificationReport, HARDENED_VERIFICATION_SCHEMA_VERSION}; + +use crate::{read_file_limited, Failure, ReportHtmlArgs}; + +pub(crate) fn html(args: ReportHtmlArgs) -> Result<(), Failure> { + let report: VerificationReport = serde_json::from_slice(&read_file_limited(&args.input, crate::default_max_input_bytes())?) + .map_err(|_| Failure::Usage("input is not a supported verification report".to_string()))?; + if report.schema_version != ethos_core::SCHEMA_VERSION && report.schema_version != HARDENED_VERIFICATION_SCHEMA_VERSION { + return Err(Failure::Usage("verification report schema_version is not supported".to_string())); + } + let crop_root = args.crop_root.as_deref().map(validate_crop_root).transpose()?; + let bytes = render(&report, crop_root.as_deref()).into_bytes(); + std::fs::write(&args.out, bytes).map_err(|_| Failure::Usage(format!("cannot write output: {}", args.out.display()))) +} + +fn validate_crop_root(value: &str) -> Result { + if value.is_empty() || value.starts_with('/') || value.contains("\\") || value.contains("..") || value.contains('?') || value.contains('#') || value.contains("://") || value.split('/').any(str::is_empty) { + return Err(Failure::Usage("--crop-root must be a safe relative prefix".to_string())); + } + Ok(value.trim_end_matches('/').to_string()) +} + +fn escape(value: &str) -> String { value.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'") } + +fn safe_crop_basename(value: &str) -> Option<&str> { + (!value.is_empty() && !value.contains(['/', '\\', '?', '#']) && !value.contains("..")) + .then_some(value) +} + +fn render(report: &VerificationReport, crop_root: Option<&str>) -> String { + let proof = report.proof_summary(); + let mut out = String::from("Ethos proof report

Ethos proof report

"); + out.push_str(&format!("

Proof status: {}; request certified: {}.

", if report.all_evidence_grounded {"ok"} else {"bad"}, proof.proof_status.as_str(), report.all_evidence_grounded)); + out.push_str(&format!("
Document fingerprint
{}
Fingerprint stale
{}
Verification config hash
{}
Grounding parser
{} {}
", escape(report.document_fingerprint.as_deref().unwrap_or("unavailable")), report.fingerprint_stale, escape(&report.verification_config_sha256), escape(&report.grounding.parser.name), escape(&report.grounding.parser.version))); + out.push_str("

Ethos verifies citation grounding, not semantic truth, answer relevance, completeness, or synthesis quality.

Checks

"); + for check in &report.checks { + out.push_str(&format!("

{}: {:?}

Status: {:?}; match method: {:?}; semantic unverified: {}.

Claim: {}

Locator: {}

", escape(&check.id), check.claim.kind, check.status, check.match_method, check.semantic_unverified, escape(check.claim.text.as_deref().unwrap_or("(presence)")), escape(&serde_json::to_string(&check.claim.citation).unwrap_or_default()))); + if let Some(evidence) = &check.evidence { + out.push_str(&format!("

Evidence: {}

Page: {}; bbox: {}

", escape(evidence.text.as_deref().unwrap_or("unavailable")), escape(evidence.page.as_deref().unwrap_or("unavailable")), escape(&evidence.bbox.map(|bbox| format!("{:?}", bbox)).unwrap_or_else(|| "unavailable".to_string())))); + if let Some(crop_ref) = evidence.crop_ref.as_deref() { + if let (Some(root), Some(name)) = (crop_root, safe_crop_basename(crop_ref)) { + out.push_str(&format!("

Crop: {}

", escape(root), escape(name), escape(name))); + } else { out.push_str(&format!("

Crop unavailable in this standalone report: {}

", escape(crop_ref))); } + } + } + out.push_str("
"); + } + out.push_str("\n"); out +} diff --git a/crates/ethos-cli/src/main.rs b/crates/ethos-cli/src/main.rs index ba955c38..22769710 100644 --- a/crates/ethos-cli/src/main.rs +++ b/crates/ethos-cli/src/main.rs @@ -83,6 +83,11 @@ enum Command { Verify(VerifyArgs), /// Verify many citation requests against one loaded grounding source VerifyBatch(VerifyBatchArgs), + /// Render a deterministic human-readable proof report + Report { + #[command(subcommand)] + command: ReportCommand, + }, /// Recompute and check a document fingerprint Fingerprint(FingerprintArgs), /// Diagnose local Ethos and caller-provided PDFium setup @@ -227,6 +232,24 @@ enum EvidenceCommand { Anchor(EvidenceAnchorArgs), } +#[derive(Subcommand)] +enum ReportCommand { + /// Render a verification report as self-contained HTML + Html(ReportHtmlArgs), +} + +#[derive(Args)] +pub(crate) struct ReportHtmlArgs { + /// Canonical verification report JSON. + pub(crate) input: PathBuf, + /// Destination HTML file. + #[arg(long)] + pub(crate) out: PathBuf, + /// Safe relative prefix used only for existing crop references. + #[arg(long)] + pub(crate) crop_root: Option, +} + #[derive(Args)] pub(crate) struct RagChunkArgs { /// Canonical document (`*.ethos.json`) @@ -417,6 +440,7 @@ fn run(cli: Cli) -> Result<(), Failure> { } => cmd::evidence::evidence_anchor(args), Command::Verify(args) => cmd::verify::verify(args), Command::VerifyBatch(args) => cmd::verify::verify_batch(args), + Command::Report { command: ReportCommand::Html(args) } => cmd::report::html(args), Command::Fingerprint(args) => cmd::doc::fingerprint(args), Command::Doctor(args) => cmd::doctor::doctor(args), Command::CropElement(args) => cmd::crop::crop_element(args), diff --git a/crates/ethos-cli/tests/verify.rs b/crates/ethos-cli/tests/verify.rs index ae2b3cad..61ff8092 100644 --- a/crates/ethos-cli/tests/verify.rs +++ b/crates/ethos-cli/tests/verify.rs @@ -2883,3 +2883,46 @@ fn case_insensitive_config_allows_literal_case_difference() { ); assert_eq!(report["all_evidence_grounded"], true); } + +#[test] +fn report_html_renders_supported_report_deterministically_and_escapes_content() { + let root = repo_root(); + let mut report = json_file(root.join("schemas/examples/verification-report.example.json")); + report["grounding"]["parser"]["name"] = serde_json::json!("parser