From 23a57327c11a583c4879cf9747f2c36062e17bfd Mon Sep 17 00:00:00 2001 From: docushell-admin Date: Mon, 20 Jul 2026 23:57:21 +0530 Subject: [PATCH] ci: make release CLI archives deterministic Signed-off-by: docushell-admin --- .github/scripts/build_release_cli_archive.py | 84 +++++++++++++++++++ .../scripts/test_build_release_cli_archive.py | 77 +++++++++++++++++ .../test_release_artifact_workflow_prep.py | 4 + .../test_release_reproducibility_scaffold.py | 3 +- .github/workflows/release.yml | 6 +- CHANGELOG.md | 2 + Makefile | 2 + 7 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/build_release_cli_archive.py create mode 100644 .github/scripts/test_build_release_cli_archive.py diff --git a/.github/scripts/build_release_cli_archive.py b/.github/scripts/build_release_cli_archive.py new file mode 100644 index 00000000..7c645620 --- /dev/null +++ b/.github/scripts/build_release_cli_archive.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 The Ethos maintainers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create a byte-identical gzip-compressed CLI release archive.""" + +from __future__ import annotations + +import argparse +import gzip +import tarfile +from pathlib import Path + + +ARCHIVE_MTIME = 0 + + +def add_file(archive: tarfile.TarFile, source: Path, arcname: str) -> None: + info = tarfile.TarInfo(arcname) + info.size = source.stat().st_size + info.mode = 0o755 if source.name == "ethos" else 0o644 + info.mtime = ARCHIVE_MTIME + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + with source.open("rb") as payload: + archive.addfile(info, payload) + + +def build_archive(artifact_dir: Path, output: Path) -> None: + artifact_dir = artifact_dir.resolve() + required_files = ("ethos", "LICENSE", "NOTICE", "pdfium-manual-setup.md") + missing = [name for name in required_files if not (artifact_dir / name).is_file()] + if missing: + raise ValueError(f"artifact directory is missing required files: {', '.join(missing)}") + + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as destination: + with gzip.GzipFile(filename="", mode="wb", fileobj=destination, mtime=ARCHIVE_MTIME) as compressed: + with tarfile.open(mode="w", fileobj=compressed, format=tarfile.PAX_FORMAT) as archive: + root = tarfile.TarInfo(f"{artifact_dir.name}/") + root.type = tarfile.DIRTYPE + root.mode = 0o755 + root.mtime = ARCHIVE_MTIME + root.uid = 0 + root.gid = 0 + root.uname = "root" + root.gname = "root" + archive.addfile(root) + for name in required_files: + add_file(archive, artifact_dir / name, f"{artifact_dir.name}/{name}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--artifact-dir", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + build_archive(args.artifact_dir, args.out) + except ValueError as error: + raise SystemExit(str(error)) from error + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test_build_release_cli_archive.py b/.github/scripts/test_build_release_cli_archive.py new file mode 100644 index 00000000..2fe63e07 --- /dev/null +++ b/.github/scripts/test_build_release_cli_archive.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 The Ethos maintainers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import importlib.util +import tarfile +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github/scripts/build_release_cli_archive.py" +SPEC = importlib.util.spec_from_file_location("build_release_cli_archive", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class BuildReleaseCliArchiveTests(unittest.TestCase): + def test_two_archives_are_byte_identical_and_normalize_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact_dir = Path(temp) / "ethos-linux-x64" + artifact_dir.mkdir() + for name, content in { + "ethos": b"#!/bin/sh\necho ethos\n", + "LICENSE": b"license\n", + "NOTICE": b"notice\n", + "pdfium-manual-setup.md": b"manual\n", + }.items(): + (artifact_dir / name).write_bytes(content) + + first = Path(temp) / "first.tar.gz" + second = Path(temp) / "second.tar.gz" + MODULE.build_archive(artifact_dir, first) + MODULE.build_archive(artifact_dir, second) + + self.assertEqual(first.read_bytes(), second.read_bytes()) + with tarfile.open(first, "r:gz") as archive: + members = archive.getmembers() + self.assertEqual( + [ + "ethos-linux-x64", + "ethos-linux-x64/ethos", + "ethos-linux-x64/LICENSE", + "ethos-linux-x64/NOTICE", + "ethos-linux-x64/pdfium-manual-setup.md", + ], + [member.name for member in members], + ) + self.assertTrue(all(member.mtime == 0 for member in members)) + self.assertTrue(all(member.uid == 0 and member.gid == 0 for member in members)) + + def test_missing_required_file_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact_dir = Path(temp) / "ethos-linux-x64" + artifact_dir.mkdir() + with self.assertRaisesRegex(ValueError, "missing required files: ethos"): + MODULE.build_archive(artifact_dir, Path(temp) / "archive.tar.gz") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/test_release_artifact_workflow_prep.py b/.github/scripts/test_release_artifact_workflow_prep.py index fec30649..0160e960 100644 --- a/.github/scripts/test_release_artifact_workflow_prep.py +++ b/.github/scripts/test_release_artifact_workflow_prep.py @@ -47,6 +47,9 @@ def test_workflow_generates_draft_artifacts_without_publication(self) -> None: self.assertIn("build-windows-verify-candidate.py", text) self.assertIn("Windows candidate archives differ", text) self.assertIn("write_release_artifact_inventory.py", text) + self.assertIn("build_release_cli_archive.py", text) + self.assertNotIn('tar -C "$(dirname "$out")" -czf', text) + self.assertIn("test_build_release_cli_archive.py", text) self.assertIn("smoke_release_cli_artifact.py", text) self.assertIn('--expected-version "ethos 0.4.0"', text) self.assertIn("--target \"${{ matrix.artifact_target }}\"", text) @@ -69,6 +72,7 @@ def test_preflight_runs_release_scope_guards_before_artifacts(self) -> None: "test_python_public_api_policy.py", "test_npm_binary_package_scaffold.py", "test_pdfium_manual_setup_contract.py", + "test_build_release_cli_archive.py", "test_windows_verify_candidate.py", ): self.assertIn(guard, text) diff --git a/.github/scripts/test_release_reproducibility_scaffold.py b/.github/scripts/test_release_reproducibility_scaffold.py index f8a5cfa6..0cf52e08 100644 --- a/.github/scripts/test_release_reproducibility_scaffold.py +++ b/.github/scripts/test_release_reproducibility_scaffold.py @@ -44,7 +44,8 @@ def test_workflow_records_rebuildable_cli_inputs(self) -> None: self.assertIn("rustup show", text) self.assertIn("cargo build --locked --release -p ethos-cli", text) - self.assertIn("tar -C", text) + self.assertIn("build_release_cli_archive.py", text) + self.assertNotIn('tar -C "$(dirname "$out")" -czf', text) self.assertIn("shasum -a 256", text) self.assertIn("ethos-${{ matrix.artifact_target }}", text) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b95b9a90..1892c29f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,8 @@ jobs: run: python3 .github/scripts/test_npm_binary_package_scaffold.py - name: PDFium manual setup contract tests run: python3 .github/scripts/test_pdfium_manual_setup_contract.py + - name: deterministic CLI archive tests + run: python3 .github/scripts/test_build_release_cli_archive.py - name: Windows verify-only candidate contract tests run: python3 .github/scripts/test_windows_verify_candidate.py @@ -54,7 +56,9 @@ jobs: cp target/release/ethos "$out/ethos" cp LICENSE NOTICE "$out/" cp docs/pdfium-manual-setup.md "$out/" - tar -C "$(dirname "$out")" -czf "$out.${{ matrix.archive_ext }}" "$(basename "$out")" + python3 .github/scripts/build_release_cli_archive.py \ + --artifact-dir "$out" \ + --out "$out.${{ matrix.archive_ext }}" shasum -a 256 "$out.${{ matrix.archive_ext }}" > "$out.${{ matrix.archive_ext }}.sha256" python3 .github/scripts/write_release_artifact_inventory.py \ --artifact "$out.${{ matrix.archive_ext }}" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index b053f559..3a154a1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- boundary-exception: normalize CLI release archive metadata and gzip headers so repeated macOS arm64 and Linux + x64 candidate builds are byte-identical before npm vendoring or publication. - build: remove unused Cargo Deny license allowances and document the unavoidable transitive `wit-bindgen` duplicate so release hygiene runs without warnings. - boundary-exception: remove completed release and milestone records together with their retired diff --git a/Makefile b/Makefile index ca86a18b..f4d4ca15 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,8 @@ v0-4-release-prep: $(MAKE) citation-emission-v1-contract PYTHON=$(PYTHON) $(MAKE) python-surface-test PYTHON=$(PYTHON) npm test --prefix packages/npm/ethos-pdf + $(PYTHON) .github/scripts/test_build_release_cli_archive.py + $(PYTHON) .github/scripts/test_release_artifact_workflow_prep.py $(PYTHON) .github/scripts/test_v0_4_0_version_activation.py $(MAKE) release-hygiene PYTHON=$(PYTHON) $(PYTHON) .github/scripts/claims_gate.py