diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3a302671..64be1f3b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -92,37 +92,9 @@ jobs: if-no-files-found: error vendored-archives: - name: Verify vendored libghostty-vt archives reproduce - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - # NOTE this is a DIFFERENT zig from the one the `linux` job pins. That one - # is cargo-zigbuild's cross-linker (0.14.0). This one must match - # ghostty's `requireZig` at our pin EXACTLY — it compares major.minor for - # equality, so a newer zig is rejected, not tolerated. The two uses are - # independent and neither version constrains the other. - - name: Read the required zig version from the pin - id: zig - run: echo "version=$(cat crates/spyc-vt-sys/REQUIRED_ZIG)" >> "$GITHUB_OUTPUT" - - uses: mlugg/setup-zig@v2 - with: - version: ${{ steps.zig.outputs.version }} - # This job REBUILDS the archives and checks they match what is committed, - # rather than building the ones we ship. The archives are committed on - # purpose so `cargo install spyc` needs no Zig — which means the only thing - # standing behind SECURITY.md's "reproducible by anyone" claim is something - # actually reproducing them. That is this job. - - name: Rebuild from the pinned source and compare checksums - run: | - cp crates/spyc-vt-sys/vendor/CHECKSUMS /tmp/CHECKSUMS.committed - make vendor-ghostty - if ! diff -u /tmp/CHECKSUMS.committed crates/spyc-vt-sys/vendor/CHECKSUMS; then - echo "::error::the committed archives are not what the pinned ghostty source produces" - echo 'Either the pin moved without running: make vendor-ghostty' - echo 'or the committed archives were edited by hand.' - exit 1 - fi - echo "committed archives reproduce from the pin" + # The rebuild-and-compare lives in its own workflow so a PR touching the + # archives, the pin or the recipe runs it too, before any tag does. + uses: ./.github/workflows/vendored-archives.yml publish: name: Sign + publish release diff --git a/.github/workflows/vendored-archives.yml b/.github/workflows/vendored-archives.yml new file mode 100644 index 00000000..1391ad79 --- /dev/null +++ b/.github/workflows/vendored-archives.yml @@ -0,0 +1,71 @@ +# spyc-vt-sys's vendored libghostty-vt archives must be what the pinned ghostty +# source produces. This rebuilds all five and compares checksums. +# +# The archives are committed so `cargo install spyc` needs no Zig, which means +# the only thing standing behind SECURITY.md's "reproducible by anyone" claim +# is something actually reproducing them: this workflow. +# +# Runs: +# - from release.yml (`workflow_call`), gating the GitHub release; +# - on a PR that touches the archives, the pin or the recipe, so a change +# proves itself before any tag does; +# - on demand (`workflow_dispatch`). +# It takes ~10 minutes, which is why it isn't on every PR. +name: Vendored archives + +on: + workflow_call: + workflow_dispatch: + pull_request: + paths: + - "crates/spyc-vt-sys/vendor/**" + - "crates/spyc-vt-sys/src/pin.rs" + - "crates/spyc-vt-sys/REQUIRED_ZIG" + - "scripts/vendor-ghostty.sh" + - "scripts/vendor-ghostty-container.sh" + - "Makefile" + - ".github/workflows/vendored-archives.yml" + +permissions: + contents: read + +jobs: + reproduce: + name: Verify vendored libghostty-vt archives reproduce + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + # NOTE this is a DIFFERENT zig from the one release.yml's `linux` job pins. + # That one is cargo-zigbuild's cross-linker (0.14.0). This one must match + # ghostty's `requireZig` at our pin EXACTLY — it compares major.minor for + # equality, so a newer zig is rejected, not tolerated. The two uses are + # independent and neither version constrains the other. + - name: Read the required zig version from the pin + id: zig + run: echo "version=$(cat crates/spyc-vt-sys/REQUIRED_ZIG)" >> "$GITHUB_OUTPUT" + # setup-zig installs ziglang.org's build of that version, the one the + # committed archives are made with. No cache: the recipe uses a private + # global cache, so a restored one would only be saved again unused. + - uses: mlugg/setup-zig@v2 + with: + version: ${{ steps.zig.outputs.version }} + use-cache: false + - name: Rebuild from the pinned source and compare checksums + run: | + cp crates/spyc-vt-sys/vendor/CHECKSUMS /tmp/CHECKSUMS.committed + make vendor-ghostty + if ! diff -u /tmp/CHECKSUMS.committed crates/spyc-vt-sys/vendor/CHECKSUMS; then + echo "::error::the committed archives are not what the pinned ghostty source produces" + echo 'Either the pin moved without running: make vendor-ghostty' + echo 'or the committed archives were built some other way.' + exit 1 + fi + echo "committed archives reproduce from the pin" + # The archives this run built, matching or not: on a mismatch, these are + # the canonical bytes to commit. + - uses: actions/upload-artifact@v7 + if: always() + with: + name: vendored-archives + path: crates/spyc-vt-sys/vendor/ + if-no-files-found: error diff --git a/Makefile b/Makefile index a55626dc..9bfaf661 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,12 @@ vendor-ghostty: ## Rebuild spyc-vt-sys's vendored libghostty-vt archives at the @# Out-of-band on purpose: the archives are committed so that `cargo install @# spyc` needs no Zig, no network and no git for this crate. Run this only on @# a deliberate pin bump, and follow it with the full harness re-run that - @# `spyc-vt-sys/src/pin.rs` (BUMP_POLICY) describes. + @# `spyc-vt-sys/src/pin.rs` (BUMP_POLICY) describes. Linux only: see + @# `vendor-ghostty-container` for a Mac. + @if [ "$$(uname -s)" != Linux ]; then \ + echo "build the archives on Linux; from a Mac: make vendor-ghostty-container"; \ + exit 1; \ + fi @command -v zig >/dev/null 2>&1 || { \ echo "zig not found. The pin needs exactly zig $$(cat crates/spyc-vt-sys/REQUIRED_ZIG)"; \ echo " ghostty's requireZig compares major.minor for EQUALITY, not as a floor."; \ @@ -150,6 +155,10 @@ vendor-ghostty: ## Rebuild spyc-vt-sys's vendored libghostty-vt archives at the fi scripts/vendor-ghostty.sh +.PHONY: vendor-ghostty-container +vendor-ghostty-container: ## Run vendor-ghostty in a Linux container (podman or docker), e.g. from a Mac + scripts/vendor-ghostty-container.sh + .PHONY: deny deny: ## Supply-chain checks: advisories, licenses, sources, bans (cargo-deny) @command -v cargo-deny >/dev/null 2>&1 || { \ diff --git a/SECURITY.md b/SECURITY.md index bf54f4d7..88dd284f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -235,9 +235,12 @@ What we do about it: carries a SHA-256 per archive, and the build script recomputes and compares before emitting a single link flag. A mismatch fails the build with both hashes printed. Verified by tampering with a byte, not by assuming. -* **Reproducible by anyone.** `make vendor-ghostty` rebuilds all five from the - pinned source with a pinned Zig version and rewrites the checksums, so a third - party can confirm the committed bytes are what the pinned source produces. +* **Reproducible by anyone.** `make vendor-ghostty` rebuilds all five on Linux + from the pinned source with a pinned Zig version (ziglang.org's build of it) + and rewrites the checksums, so a third party can confirm the committed bytes + are what the pinned source produces (`make vendor-ghostty-container` does the + same from a Mac). CI does exactly that before every release, and the release + is not published unless the bytes match. * **The alternative was worse.** The published `libghostty-vt-sys` bindings `git clone` ghostty *at build time* and run `zig build`. That fetches code from a URL during `cargo build`, is invisible to `cargo vendor` and diff --git a/crates/spyc-vt-sys/README.md b/crates/spyc-vt-sys/README.md index a76e985c..8de4b4a9 100644 --- a/crates/spyc-vt-sys/README.md +++ b/crates/spyc-vt-sys/README.md @@ -59,9 +59,33 @@ build fails with a message that says so. ## Rebuilding the archives ```sh -make vendor-ghostty # needs zig 0.16.0 (see REQUIRED_ZIG) +make vendor-ghostty # on Linux; needs zig 0.16.0 (see REQUIRED_ZIG) +make vendor-ghostty-container # from a Mac: the same, in a podman/docker container ``` +**Build on Linux.** On a macOS host ghostty's build compiles the darwin targets +against the installed Xcode SDK, so those archives would follow whatever Xcode +is installed. From Linux it uses zig's bundled Darwin headers, which the pinned +zig fixes, and any Linux host, arm64 or x86_64, builds the same five archives. +`make vendor-ghostty` refuses to run anywhere else. + +Use [ziglang.org's build](https://ziglang.org/download/) of that version, which +is what CI and the container target install. Another build of the same version +number can compile different code. The script also uses a private global zig +cache, because zig keys its cache on the version string: a shared cache would +hand this build objects an earlier, different zig build made. + +The result is byte-for-byte reproducible, which the +[`Vendored archives`](../../.github/workflows/vendored-archives.yml) workflow +checks on every tag and on any PR that touches the archives, the pin or the +recipe; it uploads the archives it built, so a mismatch hands over the bytes to +commit. What zig emits is not reproducible as it stands: a Linux archive names +each member by its absolute path in the build cache, and a darwin one's symbol +table records the build time and the builder's uid/gid. The objects inside are +identical across runs, paths and caches, so the script re-archives the same +members, in the same order, under their basenames, with this zig's own `ar` in +deterministic mode. + Zig cross-compiles all five from one host. The recipe uses `ReleaseSmall`, which is the size lever; it does **not** strip, because at `ReleaseSmall` the archives carry no debug sections and a strip step measurably changes nothing. diff --git a/crates/spyc-vt-sys/vendor/CHECKSUMS b/crates/spyc-vt-sys/vendor/CHECKSUMS index e749a884..89de7a83 100644 --- a/crates/spyc-vt-sys/vendor/CHECKSUMS +++ b/crates/spyc-vt-sys/vendor/CHECKSUMS @@ -1,5 +1,5 @@ -9274d97374cd028a1af6cd282528419ae97c5a4b4474b59069b91b8e1891218d ./aarch64-apple-darwin/libghostty-vt.a -c66c81c99db1412aea81d873981a1129c1831d5159b30d51862ea74d1e6a3f61 ./aarch64-unknown-linux-musl/libghostty-vt.a -a132421fd9cdf6e614bb491523d4e10e03b2a42a43ddb8968d4494f2996bbe1e ./x86_64-apple-darwin/libghostty-vt.a -dc23759dcd9a2b0f310ec842545c969e6e6ee18d42a8d28a62cea38ea1367baa ./x86_64-unknown-linux-gnu/libghostty-vt.a -c34fa737af76db5ffb619314e4a1006152201741d42e4417c8b5042e386f792c ./x86_64-unknown-linux-musl/libghostty-vt.a +4fd6d8db8b2c639af022e081f4400449adfbe19e367283c477c80891905ccaac ./aarch64-apple-darwin/libghostty-vt.a +4f443a75d356811fe9346d24d3a4699a1547ba70abebe86ccb891fff910ba0ab ./aarch64-unknown-linux-musl/libghostty-vt.a +3d8abeb52854ba9108feeb26d8affa875f0b5ecfb0946552e22000d999d5eb43 ./x86_64-apple-darwin/libghostty-vt.a +3cb575558a59ce08cd33494611ec3f1d6d47e1fc6170e62dd4750587cb26ac0d ./x86_64-unknown-linux-gnu/libghostty-vt.a +cda3c6629b48220ab8b9f4a2e77fb474e116cc6c5498f306393f572fb7aad42b ./x86_64-unknown-linux-musl/libghostty-vt.a diff --git a/crates/spyc-vt-sys/vendor/aarch64-apple-darwin/libghostty-vt.a b/crates/spyc-vt-sys/vendor/aarch64-apple-darwin/libghostty-vt.a index ed2c6055..ccd2841f 100644 Binary files a/crates/spyc-vt-sys/vendor/aarch64-apple-darwin/libghostty-vt.a and b/crates/spyc-vt-sys/vendor/aarch64-apple-darwin/libghostty-vt.a differ diff --git a/crates/spyc-vt-sys/vendor/aarch64-unknown-linux-musl/libghostty-vt.a b/crates/spyc-vt-sys/vendor/aarch64-unknown-linux-musl/libghostty-vt.a index 1d776b95..5b502cf1 100644 Binary files a/crates/spyc-vt-sys/vendor/aarch64-unknown-linux-musl/libghostty-vt.a and b/crates/spyc-vt-sys/vendor/aarch64-unknown-linux-musl/libghostty-vt.a differ diff --git a/crates/spyc-vt-sys/vendor/x86_64-apple-darwin/libghostty-vt.a b/crates/spyc-vt-sys/vendor/x86_64-apple-darwin/libghostty-vt.a index 93b9dec8..6a1e0255 100644 Binary files a/crates/spyc-vt-sys/vendor/x86_64-apple-darwin/libghostty-vt.a and b/crates/spyc-vt-sys/vendor/x86_64-apple-darwin/libghostty-vt.a differ diff --git a/crates/spyc-vt-sys/vendor/x86_64-unknown-linux-gnu/libghostty-vt.a b/crates/spyc-vt-sys/vendor/x86_64-unknown-linux-gnu/libghostty-vt.a index 77df1a01..0a6d0707 100644 Binary files a/crates/spyc-vt-sys/vendor/x86_64-unknown-linux-gnu/libghostty-vt.a and b/crates/spyc-vt-sys/vendor/x86_64-unknown-linux-gnu/libghostty-vt.a differ diff --git a/crates/spyc-vt-sys/vendor/x86_64-unknown-linux-musl/libghostty-vt.a b/crates/spyc-vt-sys/vendor/x86_64-unknown-linux-musl/libghostty-vt.a index 56900a53..6e41e12b 100644 Binary files a/crates/spyc-vt-sys/vendor/x86_64-unknown-linux-musl/libghostty-vt.a and b/crates/spyc-vt-sys/vendor/x86_64-unknown-linux-musl/libghostty-vt.a differ diff --git a/scripts/vendor-ghostty-container.sh b/scripts/vendor-ghostty-container.sh new file mode 100755 index 00000000..e4162573 --- /dev/null +++ b/scripts/vendor-ghostty-container.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# `make vendor-ghostty` inside a Linux container, for a host that isn't Linux. +# The archives must be built on Linux (see scripts/vendor-ghostty.sh); this +# gives a Mac the same bytes CI builds. It writes into this checkout's +# crates/spyc-vt-sys/vendor/, exactly as the native recipe does. +set -euo pipefail + +engine=$(command -v podman || command -v docker) || { + echo "needs podman or docker" + exit 1 +} +zig_version=$(cat crates/spyc-vt-sys/REQUIRED_ZIG) + +# Passed as an argument, not on stdin: apt-get reads stdin, and fed through +# `bash -s` it swallowed the rest of this script, which then exited 0 having +# built nothing. +in_container=$(cat <<'IN_CONTAINER' +set -euo pipefail +apt-get update -qq >/dev/null +apt-get install -y -qq git make perl ca-certificates curl xz-utils python3 >/dev/null +# ziglang.org's build of the pinned version, as CI's setup-zig installs, checked +# against the sha256 ziglang.org publishes for it. +curl -fsSL https://ziglang.org/download/index.json -o /tmp/index.json +read -r url sum < <(python3 -c ' +import json, os, platform +d = json.load(open("/tmp/index.json"))[os.environ["ZIG_VERSION"]][platform.machine() + "-linux"] +print(d["tarball"], d["shasum"])') +curl -fsSL "$url" -o /tmp/zig.tar.xz +echo "$sum /tmp/zig.tar.xz" | sha256sum -c - +mkdir -p /opt/zig && tar -xJf /tmp/zig.tar.xz -C /opt/zig --strip-components=1 +export PATH="/opt/zig:$PATH" +make vendor-ghostty +IN_CONTAINER +) + +"$engine" run --rm -v "$(pwd):/src" -w /src -e ZIG_VERSION="$zig_version" \ + docker.io/library/ubuntu:24.04 bash -c "$in_container" diff --git a/scripts/vendor-ghostty.sh b/scripts/vendor-ghostty.sh index d61878c7..64b97297 100755 --- a/scripts/vendor-ghostty.sh +++ b/scripts/vendor-ghostty.sh @@ -7,13 +7,29 @@ # for what else a bump owes. set -euo pipefail +# Linux only. On a macOS host ghostty's build compiles the darwin targets +# against the installed Xcode SDK, so those archives would follow whatever +# Xcode is installed; from Linux it uses zig's bundled Darwin headers, which +# the pinned zig fixes. Any Linux host, arm64 or x86_64, builds the same five. +if [ "$(uname -s)" != Linux ]; then + echo "build the archives on Linux; from a Mac: make vendor-ghostty-container" + echo " (on macOS the darwin archives would depend on the installed Xcode SDK)" + exit 1 +fi + +ROOT=$(pwd) CRATE="crates/spyc-vt-sys" PIN=$(sed -n 's/^pub const GHOSTTY_COMMIT: &str = "\(.*\)";$/\1/p' "$CRATE/src/pin.rs") [ -n "$PIN" ] || { echo "could not read GHOSTTY_COMMIT from $CRATE/src/pin.rs"; exit 1; } echo "pin: $PIN" WORK="${GHOSTTY_WORK:-$(mktemp -d)}" +mkdir -p "$WORK" && WORK=$(cd "$WORK" && pwd) SRC="$WORK/ghostty" +# A private global cache. zig keys its cache on the version string, so a shared +# one hands this build objects (compiler_rt, for one) that another build of the +# same version made earlier, and a different zig build makes different code. +export ZIG_GLOBAL_CACHE_DIR="$WORK/zig-global-cache" if [ ! -d "$SRC/.git" ]; then echo "cloning ghostty into $SRC" git clone --filter=blob:none --no-checkout https://github.com/ghostty-org/ghostty "$SRC" @@ -21,6 +37,32 @@ fi git -C "$SRC" fetch --depth=1 origin "$PIN" 2>/dev/null || git -C "$SRC" fetch origin git -C "$SRC" checkout -q "$PIN" +# Re-archive deterministically. zig's archive is not reproducible as built: a +# Linux one names each member by its absolute path in the build cache, which +# changes every run, and a darwin one's symbol table records the build time and +# the builder's uid/gid. The objects inside are byte-identical across runs, +# paths and caches, so the same members in the same order, under their +# basenames, re-archived by this same zig's `ar` in deterministic mode, give +# the same bytes on every host. +normalize() { # + local in="$1" out="$2" fmt="$3" tmp m members=() + while IFS= read -r m; do members+=("$(basename "$m")"); done < <(zig ar t "$in") + [ "${#members[@]}" -gt 0 ] || { echo "no members in $in"; exit 1; } + if [ -n "$(printf '%s\n' "${members[@]}" | sort | uniq -d)" ]; then + echo "two members of $in share a basename; renaming them would collide" + exit 1 + fi + tmp=$(mktemp -d) + # Members come out unreadable (mode 0 in the archive's headers). + ( cd "$tmp" && zig ar x "$in" && chmod 644 ./* ) + for m in "${members[@]}"; do + [ -f "$tmp/$m" ] || { echo "member $m did not extract from $in"; exit 1; } + done + rm -f "$out" + ( cd "$tmp" && zig ar rcsD --format="$fmt" "$out" "${members[@]}" ) + rm -rf "$tmp" +} + # Rust target : zig target. Five, not four — x86_64-unknown-linux-gnu is CI's # test/lint host and is not a release artifact, but the gate cannot build the # crate without it. @@ -50,7 +92,11 @@ for pair in $TARGETS; do src="$WORK/pfx-$rust/lib/libghostty-vt.a" [ -f "$src" ] || { echo "no archive at $src"; exit 1; } mkdir -p "$CRATE/vendor/$rust" - cp "$src" "$CRATE/vendor/$rust/libghostty-vt.a" + case "$rust" in + *-apple-darwin) fmt=darwin ;; + *) fmt=gnu ;; + esac + normalize "$src" "$ROOT/$CRATE/vendor/$rust/libghostty-vt.a" "$fmt" # A size budget rather than a cosmetic strip: this is what actually keeps the # published crate under crates.io's cap, so it fails loudly. sz=$(wc -c < "$CRATE/vendor/$rust/libghostty-vt.a")