secp256k1: drop pure-Python fallback, add reproducible Docker build environment

.github/dependency-review-reviewed-purls.txt

@@ -7,6 +7,8 @@ pkg:githubactions/actions/dependency-review-action
 pkg:githubactions/actions/download-artifact
 pkg:githubactions/actions/setup-python
 pkg:githubactions/actions/upload-artifact
+pkg:githubactions/docker/build-push-action
+pkg:githubactions/docker/setup-buildx-action
 pkg:githubactions/github/codeql-action/analyze
 pkg:githubactions/github/codeql-action/init
 pkg:githubactions/pypa/gh-action-pypi-publish

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
         run: python tools/verify_github_actions_pins.py --self-test
 
   tests:
-    name: Tests (Python ${{ matrix.python-version }})
+    name: Tests (Linux, Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -41,6 +41,8 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v6
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v6
         with:
@@ -54,13 +56,175 @@ jobs:
       - name: Install package
         run: python -m pip install --no-deps -e .
 
-      # TODO: Re-enable Ruff once the repository has a committed lint baseline or
-      # focused cleanup for existing violations. Ruff remains configured and
-      # installed so local cleanup can continue without blocking CI.
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
+
+      - name: Build libsecp builder image
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8
+        with:
+          context: docker
+          load: true
+          tags: embit-libsecp:ci
+          cache-from: type=gha,scope=embit-libsecp
+          cache-to: type=gha,mode=max,scope=embit-libsecp
+
+      - name: Build libsecp256k1 (linux/amd64) via container
+        # --user keeps files written into the bind-mounted workspace owned
+        # by the runner uid instead of root, so subsequent steps (pytest,
+        # SHA256SUMS write, etc.) can read and overwrite them.
+        # build.sh auto-stages the resulting .so at
+        # secp256k1/secp256k1-zkp/.libs/libsecp256k1.so, so no separate
+        # staging step is needed.
+        run: |
+          docker run --rm \
+            --user "$(id -u):$(id -g)" \
+            -v "${{ github.workspace }}":/embit \
+            embit-libsecp:ci amd64
+          file secp256k1/secp256k1-zkp/.libs/libsecp256k1.so
+
+      - name: Tests
+        run: pytest
+
+  tests-macos:
+    name: Tests (macOS native, Python 3.13)
+    runs-on: macos-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+          cache: pip
+          cache-dependency-path: constraints-dev.txt
+
+      - name: Install pinned CI dependencies
+        run: python -m pip install --require-hashes -r constraints-dev.txt
+
+      - name: Install package
+        run: python -m pip install --no-deps -e .
+
+      - name: Install build tools
+        run: brew install autoconf automake libtool pkg-config
+
+      - name: Build libsecp256k1 natively
+        working-directory: secp256k1
+        run: make
+
+      - name: Stage libsecp256k1 where the ctypes loader expects it
+        run: |
+          mkdir -p secp256k1/secp256k1-zkp/.libs
+          # Makefile output filename includes platform/arch; pick whichever .dylib landed.
+          shopt -s nullglob
+          built=(secp256k1/build/libsecp256k1_*.dylib)
+          if [ ${#built[@]} -eq 0 ]; then
+              echo "No libsecp256k1_*.dylib in secp256k1/build/" >&2
+              exit 1
+          fi
+          cp "${built[0]}" secp256k1/secp256k1-zkp/.libs/libsecp256k1.dylib
+          file secp256k1/secp256k1-zkp/.libs/libsecp256k1.dylib
 
       - name: Tests
         run: pytest
 
+  cross-compile-validation:
+    name: Cross-compile validation
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
+
+      - name: Build libsecp builder image
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8
+        with:
+          context: docker
+          load: true
+          tags: embit-libsecp:ci
+          cache-from: type=gha,scope=embit-libsecp
+          cache-to: type=gha,mode=max,scope=embit-libsecp
+
+      - name: Cross-compile every reproducible target
+        # --user keeps the build output owned by the runner uid so the
+        # SHA256SUMS step below can write into secp256k1/build/.
+        # `all` is a build.sh pseudo-target that loops over every supported
+        # arch and continues past individual failures, so this job surfaces
+        # every broken target at once instead of stopping at the first.
+        run: |
+          docker run --rm \
+            --user "$(id -u):$(id -g)" \
+            -v "${{ github.workspace }}":/embit \
+            embit-libsecp:ci all
+
+      - name: Verify each artifact reports the expected architecture
+        run: |
+          set -e
+          ls -la secp256k1/build/
+          file secp256k1/build/libsecp256k1_linux_x86_64.so   | grep -i 'x86-64'
+          file secp256k1/build/libsecp256k1_linux_armv6l.so   | grep -i 'ARM' | grep -i 'EABI5'
+          file secp256k1/build/libsecp256k1_linux_armv7l.so   | grep -i 'ARM' | grep -i 'EABI5'
+          file secp256k1/build/libsecp256k1_linux_aarch64.so  | grep -i 'aarch64'
+          file secp256k1/build/libsecp256k1_windows_amd64.dll | grep -i 'PE32+'
+
+      - name: Compute and publish SHA256SUMS for the produced binaries
+        run: |
+          set -e
+          cd secp256k1/build
+
+          # Gather the reproducible-build inputs so consumers can confirm they
+          # built against the same recipe before comparing hashes.
+          embit_commit="$(git -C "${{ github.workspace }}" rev-parse HEAD)"
+          zkp_commit="$(git -C "${{ github.workspace }}/secp256k1/secp256k1-zkp" rev-parse HEAD)"
+          debian_snapshot="$(grep -E '^ARG DEBIAN_SNAPSHOT=' "${{ github.workspace }}/docker/Dockerfile" | head -n1 | cut -d= -f2)"
+          # Scan every token after `FROM` and pick the one that starts with
+          # `debian:`. Robust against the Dockerfile's optional
+          # `--platform=linux/amd64` token between `FROM` and the image
+          # reference (and against any future flags that might land there).
+          debian_digest="$(awk '/^FROM / {for (i=2; i<=NF; i++) if ($i ~ /^debian:/) {print $i; exit}}' "${{ github.workspace }}/docker/Dockerfile")"
+
+          {
+            echo "# embit commit:        ${embit_commit}"
+            echo "# secp256k1-zkp:       ${zkp_commit}"
+            echo "# debian base image:   ${debian_digest}"
+            echo "# debian snapshot:     ${debian_snapshot}"
+            echo "#"
+            echo "# Verify against your own Docker build with:"
+            echo "#   docker build -t embit-libsecp docker/ && \\"
+            echo "#     docker run --rm -v \"\$PWD\":/embit embit-libsecp <target>"
+            echo "#   cd secp256k1/build && sha256sum -c SHA256SUMS"
+            echo ""
+            sha256sum \
+              libsecp256k1_linux_x86_64.so \
+              libsecp256k1_linux_armv6l.so \
+              libsecp256k1_linux_armv7l.so \
+              libsecp256k1_linux_aarch64.so \
+              libsecp256k1_windows_amd64.dll
+          } > SHA256SUMS
+
+          cat SHA256SUMS
+
+          {
+            echo "## libsecp256k1 SHA256SUMS"
+            echo ""
+            echo '```'
+            cat SHA256SUMS
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload SHA256SUMS as a workflow artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: libsecp256k1-sha256sums
+          path: secp256k1/build/SHA256SUMS
+
   package:
     name: Build and verify artifacts
     runs-on: ubuntu-latest

.gitignore

@@ -135,3 +135,11 @@ settings.json
 src/embit/util/prebuilt/*.so
 src/embit/util/prebuilt/*.dylib
 src/embit/util/prebuilt/*.dll
+
+## Docker builder artifacts: locally-built libsecp256k1 binaries and ccache
+## state. Neither should ever be committed.
+secp256k1/build/
+.ccache/
+
+#OS files
+.DS_Store

README.md

@@ -25,16 +25,29 @@ To install run `pip3 install embit`.
 To install in development mode (editable) clone and run `pip3 install -e .` from the root folder.
 
 PyPI artifacts are pure Python and do not bundle prebuilt `libsecp256k1` binaries.
-If a compatible system `libsecp256k1` is installed, `embit` can use the ctypes backend.
-If no compatible system library is available, `embit` automatically falls back to the pure Python implementation.
+On CPython, `libsecp256k1` is a hard runtime requirement -- there is no pure-Python
+fallback. If the library is missing, `import embit` fails fast with
+`embit.util.secp256k1.Libsecp256k1NotAvailable` (a subclass of `ImportError`)
+whose message describes how to build the library. This is a build/setup
+error, not a runtime condition to be detected later.
+
 ctypes library discovery order is:
 
 1. `secp256k1/secp256k1-zkp/.libs` (repo-local build outputs)
 2. system loader (`libsecp256k1`)
 3. system loader (`secp256k1`)
 4. `src/embit/util/prebuilt/*` (compatibility-only path; no binaries are shipped in package artifacts)
 
-To build and install `libsecp256k1` locally, see: [Building secp256k1 for `embit`](/secp256k1/README.md).
+Build options:
+- **Native build on Linux or macOS** -- the existing Makefile in
+  [`secp256k1/`](/secp256k1/README.md) handles host platform/arch detection.
+  Fast and dependency-light; recommended for macOS and Linux x86_64
+  development.
+- **Docker container** -- the self-contained cross-compile environment in
+  [`docker/`](./docker/README.md) covers ARMv6L (Pi Zero), ARMv7, aarch64,
+  and Windows. Recommended for any embedded ARM target or Windows build.
+  See the Docker README for an honest accounting of what is and isn't
+  reproducible.
 
 
 ## Using non-English BIP39 wordlists

docker/Dockerfile

@@ -0,0 +1,123 @@
+# Reproducible build environment for libsecp256k1 (the C library that the embit
+# ctypes backend wraps). Replaces shipped prebuilt binaries: consumers run this
+# container to compile their own .so/.dll for their target.
+#
+# Targets supported (Linux + Windows + ARM):
+#   - linux/amd64        (x86_64)
+#   - linux/arm/v6       (Pi Zero -- armv6l)
+#   - linux/arm/v7       (Pi 2/3 -- armv7l)
+#   - linux/arm64        (aarch64)
+#   - windows/amd64      (mingw cross to .dll)
+#
+# macOS targets are deliberately NOT supported by this image -- Apple's
+# licensing makes Mach-O cross-compilation from Linux awkward, and the native
+# `make` workflow on a Mac is fast enough that the isolation gain isn't worth
+# the third-party toolchain dependency. See docker/README.md for the macOS
+# instructions.
+#
+# This image is NOT invoked automatically by `pip install embit`. It is a
+# deliberate manual step. See docker/README.md.
+#
+# syntax=docker/dockerfile:1.6
+
+# --platform=linux/amd64 forces the image (and any docker run against it) to
+# the same host architecture CI uses. Without this, Docker Desktop on Apple
+# Silicon would build a linux/arm64 image variant, gcc would run as arm64,
+# and even with cross-compilers the produced binaries differ bit-for-bit
+# from CI's linux/amd64 build for the amd64/aarch64/windows targets (the
+# cross-compiler binary itself embeds host-arch-dependent metadata in
+# debug-info / BuildID slots). Pinning the image to linux/amd64 makes
+# every consumer's build reproducible against CI; on non-amd64 hosts the
+# build runs under QEMU (slower but deterministic).
+#
+# Pinned by digest so a clean rebuild gets the same Debian image bits as
+# the previous one. Bumping this is a deliberate maintainer action --
+# weigh the security updates in the new digest against any toolchain
+# churn it pulls in.
+# To bump: `docker pull --platform=linux/amd64 debian:bookworm-slim && docker inspect --format='{{index .RepoDigests 0}}' debian:bookworm-slim`.
+FROM --platform=linux/amd64 debian:bookworm-slim@sha256:96e378d7e6531ac9a15ad505478fcc2e69f371b10f5cdf87857c4b8188404716
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Point apt at snapshot.debian.org so package versions are deterministic.
+# Every `apt-get install <name>` against this snapshot resolves to the same
+# version every time, until a maintainer bumps DEBIAN_SNAPSHOT below.
+#
+# This image is a transient build environment processing trusted input
+# (SHA-pinned secp256k1 source); it doesn't run as a service or process
+# untrusted data, so we deliberately trade automatic security-update
+# pickup for fully-deterministic builds.
+#
+# To bump the snapshot: pick a recent date from
+# https://snapshot.debian.org/archive/debian/?year=...&month=...
+# (any YYYYMMDDTHHMMSSZ that resolves -- apt will follow snapshot.debian.org's
+# internal redirect to the nearest actual snapshot).
+ARG DEBIAN_SNAPSHOT=20260601T000000Z
+
+# snapshot.debian.org supports HTTPS, but debian:bookworm-slim ships
+# without ca-certificates -- so the first apt-get update has to go over
+# HTTP. We bootstrap ca-certificates from the snapshot's HTTP endpoint
+# (content integrity is still provided by apt's GPG signature chain),
+# then swap the sources.list to HTTPS for the larger install in the next
+# RUN layer. This adds transport-layer protection (metadata
+# confidentiality + an extra integrity check) without sacrificing the
+# snapshot-pinned reproducibility for ca-certificates itself.
+RUN echo "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/ bookworm main"                   >  /etc/apt/sources.list && \
+    echo "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/${DEBIAN_SNAPSHOT}/ bookworm-security main" >> /etc/apt/sources.list && \
+    echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/10-snapshot && \
+    echo 'Acquire::Retries "5";'              >> /etc/apt/apt.conf.d/10-snapshot && \
+    apt-get update && \
+    apt-get install --no-install-recommends -y ca-certificates && \
+    rm -rf /var/lib/apt/lists/* && \
+    sed -i 's|http://snapshot.debian.org|https://snapshot.debian.org|g' /etc/apt/sources.list
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        autoconf \
+        automake \
+        build-essential \
+        ca-certificates \
+        ccache \
+        file \
+        git \
+        gcc-aarch64-linux-gnu \
+        gcc-arm-linux-gnueabihf \
+        gcc-mingw-w64-x86-64 \
+        gcc-x86-64-linux-gnu \
+        libc6-dev-amd64-cross \
+        libc6-dev-arm64-cross \
+        libc6-dev-armhf-cross \
+        libtool \
+        make \
+        pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+# Wire ccache symlinks for every cross toolchain. Debian's `ccache` package only
+# auto-symlinks the native `gcc`; cross-compilers need explicit symlinks so the
+# Makefile's `$(TOOLCHAIN_PREFIX)gcc` invocation routes through ccache without
+# any Makefile changes. CCACHE_DIR is bind-mountable so consumers can persist
+# the cache between container runs (e.g. -v $PWD/.ccache:/ccache).
+RUN set -eux; \
+    for triple in arm-linux-gnueabihf aarch64-linux-gnu x86_64-linux-gnu x86_64-w64-mingw32; do \
+        ln -sf /usr/bin/ccache /usr/lib/ccache/${triple}-gcc; \
+    done
+
+ENV PATH="/usr/lib/ccache:${PATH}"
+ENV CCACHE_DIR=/ccache
+
+# Pre-create the ccache directory world-writable (sticky bit, like /tmp) so
+# the container works correctly whether it runs as root, as the host uid via
+# --user, or with a bind-mounted host directory at /ccache. Without this,
+# ccache's "create /ccache/tmp" probe fails with Permission denied when the
+# container runs as a non-root uid against the image's own /ccache.
+RUN mkdir -p /ccache && chmod 1777 /ccache
+
+WORKDIR /embit
+
+COPY build.sh /usr/local/bin/build.sh
+RUN chmod +x /usr/local/bin/build.sh
+
+# Default entrypoint exposes the wrapper script. Override with `--entrypoint`
+# for ad-hoc shell access.
+ENTRYPOINT ["/usr/local/bin/build.sh"]
+CMD ["--help"]