From c6094c16eae8822023aac288bfa372273b344ae4 Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 10:18:20 -0500 Subject: [PATCH 01/11] :sparkles: Ship standalone wk-* binaries on each release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a PyInstaller --onedir bundle with all seven wk-* CLIs sharing one embedded Python + dependency tree, attached as a versioned zip per arch on every published release. Builds on every PR / push / release / workflow_dispatch (so PRs catch breakage early); the publish step only fires on release: published. - packaging/pyinstaller/{warpkit.spec,build_bundle.py,launchers/} drive the build. Launchers each call multiprocessing.freeze_support() up front; PyInstaller's user runtime_hooks run before its built-in ones, so calling freeze_support from a custom hook hits the unpatched no-op and any wk-* using a process pool dies with BrokenProcessPool. - Linux builds run inside quay.io/pypa/manylinux2014_{x86_64,aarch64} to keep the glibc floor identical to the wheels (2.17). - macOS arm64 binaries are ad-hoc signed; bundle README documents the Gatekeeper xattr workaround. - Smoke test runs --version on all 7 binaries plus a real wk-unwrap-phase end-to-end against tests/data/test_data/ — exercises numpy, scipy, skimage, the C++ extension, and the multiprocessing worker pool. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/build.yml | 121 ++++++++++++++ .gitignore | 2 + packaging/pyinstaller/build_bundle.py | 157 ++++++++++++++++++ packaging/pyinstaller/bundle_README.md | 49 ++++++ .../pyinstaller/launchers/wk-apply-warp.py | 7 + .../launchers/wk-compute-fieldmap.py | 7 + .../launchers/wk-compute-jacobian.py | 7 + .../launchers/wk-convert-fieldmap.py | 7 + .../pyinstaller/launchers/wk-convert-warp.py | 7 + packaging/pyinstaller/launchers/wk-medic.py | 7 + .../pyinstaller/launchers/wk-unwrap-phase.py | 7 + packaging/pyinstaller/warpkit.spec | 79 +++++++++ 12 files changed, 457 insertions(+) create mode 100644 packaging/pyinstaller/build_bundle.py create mode 100644 packaging/pyinstaller/bundle_README.md create mode 100644 packaging/pyinstaller/launchers/wk-apply-warp.py create mode 100644 packaging/pyinstaller/launchers/wk-compute-fieldmap.py create mode 100644 packaging/pyinstaller/launchers/wk-compute-jacobian.py create mode 100644 packaging/pyinstaller/launchers/wk-convert-fieldmap.py create mode 100644 packaging/pyinstaller/launchers/wk-convert-warp.py create mode 100644 packaging/pyinstaller/launchers/wk-medic.py create mode 100644 packaging/pyinstaller/launchers/wk-unwrap-phase.py create mode 100644 packaging/pyinstaller/warpkit.spec diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 26e5bdc..49f72a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,6 +10,7 @@ on: release: types: - published + workflow_dispatch: jobs: lint: @@ -66,6 +67,126 @@ jobs: with: name: wheel-${{ matrix.os }}-cp${{ matrix.python-versions.version }} path: ./wheelhouse/*.whl + binaries-build-linux: + name: PyInstaller bundle (${{ matrix.target }}) + needs: [wheels-build] + strategy: + fail-fast: false + matrix: + include: + - target: linux-x86_64 + runner: ubuntu-latest + container: quay.io/pypa/manylinux2014_x86_64 + wheel-artifact: wheel-ubuntu-latest-cp3.11 + - target: linux-aarch64 + runner: ubuntu-24.04-arm + container: quay.io/pypa/manylinux2014_aarch64 + wheel-artifact: wheel-ubuntu-24.04-arm-cp3.11 + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Download wheel + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.wheel-artifact }} + path: wheelhouse + - name: Install warpkit + PyInstaller + run: | + /opt/python/cp311-cp311/bin/python -m venv .venv + .venv/bin/pip install --upgrade pip + .venv/bin/pip install ./wheelhouse/*.whl pyinstaller + - name: Build bundle + run: .venv/bin/python packaging/pyinstaller/build_bundle.py --target ${{ matrix.target }} + - name: Smoke test (--version per binary) + run: | + BUNDLE=$(ls -d packaging/pyinstaller/dist/warpkit-*/ | head -1) + for bin in "$BUNDLE"wk-*; do + "$bin" --version + done + - name: Smoke test (wk-unwrap-phase end-to-end) + run: | + BUNDLE=$(ls -d packaging/pyinstaller/dist/warpkit-*/ | head -1) + mkdir -p smoke-out + "$BUNDLE/wk-unwrap-phase" \ + --magnitude tests/data/test_data/*part-mag_bold.nii.gz \ + --phase tests/data/test_data/*part-phase_bold.nii.gz \ + --metadata tests/data/test_data/*part-mag_bold.json \ + --out-prefix smoke-out/unwrap + ls smoke-out/ + - uses: actions/upload-artifact@v7 + with: + name: binaries-${{ matrix.target }} + path: packaging/pyinstaller/dist/warpkit-*-${{ matrix.target }}.zip + if-no-files-found: error + binaries-build-macos: + name: PyInstaller bundle (macos-arm64) + needs: [wheels-build] + runs-on: macos-latest + env: + MACOSX_DEPLOYMENT_TARGET: "11.0" + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Download wheel + uses: actions/download-artifact@v8 + with: + name: wheel-macos-latest-cp3.11 + path: wheelhouse + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + - name: Set up Python + run: uv python install 3.11 + - name: Install warpkit + PyInstaller + run: | + uv venv --python 3.11 .venv + uv pip install --python .venv/bin/python ./wheelhouse/*.whl pyinstaller + - name: Build bundle + run: .venv/bin/python packaging/pyinstaller/build_bundle.py --target macos-arm64 + - name: Smoke test (--version per binary) + run: | + BUNDLE=$(ls -d packaging/pyinstaller/dist/warpkit-*/ | head -1) + for bin in "$BUNDLE"wk-*; do + "$bin" --version + done + - name: Smoke test (wk-unwrap-phase end-to-end) + run: | + BUNDLE=$(ls -d packaging/pyinstaller/dist/warpkit-*/ | head -1) + mkdir -p smoke-out + "$BUNDLE/wk-unwrap-phase" \ + --magnitude tests/data/test_data/*part-mag_bold.nii.gz \ + --phase tests/data/test_data/*part-phase_bold.nii.gz \ + --metadata tests/data/test_data/*part-mag_bold.json \ + --out-prefix smoke-out/unwrap + ls smoke-out/ + - uses: actions/upload-artifact@v7 + with: + name: binaries-macos-arm64 + path: packaging/pyinstaller/dist/warpkit-*-macos-arm64.zip + if-no-files-found: error + binaries-publish: + name: Attach binaries to release + if: github.event_name == 'release' && github.event.action == 'published' + needs: [binaries-build-linux, binaries-build-macos] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v8 + with: + path: bins + pattern: binaries-* + merge-multiple: true + - name: Upload zips to release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "${{ github.event.release.tag_name }}" bins/*.zip --clobber --repo "${{ github.repository }}" sdist-build: name: Sdist and coverage runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index a1910b9..42ae0a7 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,8 @@ MANIFEST # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec +# but our committed bundle spec is checked in +!packaging/pyinstaller/warpkit.spec # Installer logs pip-log.txt diff --git a/packaging/pyinstaller/build_bundle.py b/packaging/pyinstaller/build_bundle.py new file mode 100644 index 0000000..ad585d6 --- /dev/null +++ b/packaging/pyinstaller/build_bundle.py @@ -0,0 +1,157 @@ +"""Drive PyInstaller to produce a versioned --onedir bundle + zip for one target. + +Usage (from repo root, inside an env with warpkit + pyinstaller installed): + + python packaging/pyinstaller/build_bundle.py --target linux-x86_64 + +Produces: + packaging/pyinstaller/dist/warpkit-${VERSION}/ (the bundle) + packaging/pyinstaller/dist/warpkit-${VERSION}-${TARGET}.zip +""" + +from __future__ import annotations + +import argparse +import platform +import shutil +import subprocess +import sys +from pathlib import Path + +SCRIPTS = [ + "wk-medic", + "wk-unwrap-phase", + "wk-compute-fieldmap", + "wk-apply-warp", + "wk-convert-warp", + "wk-convert-fieldmap", + "wk-compute-jacobian", +] + + +def detect_target() -> str: + system = platform.system().lower() + machine = platform.machine().lower() + if system == "linux": + if machine in ("x86_64", "amd64"): + return "linux-x86_64" + if machine in ("aarch64", "arm64"): + return "linux-aarch64" + if system == "darwin": + if machine in ("arm64", "aarch64"): + return "macos-arm64" + if machine in ("x86_64", "amd64"): + return "macos-x86_64" + raise RuntimeError(f"unsupported target: {system}/{machine}") + + +def get_version() -> str: + from warpkit import __version__ + + return __version__ + + +def run_pyinstaller(spec: Path, dist: Path, work: Path) -> None: + cmd = [ + sys.executable, + "-m", + "PyInstaller", + "--noconfirm", + "--clean", + "--distpath", + str(dist), + "--workpath", + str(work), + str(spec), + ] + subprocess.run(cmd, check=True) + + +def strip_binaries(bundle: Path) -> None: + # COLLECT's strip= can mangle some libs; strip explicitly here on Linux only. + if platform.system().lower() != "linux": + return + strip = shutil.which("strip") + if strip is None: + return + for so in bundle.rglob("*.so*"): + if so.is_file() and not so.is_symlink(): + subprocess.run([strip, "--strip-unneeded", str(so)], check=False) + + +def adhoc_sign_macos(bundle: Path) -> None: + if platform.system().lower() != "darwin": + return + # Ad-hoc sign every Mach-O in the bundle. `codesign -s -` is sufficient to + # let users override Gatekeeper after the first launch (right-click → Open, + # or `xattr -d com.apple.quarantine`). + targets = [bundle / name for name in SCRIPTS] + targets.extend(p for p in bundle.rglob("*.dylib") if p.is_file()) + targets.extend(p for p in (bundle / "_internal").rglob("*.so") if p.is_file()) + for t in targets: + subprocess.run( + ["codesign", "--force", "--sign", "-", "--timestamp=none", str(t)], + check=False, + ) + + +def write_readme(bundle: Path, version: str, target: str) -> None: + template = Path(__file__).parent / "bundle_README.md" + body = ( + template.read_text().replace("@VERSION@", version).replace("@TARGET@", target) + ) + (bundle / "README.md").write_text(body) + + +def make_zip(bundle: Path, out_zip: Path) -> None: + # shutil.make_archive's base_dir keeps a tidy top-level folder inside the zip. + base_name = str(out_zip.with_suffix("")) + shutil.make_archive( + base_name=base_name, + format="zip", + root_dir=str(bundle.parent), + base_dir=bundle.name, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--target", + default=None, + help="target triple (e.g. linux-x86_64); auto-detected by default", + ) + args = parser.parse_args() + + here = Path(__file__).parent + spec = here / "warpkit.spec" + dist = here / "dist" + work = here / "build" + target = args.target or detect_target() + version = get_version() + + if dist.exists(): + shutil.rmtree(dist) + if work.exists(): + shutil.rmtree(work) + + run_pyinstaller(spec, dist, work) + + raw_bundle = dist / "warpkit" + if not raw_bundle.is_dir(): + raise RuntimeError(f"PyInstaller did not produce {raw_bundle}") + versioned = dist / f"warpkit-{version}" + raw_bundle.rename(versioned) + + strip_binaries(versioned) + adhoc_sign_macos(versioned) + write_readme(versioned, version, target) + + out_zip = dist / f"warpkit-{version}-{target}.zip" + make_zip(versioned, out_zip) + print(f"wrote {out_zip}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/pyinstaller/bundle_README.md b/packaging/pyinstaller/bundle_README.md new file mode 100644 index 0000000..432400b --- /dev/null +++ b/packaging/pyinstaller/bundle_README.md @@ -0,0 +1,49 @@ +# warpkit @VERSION@ — standalone binaries (@TARGET@) + +This bundle contains the seven `wk-*` CLIs as standalone binaries. No Python +install or system ITK required — everything is in `_internal/`. + +## Install + +Extract anywhere and put the bundle directory on your `PATH`: + +```sh +# example: install to /opt +sudo mv warpkit-@VERSION@ /opt/ +echo 'export PATH=/opt/warpkit-@VERSION@:$PATH' >> ~/.bashrc +``` + +Or symlink each `wk-*` onto an existing `PATH` entry — the bootloader resolves +`_internal/` relative to the real binary, so symlinks work fine: + +```sh +for bin in /opt/warpkit-@VERSION@/wk-*; do + sudo ln -s "$bin" /usr/local/bin/$(basename "$bin") +done +``` + +**Do not separate the binaries from `_internal/`** — they all share the +embedded interpreter and dependency tree. + +## macOS Gatekeeper + +Binaries are ad-hoc signed, not Apple-notarized, so on first launch macOS will +refuse to run them. Either: + +```sh +xattr -d com.apple.quarantine /opt/warpkit-@VERSION@/wk-* +``` + +or right-click → Open the first time on each binary. + +## Available CLIs + +- `wk-medic` — full MEDIC distortion correction pipeline +- `wk-unwrap-phase` — ROMEO multi-echo phase unwrapping +- `wk-compute-fieldmap` — compute B0 field map from unwrapped phase +- `wk-apply-warp` — apply a displacement field to an image +- `wk-convert-warp` — convert between warp field conventions +- `wk-convert-fieldmap` — convert between field-map representations +- `wk-compute-jacobian` — compute the Jacobian determinant of a warp + +Run any of them with `--help` for usage. diff --git a/packaging/pyinstaller/launchers/wk-apply-warp.py b/packaging/pyinstaller/launchers/wk-apply-warp.py new file mode 100644 index 0000000..7d2dd34 --- /dev/null +++ b/packaging/pyinstaller/launchers/wk-apply-warp.py @@ -0,0 +1,7 @@ +import multiprocessing + +multiprocessing.freeze_support() + +from warpkit.scripts.apply_warp import main # noqa: E402 + +main() diff --git a/packaging/pyinstaller/launchers/wk-compute-fieldmap.py b/packaging/pyinstaller/launchers/wk-compute-fieldmap.py new file mode 100644 index 0000000..ca5baee --- /dev/null +++ b/packaging/pyinstaller/launchers/wk-compute-fieldmap.py @@ -0,0 +1,7 @@ +import multiprocessing + +multiprocessing.freeze_support() + +from warpkit.scripts.compute_fieldmap import main # noqa: E402 + +main() diff --git a/packaging/pyinstaller/launchers/wk-compute-jacobian.py b/packaging/pyinstaller/launchers/wk-compute-jacobian.py new file mode 100644 index 0000000..1951da2 --- /dev/null +++ b/packaging/pyinstaller/launchers/wk-compute-jacobian.py @@ -0,0 +1,7 @@ +import multiprocessing + +multiprocessing.freeze_support() + +from warpkit.scripts.compute_jacobian import main # noqa: E402 + +main() diff --git a/packaging/pyinstaller/launchers/wk-convert-fieldmap.py b/packaging/pyinstaller/launchers/wk-convert-fieldmap.py new file mode 100644 index 0000000..803567c --- /dev/null +++ b/packaging/pyinstaller/launchers/wk-convert-fieldmap.py @@ -0,0 +1,7 @@ +import multiprocessing + +multiprocessing.freeze_support() + +from warpkit.scripts.convert_fieldmap import main # noqa: E402 + +main() diff --git a/packaging/pyinstaller/launchers/wk-convert-warp.py b/packaging/pyinstaller/launchers/wk-convert-warp.py new file mode 100644 index 0000000..b29ab98 --- /dev/null +++ b/packaging/pyinstaller/launchers/wk-convert-warp.py @@ -0,0 +1,7 @@ +import multiprocessing + +multiprocessing.freeze_support() + +from warpkit.scripts.convert_warp import main # noqa: E402 + +main() diff --git a/packaging/pyinstaller/launchers/wk-medic.py b/packaging/pyinstaller/launchers/wk-medic.py new file mode 100644 index 0000000..4dc023d --- /dev/null +++ b/packaging/pyinstaller/launchers/wk-medic.py @@ -0,0 +1,7 @@ +import multiprocessing + +multiprocessing.freeze_support() + +from warpkit.scripts.medic import main # noqa: E402 + +main() diff --git a/packaging/pyinstaller/launchers/wk-unwrap-phase.py b/packaging/pyinstaller/launchers/wk-unwrap-phase.py new file mode 100644 index 0000000..ad598f1 --- /dev/null +++ b/packaging/pyinstaller/launchers/wk-unwrap-phase.py @@ -0,0 +1,7 @@ +import multiprocessing + +multiprocessing.freeze_support() + +from warpkit.scripts.unwrap_phase import main # noqa: E402 + +main() diff --git a/packaging/pyinstaller/warpkit.spec b/packaging/pyinstaller/warpkit.spec new file mode 100644 index 0000000..9209b34 --- /dev/null +++ b/packaging/pyinstaller/warpkit.spec @@ -0,0 +1,79 @@ +# PyInstaller multi-binary spec: one --onedir bundle, all wk-* CLIs share _internal/. +# Driven by packaging/pyinstaller/build_bundle.py. + +from pathlib import Path + +LAUNCHERS_DIR = Path(SPECPATH) / "launchers" + +SCRIPTS = [ + "wk-medic", + "wk-unwrap-phase", + "wk-compute-fieldmap", + "wk-apply-warp", + "wk-convert-warp", + "wk-convert-fieldmap", + "wk-compute-jacobian", +] + +# nibabel + indexed_gzip rely on string-based imports that PyInstaller's static +# analysis misses; everything else (numpy/scipy/skimage/transforms3d) has a hook +# shipped with PyInstaller. +HIDDEN_IMPORTS = [ + "indexed_gzip", + "nibabel.streamlines", + "nibabel.nifti1", + "nibabel.nifti2", +] + +analyses = [] +for name in SCRIPTS: + a = Analysis( + [str(LAUNCHERS_DIR / f"{name}.py")], + pathex=[], + binaries=[], + datas=[], + hiddenimports=HIDDEN_IMPORTS, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + ) + analyses.append(a) + +# Deduplicate shared libraries/data across all analyses so _internal/ has one copy. +MERGE(*[(a, name, name) for a, name in zip(analyses, SCRIPTS)]) + +exe_list = [] +for a, name in zip(analyses, SCRIPTS): + pyz = PYZ(a.pure, a.zipped_data) + exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name=name, + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=False, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, + ) + exe_list.append(exe) + +collect_args = list(exe_list) +for a in analyses: + collect_args.extend([a.binaries, a.zipfiles, a.datas]) + +COLLECT( + *collect_args, + strip=False, + upx=False, + upx_exclude=[], + name="warpkit", +) From e05341c909e63da3ce070056443141be27313d3e Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 10:28:32 -0500 Subject: [PATCH 02/11] :bug: Use manylinux_2_28 for binaries job (glibc 2.17 too old for Node 24) actions/checkout@v6 ships with Node 24, which requires glibc 2.27+; manylinux2014 is based on CentOS 7 with glibc 2.17, so Node fails to load before any of our build steps run. cibuildwheel 3.x already defaults to manylinux_2_28 for cp39+, so the binaries' glibc floor (2.28) now matches the wheels' rather than being more conservative than them. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 49f72a2..5b6217c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,11 +76,11 @@ jobs: include: - target: linux-x86_64 runner: ubuntu-latest - container: quay.io/pypa/manylinux2014_x86_64 + container: quay.io/pypa/manylinux_2_28_x86_64 wheel-artifact: wheel-ubuntu-latest-cp3.11 - target: linux-aarch64 runner: ubuntu-24.04-arm - container: quay.io/pypa/manylinux2014_aarch64 + container: quay.io/pypa/manylinux_2_28_aarch64 wheel-artifact: wheel-ubuntu-24.04-arm-cp3.11 runs-on: ${{ matrix.runner }} container: ${{ matrix.container }} From 615423d50dd37149204ce7840ebb1c18aed54364 Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 10:45:47 -0500 Subject: [PATCH 03/11] :bug: Fix Linux + macOS PyInstaller failures in CI Two unrelated failures from the first CI run: 1. Linux: manylinux's /opt/python/cp311-cp311 is built without --enable-shared, so PyInstaller cannot find libpython3.11.so. Switch to uv-managed Python (built --enable-shared, matching the macOS job). 2. macOS: warpkit_cpp.cpython-*-darwin.so was missing from the bundle when warpkit is wheel-installed (works locally only because the editable install routes the import through the repo build dir so PyInstaller's import-graph picks it up; a wheel install does not). Add a hook that explicitly collects the .so. Note default collect_dynamic_libs search_patterns is `lib*.so` which excludes our extension since it has no `lib` prefix; passing `*.so` explicitly fixes it. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/build.yml | 9 ++++++--- packaging/pyinstaller/hooks/hook-warpkit.py | 9 +++++++++ packaging/pyinstaller/warpkit.spec | 3 ++- 3 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 packaging/pyinstaller/hooks/hook-warpkit.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5b6217c..9bfbe21 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,11 +94,14 @@ jobs: with: name: ${{ matrix.wheel-artifact }} path: wheelhouse + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + - name: Set up Python + run: uv python install 3.11 - name: Install warpkit + PyInstaller run: | - /opt/python/cp311-cp311/bin/python -m venv .venv - .venv/bin/pip install --upgrade pip - .venv/bin/pip install ./wheelhouse/*.whl pyinstaller + uv venv --python 3.11 .venv + uv pip install --python .venv/bin/python ./wheelhouse/*.whl pyinstaller - name: Build bundle run: .venv/bin/python packaging/pyinstaller/build_bundle.py --target ${{ matrix.target }} - name: Smoke test (--version per binary) diff --git a/packaging/pyinstaller/hooks/hook-warpkit.py b/packaging/pyinstaller/hooks/hook-warpkit.py new file mode 100644 index 0000000..be71503 --- /dev/null +++ b/packaging/pyinstaller/hooks/hook-warpkit.py @@ -0,0 +1,9 @@ +from PyInstaller.utils.hooks import collect_dynamic_libs + +# Default search_patterns is `lib*.so` (system-style libs) which misses +# `warpkit_cpp.cpython-*.so` since it has no `lib` prefix. The local repo +# build picks the .so up via PyInstaller's import-graph analysis, but a +# wheel-installed warpkit (what CI / users have) does not — so force the +# bundle here. +binaries = collect_dynamic_libs("warpkit", search_patterns=["*.so", "*.dylib", "*.dll"]) +hiddenimports = ["warpkit.warpkit_cpp"] diff --git a/packaging/pyinstaller/warpkit.spec b/packaging/pyinstaller/warpkit.spec index 9209b34..1f10b2c 100644 --- a/packaging/pyinstaller/warpkit.spec +++ b/packaging/pyinstaller/warpkit.spec @@ -4,6 +4,7 @@ from pathlib import Path LAUNCHERS_DIR = Path(SPECPATH) / "launchers" +HOOKS_DIR = Path(SPECPATH) / "hooks" SCRIPTS = [ "wk-medic", @@ -33,7 +34,7 @@ for name in SCRIPTS: binaries=[], datas=[], hiddenimports=HIDDEN_IMPORTS, - hookspath=[], + hookspath=[str(HOOKS_DIR)], hooksconfig={}, runtime_hooks=[], excludes=[], From 0f7684dc6dff43cef68335f40b8c271169dd066e Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 10:47:51 -0500 Subject: [PATCH 04/11] :bug: Exclude packaging/ from setuptools package discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running build_bundle.py creates packaging/pyinstaller/build/warpkit/ (PyInstaller's workdir). Without an explicit exclude, setuptools' packages.find walks into it, sees a `warpkit` directory, and treats it as the warpkit package — failing on `packaging/pyinstaller/build/warpkit/localpycs` does not exist during subsequent uv sync runs. Co-Authored-By: Claude Opus 4.7 (1M context) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f677180..9c466b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ build-backend = "setuptools.build_meta" zip-safe = true [tool.setuptools.packages.find] -exclude = ["tests"] +exclude = ["tests", "packaging*"] [tool.setuptools.package-data] warpkit = ["py.typed", "*.pyi"] From 142d3468d7cf3fa13b17dae7b9e090ba4bb2e503 Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 11:06:01 -0500 Subject: [PATCH 05/11] :bug: Explicitly resolve warpkit_cpp via importlib.util.find_spec collect_dynamic_libs was returning empty even with explicit *.so search patterns when warpkit is wheel-installed in CI. Switch to importlib's import resolution which finds the .so regardless of install layout, and add diagnostic prints so the next failure is debuggable from the build log. Co-Authored-By: Claude Opus 4.7 (1M context) --- packaging/pyinstaller/hooks/hook-warpkit.py | 37 ++++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/packaging/pyinstaller/hooks/hook-warpkit.py b/packaging/pyinstaller/hooks/hook-warpkit.py index be71503..0908117 100644 --- a/packaging/pyinstaller/hooks/hook-warpkit.py +++ b/packaging/pyinstaller/hooks/hook-warpkit.py @@ -1,9 +1,30 @@ -from PyInstaller.utils.hooks import collect_dynamic_libs - -# Default search_patterns is `lib*.so` (system-style libs) which misses -# `warpkit_cpp.cpython-*.so` since it has no `lib` prefix. The local repo -# build picks the .so up via PyInstaller's import-graph analysis, but a -# wheel-installed warpkit (what CI / users have) does not — so force the -# bundle here. -binaries = collect_dynamic_libs("warpkit", search_patterns=["*.so", "*.dylib", "*.dll"]) +import importlib.util +import sys + +from PyInstaller.utils.hooks import collect_dynamic_libs, get_package_paths + +# Resolve the C extension via Python's own importlib — this catches the .so +# regardless of whether warpkit is editable- or wheel-installed and whether +# the file matches PyInstaller's `lib*.so` default pattern (it does not, since +# the extension is `warpkit_cpp.cpython-*.so` — no `lib` prefix). +binaries = [] +spec = importlib.util.find_spec("warpkit.warpkit_cpp") +if spec is not None and spec.origin: + binaries.append((spec.origin, "warpkit")) + +# Diagnostic prints so any future bundle-without-the-.so failure is debuggable +# from the build log. +print( + f"[hook-warpkit] find_spec origin: {spec.origin if spec else None}", file=sys.stderr +) +print( + f"[hook-warpkit] get_package_paths: {get_package_paths('warpkit')}", file=sys.stderr +) +print( + f"[hook-warpkit] collect_dynamic_libs(*.so): " + f"{collect_dynamic_libs('warpkit', search_patterns=['*.so', '*.dylib'])}", + file=sys.stderr, +) +print(f"[hook-warpkit] binaries: {binaries}", file=sys.stderr) + hiddenimports = ["warpkit.warpkit_cpp"] From c55b3ea504c97aaf4c546c26456b6fe2a3765968 Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 11:27:59 -0500 Subject: [PATCH 06/11] :wrench: Build PyInstaller bundles from editable install, drop wheel dep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local repro showed the wheel-install path was tripping over the source tree's `warpkit/` being on sys.path (CWD=repo root) but lacking the .so next to it: PyInstaller's import-graph analyzer found the source tree first and never consulted the venv's wheel-installed copy. Switch the binaries jobs to `uv sync --group dev` (editable install via cmake-build-extension), which drops the .so right into the source tree's `warpkit/` directory — exactly where the analyzer is already looking. Side benefits: - No more `needs: [wheels-build]`, so binaries build in parallel with wheels instead of waiting for the slowest matrix entry. - One less artifact dance (download wheel → install). - The source-tree build is the same path local devs use, so any future CI failure is reproducible with `uv sync` + the build script. The hook-warpkit.py keeps a simple importlib-based binary registration as defense-in-depth (and so it works for both editable and wheel installs); the diagnostic prints are gone. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/build.yml | 30 ++++++--------------- packaging/pyinstaller/hooks/hook-warpkit.py | 29 +++++--------------- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bfbe21..452ba93 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,7 +69,6 @@ jobs: path: ./wheelhouse/*.whl binaries-build-linux: name: PyInstaller bundle (${{ matrix.target }}) - needs: [wheels-build] strategy: fail-fast: false matrix: @@ -77,11 +76,9 @@ jobs: - target: linux-x86_64 runner: ubuntu-latest container: quay.io/pypa/manylinux_2_28_x86_64 - wheel-artifact: wheel-ubuntu-latest-cp3.11 - target: linux-aarch64 runner: ubuntu-24.04-arm container: quay.io/pypa/manylinux_2_28_aarch64 - wheel-artifact: wheel-ubuntu-24.04-arm-cp3.11 runs-on: ${{ matrix.runner }} container: ${{ matrix.container }} steps: @@ -89,19 +86,14 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - - name: Download wheel - uses: actions/download-artifact@v8 - with: - name: ${{ matrix.wheel-artifact }} - path: wheelhouse - name: Install uv uses: astral-sh/setup-uv@v8.1.0 - name: Set up Python run: uv python install 3.11 - - name: Install warpkit + PyInstaller - run: | - uv venv --python 3.11 .venv - uv pip install --python .venv/bin/python ./wheelhouse/*.whl pyinstaller + - name: Build extension + install dev deps + run: uv sync --group dev --config-setting editable_mode=strict -v + - name: Install PyInstaller + run: uv pip install --python .venv/bin/python pyinstaller - name: Build bundle run: .venv/bin/python packaging/pyinstaller/build_bundle.py --target ${{ matrix.target }} - name: Smoke test (--version per binary) @@ -127,7 +119,6 @@ jobs: if-no-files-found: error binaries-build-macos: name: PyInstaller bundle (macos-arm64) - needs: [wheels-build] runs-on: macos-latest env: MACOSX_DEPLOYMENT_TARGET: "11.0" @@ -136,19 +127,14 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - - name: Download wheel - uses: actions/download-artifact@v8 - with: - name: wheel-macos-latest-cp3.11 - path: wheelhouse - name: Install uv uses: astral-sh/setup-uv@v8.1.0 - name: Set up Python run: uv python install 3.11 - - name: Install warpkit + PyInstaller - run: | - uv venv --python 3.11 .venv - uv pip install --python .venv/bin/python ./wheelhouse/*.whl pyinstaller + - name: Build extension + install dev deps + run: uv sync --group dev --config-setting editable_mode=strict -v + - name: Install PyInstaller + run: uv pip install --python .venv/bin/python pyinstaller - name: Build bundle run: .venv/bin/python packaging/pyinstaller/build_bundle.py --target macos-arm64 - name: Smoke test (--version per binary) diff --git a/packaging/pyinstaller/hooks/hook-warpkit.py b/packaging/pyinstaller/hooks/hook-warpkit.py index 0908117..26bb8c1 100644 --- a/packaging/pyinstaller/hooks/hook-warpkit.py +++ b/packaging/pyinstaller/hooks/hook-warpkit.py @@ -1,30 +1,15 @@ import importlib.util -import sys -from PyInstaller.utils.hooks import collect_dynamic_libs, get_package_paths - -# Resolve the C extension via Python's own importlib — this catches the .so -# regardless of whether warpkit is editable- or wheel-installed and whether -# the file matches PyInstaller's `lib*.so` default pattern (it does not, since -# the extension is `warpkit_cpp.cpython-*.so` — no `lib` prefix). +# PyInstaller's static analysis usually picks up `warpkit_cpp.cpython-*.so` via +# the import graph (`from .warpkit_cpp import *` in warpkit/__init__.py), but +# only when the resolved warpkit package directory has the .so right next to +# __init__.py — which is the case for an editable install (cmake-build-extension +# drops the .so into the source tree) and for a wheel install (the .so lives in +# site-packages/warpkit/). Resolve the .so explicitly via importlib.util so the +# hook works regardless of install layout. binaries = [] spec = importlib.util.find_spec("warpkit.warpkit_cpp") if spec is not None and spec.origin: binaries.append((spec.origin, "warpkit")) -# Diagnostic prints so any future bundle-without-the-.so failure is debuggable -# from the build log. -print( - f"[hook-warpkit] find_spec origin: {spec.origin if spec else None}", file=sys.stderr -) -print( - f"[hook-warpkit] get_package_paths: {get_package_paths('warpkit')}", file=sys.stderr -) -print( - f"[hook-warpkit] collect_dynamic_libs(*.so): " - f"{collect_dynamic_libs('warpkit', search_patterns=['*.so', '*.dylib'])}", - file=sys.stderr, -) -print(f"[hook-warpkit] binaries: {binaries}", file=sys.stderr) - hiddenimports = ["warpkit.warpkit_cpp"] From 8044b6512211490bd130bb7dae9a1578b3aa7f53 Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 11:32:16 -0500 Subject: [PATCH 07/11] :bug: Trust workspace dir for git in manylinux container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setuptools-scm calls git during uv sync to compute the version, but the manylinux container runs as a different UID than the actions runner — git's "dubious ownership" check trips and the build fails. actions/checkout sets safe.directory for the runner user, not for processes inside the container. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 452ba93..551d517 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,6 +86,8 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 + - name: Trust workspace dir for git (container UID mismatch) + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Install uv uses: astral-sh/setup-uv@v8.1.0 - name: Set up Python From 3a1f955ecc3f675fb4e4b24fa69cf4d5afd819fb Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 11:43:53 -0500 Subject: [PATCH 08/11] :bug: Don't strip linux .so files (corrupts NumPy's bundled OpenBLAS) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running `strip --strip-unneeded` on every `.so*` in the bundle breaks ELF segment alignment in NumPy's bundled libscipy_openblas64_*.so — NumPy then fails to import at runtime with `ELF load command address/offset not properly aligned`. Drop the strip step entirely; the size cost (~30–50MB unzipped, less zipped) is acceptable for a release artifact and removes a class of bugs where strip mangles SIMD-aligned scientific libraries. Co-Authored-By: Claude Opus 4.7 (1M context) --- packaging/pyinstaller/build_bundle.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/packaging/pyinstaller/build_bundle.py b/packaging/pyinstaller/build_bundle.py index ad585d6..b7bffb5 100644 --- a/packaging/pyinstaller/build_bundle.py +++ b/packaging/pyinstaller/build_bundle.py @@ -67,18 +67,6 @@ def run_pyinstaller(spec: Path, dist: Path, work: Path) -> None: subprocess.run(cmd, check=True) -def strip_binaries(bundle: Path) -> None: - # COLLECT's strip= can mangle some libs; strip explicitly here on Linux only. - if platform.system().lower() != "linux": - return - strip = shutil.which("strip") - if strip is None: - return - for so in bundle.rglob("*.so*"): - if so.is_file() and not so.is_symlink(): - subprocess.run([strip, "--strip-unneeded", str(so)], check=False) - - def adhoc_sign_macos(bundle: Path) -> None: if platform.system().lower() != "darwin": return @@ -143,7 +131,6 @@ def main() -> int: versioned = dist / f"warpkit-{version}" raw_bundle.rename(versioned) - strip_binaries(versioned) adhoc_sign_macos(versioned) write_readme(versioned, version, target) From 13108b34d74d2a8b408da3c3952a7628d3e14259 Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 11:57:11 -0500 Subject: [PATCH 09/11] :memo: macOS xattr instructions need -r to also strip libraries Gatekeeper flags every .dylib and .so under _internal/, not just the top-level wk-* binaries. Drop the right-click alternative (it would need to be done per-binary AND the libs still wouldn't be cleared) and use `xattr -r -d` on the whole bundle dir. Co-Authored-By: Claude Opus 4.7 (1M context) --- packaging/pyinstaller/bundle_README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/pyinstaller/bundle_README.md b/packaging/pyinstaller/bundle_README.md index 432400b..0be6e8d 100644 --- a/packaging/pyinstaller/bundle_README.md +++ b/packaging/pyinstaller/bundle_README.md @@ -28,14 +28,14 @@ embedded interpreter and dependency tree. ## macOS Gatekeeper Binaries are ad-hoc signed, not Apple-notarized, so on first launch macOS will -refuse to run them. Either: +quarantine them — and not just the top-level `wk-*` binaries: every `.dylib` +and `.so` inside `_internal/` is also flagged. Strip the quarantine attribute +recursively from the whole bundle: ```sh -xattr -d com.apple.quarantine /opt/warpkit-@VERSION@/wk-* +xattr -r -d com.apple.quarantine /opt/warpkit-@VERSION@ ``` -or right-click → Open the first time on each binary. - ## Available CLIs - `wk-medic` — full MEDIC distortion correction pipeline From 9fc8207895effda8cc28669ccfe7833fa545b0f7 Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 11:58:19 -0500 Subject: [PATCH 10/11] :memo: Document standalone binaries + correct wheel matrix in README - Add a Standalone binaries section pointing at the per-arch zips attached to each GitHub release, so PyPI/README readers can find them without Python. - Fix the wheel matrix line: linux aarch64 wheels are also published. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f5601d9..5a45ea4 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,11 @@ The phase-unwrapping core is a self-contained C++17 port of [ROMEO](https://gith pip install warpkit ``` -Pre-built wheels are published for Linux (x86_64) and macOS (universal2). If `pip` falls back to a source build and fails, please open an issue with the output of `pip install warpkit -v`. +Pre-built wheels are published for Linux (x86_64 + aarch64) and macOS (universal2). If `pip` falls back to a source build and fails, please open an issue with the output of `pip install warpkit -v`. + +### Standalone binaries (no Python required) + +Each [GitHub release](https://github.com/vanandrew/warpkit/releases) attaches a zip per arch (`linux-x86_64`, `linux-aarch64`, `macos-arm64`) containing all seven `wk-*` CLIs as standalone binaries — no Python install or system ITK needed. Extract, add to `PATH`, and run. See the bundled `README.md` inside the zip for install/PATH instructions and the macOS Gatekeeper note. ### Docker From 1861663f61a2742ff7898d11077846fbfdf5c9dc Mon Sep 17 00:00:00 2001 From: Andrew Van Date: Sat, 25 Apr 2026 12:02:56 -0500 Subject: [PATCH 11/11] :bug: Address Copilot PR review comments on build_bundle.py - adhoc_sign_macos: raise on codesign failures instead of silently shipping a bundle Gatekeeper would reject. Collect every failure with stderr and surface them all in one RuntimeError. - write_readme: pass encoding="utf-8" explicitly to read_text / write_text so bundle generation is deterministic across platforms with non-UTF-8 default locales. Co-Authored-By: Claude Opus 4.7 (1M context) --- packaging/pyinstaller/build_bundle.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/packaging/pyinstaller/build_bundle.py b/packaging/pyinstaller/build_bundle.py index b7bffb5..0e877a0 100644 --- a/packaging/pyinstaller/build_bundle.py +++ b/packaging/pyinstaller/build_bundle.py @@ -71,24 +71,35 @@ def adhoc_sign_macos(bundle: Path) -> None: if platform.system().lower() != "darwin": return # Ad-hoc sign every Mach-O in the bundle. `codesign -s -` is sufficient to - # let users override Gatekeeper after the first launch (right-click → Open, - # or `xattr -d com.apple.quarantine`). + # let users override Gatekeeper after recursively clearing the + # com.apple.quarantine xattr (see bundle_README.md). targets = [bundle / name for name in SCRIPTS] targets.extend(p for p in bundle.rglob("*.dylib") if p.is_file()) targets.extend(p for p in (bundle / "_internal").rglob("*.so") if p.is_file()) + failures: list[str] = [] for t in targets: - subprocess.run( + result = subprocess.run( ["codesign", "--force", "--sign", "-", "--timestamp=none", str(t)], - check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + failures.append(f"{t} (exit {result.returncode}): {result.stderr.rstrip()}") + if failures: + raise RuntimeError( + "ad-hoc codesign failed for one or more targets:\n " + + "\n ".join(failures) ) def write_readme(bundle: Path, version: str, target: str) -> None: template = Path(__file__).parent / "bundle_README.md" body = ( - template.read_text().replace("@VERSION@", version).replace("@TARGET@", target) + template.read_text(encoding="utf-8") + .replace("@VERSION@", version) + .replace("@TARGET@", target) ) - (bundle / "README.md").write_text(body) + (bundle / "README.md").write_text(body, encoding="utf-8") def make_zip(bundle: Path, out_zip: Path) -> None: