CI: validate complete RayD distribution build matrix #136
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build And Publish RayD Distributions | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - "codex/**" | |
| pull_request: | |
| release: | |
| types: [published] | |
| workflow_dispatch: | |
| concurrency: | |
| group: distributions-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name != 'release' }} | |
| permissions: | |
| contents: read | |
| env: | |
| CMAKE_BUILD_PARALLEL_LEVEL: "2" | |
| RAYD_CUDA_GENCODE_ARCHES: "70,75,80,86,87,89,90,100,101,120" | |
| RAYD_CUDA_PTX_ARCH: "120" | |
| TORCH_CUDA_ARCH_LIST: "7.0;7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;12.0+PTX" | |
| jobs: | |
| metadata: | |
| name: Packaging metadata / py${{ matrix.python-version }} | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: python -m pip install "tomli>=2; python_version < '3.11'" | |
| - name: Validate package and release metadata | |
| shell: bash | |
| run: | | |
| python -m unittest tests.packaging.test_project_metadata -v | |
| (cd backends/drjit && python -m unittest tests.test_project_metadata -v) | |
| (cd backends/torch && python -m unittest tests.test_project_metadata -v) | |
| build-drjit-linux: | |
| name: rayd-drjit / manylinux_2_28 / py3.10-3.14 | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 240 | |
| needs: metadata | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Free disk space | |
| shell: bash | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android \ | |
| /usr/local/share/boost /opt/hostedtoolcache/CodeQL \ | |
| /usr/local/share/powershell /usr/share/swift || true | |
| sudo docker image prune --all --force || true | |
| - name: Build and repair Dr.Jit wheels | |
| uses: pypa/cibuildwheel@v3.3.0 | |
| env: | |
| CIBW_BUILD: "cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64" | |
| CIBW_SKIP: "*-musllinux_*" | |
| CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 | |
| CIBW_BUILD_VERBOSITY: "1" | |
| CIBW_BEFORE_ALL_LINUX: >- | |
| dnf install -y dnf-plugins-core gcc-toolset-12 && | |
| rpm --import https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/D42D0685.pub && | |
| dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && | |
| dnf clean expire-cache && | |
| dnf install -y cuda-toolkit-12-8 && | |
| ln -sfn /usr/local/cuda-12.8 /usr/local/cuda && | |
| /usr/local/cuda/bin/nvcc --version | |
| CIBW_ENVIRONMENT_LINUX: >- | |
| CUDA_HOME=/usr/local/cuda | |
| CUDA_PATH=/usr/local/cuda | |
| CUDA_ROOT=/usr/local/cuda | |
| CUDA_BIN_PATH=/usr/local/cuda | |
| CUDACXX=/usr/local/cuda/bin/nvcc | |
| RAYD_CUDA_GENCODE_ARCHES=70,75,80,86,87,89,90,100,101,120 | |
| RAYD_CUDA_PTX_ARCH=120 | |
| PIP_CONSTRAINT=/project/.github/constraints/drjit-build.txt | |
| CMAKE_BUILD_PARALLEL_LEVEL=2 | |
| CC=/opt/rh/gcc-toolset-12/root/usr/bin/gcc | |
| CXX=/opt/rh/gcc-toolset-12/root/usr/bin/g++ | |
| CUDAHOSTCXX=/opt/rh/gcc-toolset-12/root/usr/bin/g++ | |
| PATH=/usr/local/cuda/bin:$PATH | |
| LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: >- | |
| auditwheel repair --plat manylinux_2_28_x86_64 | |
| --exclude libcuda.so.1 | |
| --exclude libnvoptix.so.1 | |
| --exclude libdrjit-core.so | |
| --exclude libdrjit-extra.so | |
| --exclude libnanothread.so | |
| -w {dest_dir} {wheel} && | |
| python /project/backends/drjit/scripts/verify_cuda_binary_arches.py | |
| --stem _C {dest_dir}/*.whl | |
| with: | |
| package-dir: backends/drjit | |
| output-dir: dist/drjit | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-rayd-drjit-linux | |
| path: dist/drjit/*.whl | |
| if-no-files-found: error | |
| build-torch-linux: | |
| name: rayd-torch / manylinux_2_28 / py${{ matrix.python-version }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 180 | |
| needs: metadata | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - {python-version: "3.10", cibw-build: "cp310-manylinux_x86_64"} | |
| - {python-version: "3.11", cibw-build: "cp311-manylinux_x86_64"} | |
| - {python-version: "3.12", cibw-build: "cp312-manylinux_x86_64"} | |
| - {python-version: "3.13", cibw-build: "cp313-manylinux_x86_64"} | |
| - {python-version: "3.14", cibw-build: "cp314-manylinux_x86_64"} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Check out OptiX 8.1 headers | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: NVIDIA/optix-dev | |
| ref: v8.1.0 | |
| path: .optix | |
| persist-credentials: false | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Free disk space | |
| shell: bash | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android \ | |
| /usr/local/share/boost /opt/hostedtoolcache/CodeQL \ | |
| /usr/local/share/powershell /usr/share/swift || true | |
| sudo docker image prune --all --force || true | |
| - name: Build and repair Torch wheels | |
| uses: pypa/cibuildwheel@v3.3.0 | |
| env: | |
| CIBW_BUILD: ${{ matrix.cibw-build }} | |
| CIBW_SKIP: "*-musllinux_*" | |
| CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 | |
| CIBW_BUILD_VERBOSITY: "1" | |
| CIBW_BEFORE_ALL_LINUX: >- | |
| dnf install -y dnf-plugins-core gcc-toolset-12 && | |
| rpm --import https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/D42D0685.pub && | |
| dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && | |
| dnf clean expire-cache && | |
| dnf install -y cuda-toolkit-12-8 && | |
| ln -sfn /usr/local/cuda-12.8 /usr/local/cuda && | |
| /usr/local/cuda/bin/nvcc --version | |
| CIBW_ENVIRONMENT_LINUX: >- | |
| CUDA_HOME=/usr/local/cuda | |
| CUDA_PATH=/usr/local/cuda | |
| CUDA_ROOT=/usr/local/cuda | |
| CUDA_BIN_PATH=/usr/local/cuda | |
| CUDACXX=/usr/local/cuda/bin/nvcc | |
| OPTIX_INCLUDE_DIR=/project/.optix/include | |
| CMAKE_CUDA_ARCHITECTURES="70-real;75-real;80-real;86-real;87-real;89-real;90-real;100-real;101-real;120-real;120-virtual" | |
| TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;12.0+PTX" | |
| CMAKE_BUILD_PARALLEL_LEVEL=2 | |
| PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu128 | |
| CC=/opt/rh/gcc-toolset-12/root/usr/bin/gcc | |
| CXX=/opt/rh/gcc-toolset-12/root/usr/bin/g++ | |
| CUDAHOSTCXX=/opt/rh/gcc-toolset-12/root/usr/bin/g++ | |
| PATH=/usr/local/cuda/bin:$PATH | |
| LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: >- | |
| auditwheel repair --plat manylinux_2_28_x86_64 | |
| --exclude libcuda.so.1 | |
| --exclude libnvoptix.so.1 | |
| --exclude libcudart.so.12 | |
| --exclude libc10.so | |
| --exclude libc10_cuda.so | |
| --exclude libtorch.so | |
| --exclude libtorch_cpu.so | |
| --exclude libtorch_cuda.so | |
| --exclude libtorch_python.so | |
| --exclude libtorch_global_deps.so | |
| -w {dest_dir} {wheel} && | |
| python /project/backends/drjit/scripts/verify_cuda_binary_arches.py | |
| --stem _legacy_ops --stem _stable_ops {dest_dir}/*.whl && | |
| python /project/backends/torch/scripts/verify_stable_abi.py | |
| --source-root /project/backends/torch/src/stable {dest_dir}/*.whl | |
| with: | |
| package-dir: backends/torch | |
| output-dir: dist/torch | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-rayd-torch-linux-py${{ matrix.python-version }} | |
| path: dist/torch/*.whl | |
| if-no-files-found: error | |
| build-windows-wheels: | |
| name: rayd-${{ matrix.backend }} / Windows / py${{ matrix.python-version }} | |
| runs-on: windows-2022 | |
| timeout-minutes: 180 | |
| needs: metadata | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| backend: [drjit, torch] | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| env: | |
| CMAKE_CUDA_ARCHITECTURES: "70-real;75-real;80-real;86-real;87-real;89-real;90-real;100-real;101-real;120-real;120-virtual" | |
| OPTIX_INCLUDE_DIR: ${{ github.workspace }}\.optix\include | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Check out OptiX 8.1 headers | |
| if: matrix.backend == 'torch' | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: NVIDIA/optix-dev | |
| ref: v8.1.0 | |
| path: .optix | |
| persist-credentials: false | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install CUDA Toolkit 12.8 | |
| uses: Jimver/cuda-toolkit@v0.2.35 | |
| with: | |
| cuda: "12.8.1" | |
| method: local | |
| log-file-suffix: "${{ matrix.backend }}-windows-py${{ matrix.python-version }}" | |
| - name: Set up MSVC | |
| uses: ilammy/msvc-dev-cmd@v1 | |
| with: | |
| arch: x64 | |
| - name: Export CUDA paths | |
| shell: pwsh | |
| run: | | |
| $cudaRoot = $env:CUDA_PATH | |
| if (-not $cudaRoot) { | |
| $nvcc = (Get-Command nvcc -ErrorAction Stop).Source | |
| $cudaRoot = Split-Path (Split-Path $nvcc -Parent) -Parent | |
| } | |
| "CUDA_HOME=$cudaRoot" >> $env:GITHUB_ENV | |
| "CUDA_PATH=$cudaRoot" >> $env:GITHUB_ENV | |
| - name: Install Dr.Jit build dependencies | |
| if: matrix.backend == 'drjit' | |
| shell: pwsh | |
| run: | | |
| python -m pip install --upgrade pip build twine "scikit-build-core>=0.10" "cmake>=3.22" ninja | |
| python -m pip install --constraint .github/constraints/drjit-build.txt "nanobind==2.9.2" "drjit==1.3.1" | |
| - name: Install Torch build dependencies | |
| if: matrix.backend == 'torch' | |
| shell: pwsh | |
| run: | | |
| python -m pip install --upgrade pip build twine "scikit-build-core>=0.10" "cmake>=3.26" ninja | |
| python -m pip install torch==2.10.0 --index-url https://download.pytorch.org/whl/cu128 | |
| - name: Build Dr.Jit wheel | |
| if: matrix.backend == 'drjit' | |
| working-directory: backends/drjit | |
| run: python -m build --wheel --no-isolation --outdir ../../dist/drjit | |
| - name: Build Torch wheel | |
| if: matrix.backend == 'torch' | |
| working-directory: backends/torch | |
| run: >- | |
| python -m build --wheel --no-isolation --outdir ../../dist/torch | |
| -Cbuild-dir=artifacts/skbuild | |
| - name: Verify Dr.Jit wheel | |
| if: matrix.backend == 'drjit' | |
| shell: pwsh | |
| run: | | |
| $wheel = (Get-ChildItem dist/drjit/*.whl).FullName | |
| python backends/drjit/scripts/verify_cuda_binary_arches.py --stem _C $wheel | |
| python -m twine check $wheel | |
| - name: Verify Torch wheel | |
| if: matrix.backend == 'torch' | |
| shell: pwsh | |
| run: | | |
| $wheel = (Get-ChildItem dist/torch/*.whl).FullName | |
| python backends/drjit/scripts/verify_cuda_binary_arches.py --stem _legacy_ops --stem _stable_ops $wheel | |
| python backends/torch/scripts/verify_stable_abi.py --source-root backends/torch/src/stable $wheel | |
| python -m twine check $wheel | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-rayd-${{ matrix.backend }}-windows-py${{ matrix.python-version }} | |
| path: dist/${{ matrix.backend }}/*.whl | |
| if-no-files-found: error | |
| test-stable-torch-abi: | |
| name: Stable ABI / ${{ matrix.os }} / py${{ matrix.compatibility.python-version }} / torch${{ matrix.compatibility.torch-version }} | |
| needs: [build-torch-linux, build-windows-wheels] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-22.04, windows-2022] | |
| compatibility: | |
| - {python-version: "3.10", torch-version: "2.10.0", cuda-index: "cu128"} | |
| - {python-version: "3.11", torch-version: "2.10.0", cuda-index: "cu128"} | |
| - {python-version: "3.12", torch-version: "2.10.0", cuda-index: "cu128"} | |
| - {python-version: "3.13", torch-version: "2.10.0", cuda-index: "cu128"} | |
| - {python-version: "3.14", torch-version: "2.10.0", cuda-index: "cu128"} | |
| - {python-version: "3.14", torch-version: "2.11.0", cuda-index: "cu128"} | |
| - {python-version: "3.14", torch-version: "2.12.0", cuda-index: "cu126"} | |
| - {python-version: "3.14", torch-version: "2.13.0", cuda-index: "cu126"} | |
| steps: | |
| - name: Set up compatibility Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.compatibility.python-version }} | |
| - name: Download Linux Torch wheels | |
| if: runner.os == 'Linux' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: release-rayd-torch-linux-py3.10 | |
| path: wheel-artifact | |
| - name: Download Windows Torch wheel | |
| if: runner.os == 'Windows' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: release-rayd-torch-windows-py3.10 | |
| path: wheel-artifact | |
| - name: Install compatibility Torch | |
| shell: bash | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install --no-cache-dir "torch==${{ matrix.compatibility.torch-version }}" \ | |
| --index-url "https://download.pytorch.org/whl/${{ matrix.compatibility.cuda-index }}" | |
| - name: Load the same Stable ABI library | |
| shell: bash | |
| env: | |
| EXPECTED_TORCH: ${{ matrix.compatibility.torch-version }} | |
| run: | | |
| python - <<'PY' | |
| import os | |
| import zipfile | |
| from pathlib import Path | |
| import torch | |
| assert torch.__version__.split("+", 1)[0] == os.environ["EXPECTED_TORCH"] | |
| wheels = sorted(Path("wheel-artifact").glob("*.whl")) | |
| assert wheels, "RayD Torch wheel artifact is missing" | |
| with zipfile.ZipFile(wheels[0]) as archive: | |
| members = [ | |
| name | |
| for name in archive.namelist() | |
| if Path(name).name.startswith("_stable_ops") | |
| and Path(name).suffix in {".dll", ".so"} | |
| ] | |
| assert len(members) == 1, members | |
| archive.extract(members[0], "stable-library") | |
| library = Path("stable-library") / members[0] | |
| torch.ops.load_library(str(library.resolve())) | |
| assert torch.ops.rayd_torch_stable.camera_sample_to_world is not None | |
| assert torch.ops.rayd_torch_stable.intersection_valid is not None | |
| print(f"Loaded {library.name} with torch {torch.__version__}") | |
| PY | |
| build-meta: | |
| name: rayd meta wheel and sdist | |
| runs-on: ubuntu-22.04 | |
| needs: metadata | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - run: python -m pip install --upgrade pip build twine "setuptools>=77" | |
| - run: python -m build --wheel --sdist --no-isolation --outdir dist/meta | |
| - run: python -m twine check dist/meta/* | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-rayd-meta | |
| path: dist/meta/* | |
| if-no-files-found: error | |
| validate-wheel-set: | |
| name: Validate complete release artifact set | |
| runs-on: ubuntu-22.04 | |
| needs: [build-drjit-linux, build-torch-linux, build-windows-wheels, test-stable-torch-abi, build-meta] | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| pattern: release-rayd-* | |
| merge-multiple: true | |
| - run: python -m pip install --upgrade pip twine | |
| - name: Validate wheel tags and artifact counts | |
| env: | |
| RAYD_RELEASE_DIST_DIR: ${{ github.workspace }}/dist | |
| run: python -m unittest tests.packaging.test_release_artifact_matrix -v | |
| - name: Validate representative wheel layout | |
| shell: bash | |
| run: | | |
| echo "RAYD_META_WHEEL=$(find dist -maxdepth 1 -name 'rayd-*-none-any.whl' -print -quit)" >> "$GITHUB_ENV" | |
| echo "RAYD_DRJIT_WHEEL=$(find dist -maxdepth 1 -name 'rayd_drjit-*-cp312-cp312-manylinux_2_28_x86_64.whl' -print -quit)" >> "$GITHUB_ENV" | |
| echo "RAYD_TORCH_WHEEL=$(find dist -maxdepth 1 -name 'rayd_torch-*-cp312-cp312-manylinux_2_28_x86_64.whl' -print -quit)" >> "$GITHUB_ENV" | |
| - run: python -m unittest tests.packaging.test_wheel_layout -v | |
| - run: python -m twine check dist/* | |
| publish-drjit: | |
| name: Publish rayd-drjit to PyPI | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| needs: validate-wheel-set | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: pypi-rayd-drjit | |
| url: https://pypi.org/p/rayd-drjit | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| pattern: release-rayd-drjit-* | |
| merge-multiple: true | |
| - uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: dist | |
| verbose: true | |
| publish-torch: | |
| name: Publish rayd-torch to PyPI | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| needs: validate-wheel-set | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: pypi-rayd-torch | |
| url: https://pypi.org/p/rayd-torch | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| pattern: release-rayd-torch-* | |
| merge-multiple: true | |
| - uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: dist | |
| verbose: true | |
| publish-rayd: | |
| name: Publish rayd meta distribution to PyPI | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| needs: [build-meta, publish-drjit, publish-torch] | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/rayd | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: release-rayd-meta | |
| path: dist | |
| - uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| packages-dir: dist | |
| verbose: true |