multi_gpu.yml

# Multi-GPU acceptance (ADR-0038)
#
# This workflow is the CI half of Phase 4 item 3 of
# `docs/dev/multi_gpu_plan.md`. It runs the multi-device acceptance set that
# the Phase 2-3 verifications ran by hand. The hosted workflows (`ci.yml`,
# `stable-abi-ci.yml`, `pypi.yml`) build wheels without CUDA devices. The
# separate `single_gpu.yml` workflow owns routine one-or-more-GPU CUDA/OptiX
# execution; this file owns only the two-device replicated-scene acceptance.
#
# EXTERNAL RUNNER REQUIRED. GitHub-hosted runners do not satisfy the labels
# below. A labelled PR, scheduled run, or manual dispatch only queues while no
# matching self-hosted runner is online; a queued job is not evidence.
#
# To activate: register a runner carrying *all* of the labels in `runs-on`
# below (`self-hosted`, `linux`, `x64`, `cuda`, `multi-gpu`) on a host with at
# least two CUDA devices. PRs opt in with `run-multi-gpu-ci`; the weekly run
# detects runner/image/performance drift; manual dispatch remains available.
#
# While the runner is unavailable the same acceptance set is run by hand. The commands below are
# the ones in section 8 of `docs/dev/multi_gpu_operations.md`, which is what
# the Phase 2-3 verifications used on the 2x RTX A6000 host.

name: Multi-GPU Acceptance

on:
  pull_request:
    types: [labeled]
  schedule:
    # Weekly, outside the verification host's normal working hours.
    - cron: "17 9 * * 1"
  workflow_dispatch:
    inputs:
      python:
        description: >-
          Interpreter of the prepared environment on the runner (CUDA Torch
          plus a built in-tree `rayd.torch` extension). An absolute path, e.g.
          /home/ci/miniconda3/envs/maxwell/bin/python.
        required: false
        default: python
      run_benchmark:
        description: >-
          Also run benchmarks/torch/benchmark_multi_device.py and retain
          its schema-versioned JSON artifact. Benchmark execution errors fail
          the job; speedup values remain measurements rather than thresholds.
        required: false
        default: false
        type: boolean

concurrency:
  group: multi-gpu-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  multi-gpu:
    name: Multi-GPU acceptance / 2x CUDA (self-hosted)
    if: >-
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'schedule' ||
      (github.event_name == 'pull_request' &&
       github.event.label.name == 'run-multi-gpu-ci')
    # All five labels are required. A one-GPU runner must not pick this up:
    # every multi-device test skips itself below two devices, so such a job
    # would report green while having verified nothing.
    runs-on: [self-hosted, linux, x64, cuda, multi-gpu]
    timeout-minutes: 120
    env:
      PYTHON: ${{ inputs.python || 'python' }}
      # The extension is built in-tree on this route (see the "Build" step),
      # so the backend package is imported from the working copy.
      PYTHONPATH: ${{ github.workspace }}/python

    steps:
      - uses: actions/checkout@v5

      - name: Report the runner's GPUs
        shell: bash
        run: nvidia-smi

      - name: Require two visible CUDA devices
        shell: bash
        # Fail loudly rather than skip. `unittest.skipUnless(device_count()
        # >= 2, ...)` guards every multi-device module, so on a single-device
        # runner the acceptance set passes vacuously.
        run: |
          "$PYTHON" - <<'PY'
          import torch

          count = torch.cuda.device_count()
          print(f"torch {torch.__version__}, {count} CUDA device(s)")
          for index in range(count):
              print(f"  cuda:{index} {torch.cuda.get_device_name(index)}")
          assert count >= 2, f"this job needs two CUDA devices, found {count}"
          PY

      - name: Build the Torch backend in tree
        shell: bash
        # The runner is expected to carry CUDA, the OptiX headers and a CUDA
        # Torch already; only RayD itself is built here, with the editable dev
        # command `torch/scripts/dev_build_native.ps1` documents. Set
        # RAYD_CI_PREBUILT=1 on a runner that keeps a warm build tree instead
        # of rebuilding from scratch on every dispatch.
        run: |
          if [ "${RAYD_CI_PREBUILT:-0}" = "1" ]; then
            echo "RAYD_CI_PREBUILT=1: reusing the runner's existing build"
          else
            "$PYTHON" -m pip install --no-build-isolation -e torch \
              -Cbuild-dir=build/ci-multi-gpu
          fi
          # Prints the module that will actually be tested, so a stale or
          # shadowed package (PYTHONPATH vs site-packages) is visible in the
          # log rather than silently exercised.
          "$PYTHON" -c "import rayd.torch as rt; print(rt.__file__)"

      - name: Multi-device modules (one fresh process each)
        shell: bash
        # Fresh process per module on purpose: the layer's single-device
        # invariant is partly an import-time property (a default `Scene()`
        # must never import `rayd._impl.multi`), and OptiX pipeline state is
        # per process, so modules must not share one.
        run: |
          status=0
          for module in \
            tests.scene.test_multi_device_smoke \
            tests.scene.test_multi_device_stress \
            tests.scene.test_multi_device_scene \
            tests.scene.test_chunked_executor \
            tests.scene.test_multi_device_policy \
            tests.scene.test_multi_device_resilience \
            tests.diffraction.test_lane_offset \
            tests.test_multi_device_diffraction_paths \
            tests.test_multi_device_coherent_accum \
            tests.test_multi_device_reflection_accum
          do
            echo "::group::$module"
            "$PYTHON" -m unittest "$module" -v || status=1
            echo "::endgroup::"
          done
          exit "$status"

      - name: Distributed recipe (torchrun, 2 ranks)
        shell: bash
        # Launches both examples under `torchrun --nproc_per_node=2`. The test
        # gives each rank its own OPTIX_CACHE_PATH under a temporary root, so
        # nothing has to be set here; it skips itself if `torchrun` is missing.
        run: |
          "$PYTHON" -m unittest \
            tests.scene.test_distributed_recipe -v

      - name: Governance suite
        shell: bash
        # The ADR-0038 guard plus the contract, manifest, committed-PTX and
        # compile-flag records it must stay consistent with. These need no
        # GPU; they run here so a multi-GPU change cannot land with its
        # declarations drifted.
        run: |
          "$PYTHON" -m unittest \
            tests.test_adr0038_multi_device \
            tests.test_multi_device_benchmark_evidence \
            tests.test_shared_operation_contract \
            tests.test_public_api_manifest \
            tests.test_ptx_source_digest \
            tests.test_compile_flag_policy_contract -v

      - name: Multi-device benchmark (schema-versioned measurement)
        if: >-
          github.event_name == 'schedule' ||
          github.event_name == 'pull_request' ||
          inputs.run_benchmark
        shell: bash
        run: |
          mkdir -p artifacts/multi_gpu
          "$PYTHON" -m benchmarks.torch.benchmark_multi_device \
            --json artifacts/multi_gpu/benchmark.json

      - name: Upload multi-device benchmark JSON
        if: >-
          always() &&
          (github.event_name == 'schedule' ||
           github.event_name == 'pull_request' ||
           inputs.run_benchmark)
        uses: actions/upload-artifact@v4
        with:
          name: multi-gpu-benchmark-${{ github.run_id }}-${{ github.run_attempt }}
          path: artifacts/multi_gpu/benchmark.json
          if-no-files-found: error
          retention-days: 30