From cbe371f150a04c5c89d7b196b4ef9da8661bdfc8 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Mon, 6 Jul 2026 14:00:29 -0600 Subject: [PATCH 1/3] CI: Create Dockerfiles for running NIXL ontop of ROCm The base Dockerfile creates a minimal install of ROCm, along with a few other pre-requisites before the build image takes over. This may be replaced once a sufficiently new official ROCm docker image is available. Dockerfile.rocm is based on the existing build.sh script used by NIXL for building the NIXL environment. Instead of running in a script, the source dependency build & install steps are extracted into a multi-stage Docker build process to streamline the overall build. A build-rocm.sh is a copy of build.sh that is slightly modified to build NIXL with ROCm support that may be used by existing tools. Signed-off-by: Riley Dixon --- .ci/dockerfiles/Dockerfile.rocm | 611 +++++++++++++++++++++++++++ .ci/dockerfiles/Dockerfile.rocm.base | 73 ++++ .ci/scripts/common.sh | 8 +- .gitlab/build-rocm.sh | 447 ++++++++++++++++++++ 4 files changed, 1138 insertions(+), 1 deletion(-) create mode 100644 .ci/dockerfiles/Dockerfile.rocm create mode 100644 .ci/dockerfiles/Dockerfile.rocm.base create mode 100755 .gitlab/build-rocm.sh diff --git a/.ci/dockerfiles/Dockerfile.rocm b/.ci/dockerfiles/Dockerfile.rocm new file mode 100644 index 0000000000..81bfd95d8d --- /dev/null +++ b/.ci/dockerfiles/Dockerfile.rocm @@ -0,0 +1,611 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# syntax=docker/dockerfile:1 +# +# Parallel multi-stage ROCm build for NIXL. +# +# This is a BuildKit-parallel refactor of the sequential .gitlab/build-rocm.sh flow. +# Where that script builds ~12 source deps sequentially inside one giant RUN, this +# fans each dep out into its own `FROM builder-base AS -build` stage so BuildKit's +# DAG scheduler runs them concurrently, then COPY --from= merges each dep's +# staged /out tree into a final deps+UCX ENVIRONMENT image. +# +# Usage (self-contained, local dev — single build, stages run in parallel): +# DOCKER_BUILDKIT=1 docker build -f .ci/dockerfiles/Dockerfile.rocm \ +# --build-arg ROCM_BASE_IMAGE= -t nixl-rocm . +# +# ROCM_BASE_IMAGE should point towards an image that has ROCm already installed. +# Dockerfile.rocm.base may be used if a base image does not exist. Currently, this +# base image must also have a source-able /etc/rocm-build.env file that contains the +# installed ROCm version and install location. +# +# ============================================================================ +# Dependency version refs (kept identical to build-rocm.sh; see mutability notes) +# ---------------------------------------------------------------------------- +# Pinned to IMMUTABLE tags (safe to cache; only audit when deliberately bumping): +# abseil lts_2025_08_14, gRPC v1.73.0, AWS 1.11.760, Mooncake v0.3.10.post1, +# Azure azure-storage-blobs_12.15.0, libfabric v1.21.0 +# MUTABLE ref (WILL serve stale layer cache after upstream moves — audit before trust): +# UCX v1.21.x (moving branch), Node.js `nvm install --lts` (floating) +# BARE HEAD, no ref at all — fill in the *_REF args below to make these cacheable: +# etcd-cpp-apiv3, gusli, gtest-parallel +# ============================================================================ + +# ---- Global build args (must be re-declared inside each stage that uses them) ---- +ARG ROCM_BASE_IMAGE + +# Per-dep image indirection. Default = the in-graph stage name (self-contained mode). +# Override to a published registry tag (e.g. reg/nixl-rocm-dep-grpc:v1.73.0) to consume +# a prebuilt dep image instead of rebuilding it (prebuilt / multi-version CI mode). +ARG BUILDER_BASE_IMAGE=builder-base +ARG ABSEIL_IMAGE=abseil-build +ARG GRPC_IMAGE=grpc-build +ARG ETCD_IMAGE=etcd-build +ARG LIBFABRIC_IMAGE=libfabric-build +ARG AWS_IMAGE=aws-build +ARG AZURE_IMAGE=azure-build +ARG MOONCAKE_IMAGE=mooncake-build +ARG GUSLI_IMAGE=gusli-build +ARG GTEST_IMAGE=gtest-build +ARG UCX_IMAGE=ucx-build +ARG TOOLCHAIN_IMAGE=toolchain-build + +# ============================================================================ +# builder-base: Ubuntu+ROCm + the apt/pip toolchain every dep C++ build needs. +# This is the shared parent of every *-build stage. Mirrors the top of the +# PRE_INSTALLED_ENV block in build-rocm.sh, plus Mooncake's runtime apt packages. +# ============================================================================ +FROM ${ROCM_BASE_IMAGE} AS builder-base + +ARG NIXL_INSTALL_DIR=/opt/nixl +ENV INSTALL_DIR=${NIXL_INSTALL_DIR} + +# ---- Point every package manager at the system CA store ---- +# Many tools ship their own trust bundle and ignore the system store, which breaks TLS +# behind a proxy that needs a corporate root cert (expected to be in the system store). +# Inherited by every *-build stage and `final`. +ENV SYSTEM_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +ENV PIP_CERT=${SYSTEM_CA_BUNDLE} \ + REQUESTS_CA_BUNDLE=${SYSTEM_CA_BUNDLE} \ + SSL_CERT_FILE=${SYSTEM_CA_BUNDLE} \ + CURL_CA_BUNDLE=${SYSTEM_CA_BUNDLE} \ + GIT_SSL_CAINFO=${SYSTEM_CA_BUNDLE} \ + NODE_EXTRA_CA_CERTS=${SYSTEM_CA_BUNDLE} \ + CARGO_HTTP_CAINFO=${SYSTEM_CA_BUNDLE} + +# Put the dep install prefixes on the loader search path, the same way build-rocm.sh +# does. Needed so an INSTALLED tool linked against a dep .so resolves at build time — +# e.g. etcd's build invokes /opt/nixl/bin/grpc_cpp_plugin, which needs +# libgrpc_plugin_support.so from /opt/nixl/lib. Every stage inherits this ENV. +ENV LD_LIBRARY_PATH="${NIXL_INSTALL_DIR}/lib:${NIXL_INSTALL_DIR}/lib64:/usr/local/lib:/usr/local/lib64" + +# Some ROCm base images ship broken cmake configs for these. +RUN rm -rf /usr/lib/cmake/grpc /usr/lib/cmake/protobuf + +# Core build toolchain (build-rocm.sh apt list) + Mooncake dependencies.sh +# SYSTEM_PACKAGES (Mooncake runtime deps: libunwind, libyaml-cpp, libhiredis, +# libjemalloc, libmsgpack, libzstd, libasio, libxxhash, unzip). These live here so +# both the mooncake-build stage AND the final image share them (build==runtime +# parity for transfer_engine.so's DT_NEEDED links). +# NOTE: bumping MOONCAKE_VERSION may change this apt set -> treat as invalidating +# builder-base. +RUN apt-get -qq update && \ + apt-get -qq install -y \ + automake \ + autotools-dev \ + build-essential \ + clang \ + cmake \ + curl \ + doxygen \ + etcd-server \ + flex \ + git \ + hwloc \ + iproute2 \ + libaio-dev \ + libasio-dev \ + libboost-all-dev \ + libcpprest-dev \ + libcurl4-openssl-dev \ + libelf-dev \ + libgflags-dev \ + libgmock-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libhiredis-dev \ + libhwloc-dev \ + libibmad-dev \ + libiberty-dev \ + libjemalloc-dev \ + libjsoncpp-dev \ + libmsgpack-dev \ + libnuma-dev \ + libpci-dev \ + libprotobuf-dev \ + libpython3-dev \ + libssl-dev \ + libtool \ + libunwind-dev \ + liburing-dev \ + libxml2-dev \ + libxxhash-dev \ + libyaml-cpp-dev \ + libz-dev \ + libzstd-dev \ + meson \ + net-tools \ + ninja-build \ + numactl \ + parallel \ + patchelf \ + pciutils \ + pkg-config \ + protobuf-compiler-grpc \ + pybind11-dev \ + python3-dev \ + python3-pip \ + unzip \ + uuid-dev \ + wget \ + zlib1g-dev && \ + apt-mark hold liburing2 liburing-dev && \ + rm -rf /var/lib/apt/lists/* + +# Ubuntu 22.04: upgrade pip for --break-system-packages support. From build.sh +RUN if grep -q "Ubuntu 22.04" /etc/os-release 2>/dev/null; then \ + pip3 install --upgrade pip; \ + fi + +# Python build/runtime deps (build-rocm.sh pip list). +RUN pip3 --no-cache-dir install --break-system-packages \ + auditwheel \ + click \ + filelock \ + fsspec \ + jinja2 \ + MarkupSafe \ + meson \ + meson-python \ + mpmath \ + nanobind \ + networkx \ + numpy \ + patchelf \ + pybind11 \ + pytest \ + pytest-timeout \ + sympy \ + tabulate \ + tomlkit \ + typing-extensions \ + zmq + +# torch from the ROCm package repo (matches build-rocm.sh:165-174). +# Currently targeting ROCm pre-releases newer than ROCm 7.2. +RUN . /etc/rocm-build.env && \ + ROCM_TORCH_VERSION="2.11.0" && \ + ROCM_TORCH_INDEX_URL="${ROCM_WHL_INDEX_URL:-https://rocm.prereleases.amd.com/whl-multi-arch/}" && \ + ROCM_TORCH_SUFFIX="${ROCM_TORCH_SUFFIX:-.0rc0}" && \ + pip3 --no-cache-dir install --break-system-packages \ + --cert /etc/ssl/certs/ca-certificates.crt \ + --index-url "${ROCM_TORCH_INDEX_URL}" \ + "torch==${ROCM_TORCH_VERSION}+rocm${ROCM_MAJOR}.${ROCM_MINOR}${ROCM_TORCH_SUFFIX}" + +# RDMA packages (ROCm has no DOCA; install the default verbs/rdma stack). +RUN apt-get -qq update && \ + apt-get -qq -y install --reinstall \ + libibverbs-dev \ + rdma-core \ + ibverbs-utils \ + libibumad-dev \ + libnuma-dev \ + librdmacm-dev \ + ibverbs-providers && \ + rm -rf /var/lib/apt/lists/* + +# ============================================================================ +# abseil-build (head of the abseil->grpc->etcd critical path) +# Staged install: DESTDIR=/out -> /out/opt/nixl/{lib,include,lib/cmake/absl} +# DEPS_SANITIZE (e.g. "address") builds the ABI-shared Abseil instrumented so ASan +# NIXL builds match (see build-rocm.sh:24-46). +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS abseil-build +SHELL ["/bin/bash", "-c"] +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG DEPS_SANITIZE="" +ARG ABSL_TAG=lts_2025_08_14 +RUN set -eux; \ + NPROC=$(nproc); \ + SAN_ARGS=(); \ + if [ -n "$DEPS_SANITIZE" ]; then \ + cxxflags="-fsanitize=${DEPS_SANITIZE}"; \ + case ",${DEPS_SANITIZE}," in \ + *,undefined,*) cxxflags="${cxxflags} -fno-sanitize=null,nonnull-attribute,returns-nonnull-attribute" ;; \ + esac; \ + SAN_ARGS=("-DCMAKE_C_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_CXX_FLAGS=${cxxflags}" "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}"); \ + fi; \ + cd /tmp; \ + git clone --depth 1 --branch "${ABSL_TAG}" https://github.com/abseil/abseil-cpp.git; \ + cd abseil-cpp; mkdir -p build; cd build; \ + cmake .. \ + "${SAN_ARGS[@]}" \ + -DCMAKE_INSTALL_PREFIX="${NIXL_INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +# ---- Alias stages: FROM supports var expansion, COPY --from does not. These let a +# COPY --from= use the static alias while the alias resolves ${*_IMAGE} to +# either the in-graph *-build stage (default) or a prebuilt registry tag. ---- +FROM ${ABSEIL_IMAGE} AS abseil-src + +# ============================================================================ +# grpc-build (longest single compile, on the critical path) +# Needs Abseil's cmake config at configure time -> COPY abseil's /out to the real +# prefix first, then point -Dabsl_DIR at it. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS grpc-build +SHELL ["/bin/bash", "-c"] +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG DEPS_SANITIZE="" +ARG GRPC_TAG=v1.73.0 +# Clone before the COPY so it runs concurrently with abseil's compile. +# gRPC's submodules make this clone particularly slow, so overlapping it helps. +RUN set -eux; \ + cd /tmp; \ + git clone --recurse-submodules -b "${GRPC_TAG}" --depth 1 --shallow-submodules https://github.com/grpc/grpc +COPY --from=abseil-src /out/ / +RUN set -eux; \ + NPROC=$(nproc); \ + SAN_ARGS=(); \ + if [ -n "$DEPS_SANITIZE" ]; then \ + cxxflags="-fsanitize=${DEPS_SANITIZE}"; \ + case ",${DEPS_SANITIZE}," in \ + *,undefined,*) cxxflags="${cxxflags} -fno-sanitize=null,nonnull-attribute,returns-nonnull-attribute" ;; \ + esac; \ + SAN_ARGS=("-DCMAKE_C_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_CXX_FLAGS=${cxxflags}" "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}"); \ + fi; \ + cd /tmp/grpc; mkdir -p cmake/build; cd cmake/build; \ + cmake ../.. \ + "${SAN_ARGS[@]}" \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${NIXL_INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="${NIXL_INSTALL_DIR}" \ + -Dabsl_DIR="${NIXL_INSTALL_DIR}/lib/cmake/absl" \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=module \ + -DgRPC_ZLIB_PROVIDER=package; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +FROM ${GRPC_IMAGE} AS grpc-src + +# ============================================================================ +# etcd-build (small codebase; tail of the critical path) +# Needs both abseil + grpc present at configure time (find_package(gRPC)). +# BARE HEAD clone today -> set ETCD_REF to pin+cache. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS etcd-build +SHELL ["/bin/bash", "-c"] +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG DEPS_SANITIZE="" +ARG ETCD_REF="" +COPY --from=abseil-src /out/ / +COPY --from=grpc-src /out/ / +RUN set -eux; \ + NPROC=$(nproc); \ + SAN_ARGS=(); \ + if [ -n "$DEPS_SANITIZE" ]; then \ + cxxflags="-fsanitize=${DEPS_SANITIZE}"; \ + case ",${DEPS_SANITIZE}," in \ + *,undefined,*) cxxflags="${cxxflags} -fno-sanitize=null,nonnull-attribute,returns-nonnull-attribute" ;; \ + esac; \ + SAN_ARGS=("-DCMAKE_C_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_CXX_FLAGS=${cxxflags}" "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}"); \ + fi; \ + cd /tmp; \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git; \ + cd etcd-cpp-apiv3; \ + if [ -n "$ETCD_REF" ]; then git fetch --depth 1 origin "$ETCD_REF" && git checkout FETCH_HEAD; fi; \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake; \ + mkdir build; cd build; \ + cmake .. \ + "${SAN_ARGS[@]}" \ + -DBUILD_ETCD_CORE_ONLY=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DETCD_CMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_INSTALL_PREFIX="${NIXL_INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="${NIXL_INSTALL_DIR}"; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +# ============================================================================ +# libfabric-build (independent, parallel with the abseil chain) +# Release tarball, not a clone. Installs to ${NIXL_INSTALL_DIR}. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS libfabric-build +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG LIBFABRIC_VERSION=v1.21.0 +RUN set -eux; \ + NPROC=$(nproc); \ + cd /tmp; \ + ver="${LIBFABRIC_VERSION#v}"; \ + wget --tries=3 --waitretry=5 -O "libfabric-${ver}.tar.bz2" "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${ver}.tar.bz2"; \ + tar xjf "libfabric-${ver}.tar.bz2"; \ + cd libfabric-*; \ + ./autogen.sh; \ + ./configure \ + --prefix="${NIXL_INSTALL_DIR}" \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +# ============================================================================ +# aws-build (s3 + s3-crt pulls the whole CRT stack) +# Installs to /usr/local -> /out/usr/local. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS aws-build +ARG AWS_SDK_TAG=1.11.760 +RUN set -eux; \ + NPROC=$(nproc); \ + cd /tmp; \ + git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch "${AWS_SDK_TAG}"; \ + mkdir aws_sdk_build; cd aws_sdk_build; \ + cmake ../aws-sdk-cpp/ \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_ONLY="s3;s3-crt" \ + -DENABLE_TESTING=OFF \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_INSTALL_PREFIX=/usr/local; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +# ============================================================================ +# azure-build +# Installs to /usr/local -> /out/usr/local. Builds only the targets NIXL needs. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS azure-build +ARG AZURE_SDK_TAG=azure-storage-blobs_12.15.0 +RUN set -eux; \ + NPROC=$(nproc); \ + cd /tmp; \ + git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch "${AZURE_SDK_TAG}"; \ + cd azure-sdk-for-cpp; mkdir build; cd build; \ + AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DDISABLE_AMQP=ON \ + -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DCMAKE_CXX_STANDARD=20; \ + cmake --build . --parallel "$NPROC" --target azure-storage-blobs azure-identity; \ + cmake --install sdk/core --prefix /out/usr/local; \ + cmake --install sdk/storage/azure-storage-common --prefix /out/usr/local; \ + cmake --install sdk/storage/azure-storage-blobs --prefix /out/usr/local; \ + cmake --install sdk/identity --prefix /out/usr/local + +# ============================================================================ +# mooncake-build +# dependencies.sh does 3 things: (1) apt SYSTEM_PACKAGES — already in builder-base, so +# this re-install is fast/redundant but harmless; the copies that matter for runtime +# live in builder-base (and thus final) for build==runtime parity; (2) yalantinglibs, +# header-only, built here for this compile only; (3) Go — installed by the script but +# unused here (WITH_STORE=OFF) and not propagated to final. +# Installs to /usr/local -> /out/usr/local. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS mooncake-build +ARG MOONCAKE_VERSION=v0.3.10.post1 +RUN set -eux; \ + NPROC=$(nproc); \ + cd /tmp; \ + git clone --depth 1 --branch "${MOONCAKE_VERSION}" https://github.com/kvcache-ai/Mooncake.git; \ + cd Mooncake; \ + sed -i '/liburing-dev/d' dependencies.sh; \ + bash dependencies.sh -y; \ + mkdir build; cd build; \ + cmake .. -DBUILD_SHARED_LIBS=ON -DWITH_STORE=OFF -G Ninja; \ + ninja -j"$NPROC"; \ + DESTDIR=/out ninja install + +# ============================================================================ +# gusli-build (tiny build) +# NOTE: build-rocm.sh runs `make all` which installs NOTHING (then rm -rf's it) — the +# monolithic image ships no gusli libs. We run `make install INSTALL_DIR=/out/usr` +# instead (its Makefile has a proper install target) so future GUSLI-plugin builds work. +# BARE HEAD clone today -> set GUSLI_REF to pin+cache. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS gusli-build +ARG GUSLI_REF="" +RUN set -eux; \ + cd /tmp; \ + git clone --depth 1 https://github.com/nvidia/gusli.git; \ + cd gusli; \ + if [ -n "$GUSLI_REF" ]; then git fetch --depth 1 origin "$GUSLI_REF" && git checkout FETCH_HEAD; fi; \ + make all CXX="g++ -std=c++20" BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0; \ + make install INSTALL_DIR=/out/usr + +# ============================================================================ +# gtest-build (just copies scripts) -> /out/opt/nixl/bin +# BARE HEAD clone today -> set GTEST_PARALLEL_REF to pin+cache. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS gtest-build +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG GTEST_PARALLEL_REF="" +RUN set -eux; \ + cd /tmp; \ + git clone --depth 1 https://github.com/google/gtest-parallel.git; \ + cd gtest-parallel; \ + if [ -n "$GTEST_PARALLEL_REF" ]; then git fetch --depth 1 origin "$GTEST_PARALLEL_REF" && git checkout FETCH_HEAD; fi; \ + mkdir -p "/out${NIXL_INSTALL_DIR}/bin"; \ + cp /tmp/gtest-parallel/* "/out${NIXL_INSTALL_DIR}/bin/" + +# ============================================================================ +# ucx-build (independent, parallel with everything) +# MUTABLE ref v1.21.x -> layer cache may go stale; audit on refresh. +# Installs to ${UCX_INSTALL_DIR} (defaults to ${NIXL_INSTALL_DIR}) -> /out/opt/nixl. +# ROCm path: no CUDA/gdrcopy/UCCL. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS ucx-build +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG UCX_VERSION=v1.21.x +RUN . /etc/rocm-build.env; \ + set -eux; \ + NPROC=$(nproc); \ + UCX_INSTALL_DIR="${NIXL_INSTALL_DIR}"; \ + cd /tmp; \ + git clone --depth 1 --branch "${UCX_VERSION}" https://github.com/openucx/ucx.git ucx; \ + cd ucx; \ + ./autogen.sh; \ + ./contrib/configure-release-mt \ + --prefix="${UCX_INSTALL_DIR}" \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-rocm="${ROCM_INSTALL_PATH}" \ + --with-verbs \ + --with-dm \ + --without-gdrcopy; \ + make -j"$NPROC"; \ + make -j install-strip DESTDIR=/out + +# ============================================================================ +# toolchain-build (I/O-bound: installers to $HOME; run as root, copy tree in final) +# Rust 1.86.0, uv, nvm+node(--lts)+azurite. Installs under /root; final stage +# relocates into the svc-nixl home. Node.js --lts is a FLOATING ref. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS toolchain-build +SHELL ["/bin/bash", "-c"] +# Force HOME so rustup/uv/nvm land in a known tree the final stage can COPY from +# (BuildKit RUN may leave HOME unset -> installers would scatter to /). +ENV HOME=/root +ARG ARCH=x86_64 +ARG AZURITE_VER=3.35.0 +RUN set -eux; \ + cd /tmp; \ + wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/dist/${ARCH}-unknown-linux-gnu/rustup-init" -O rustup-init; \ + chmod +x rustup-init; \ + ./rustup-init -y --default-toolchain 1.86.0; \ + wget --tries=3 --waitretry=5 "https://astral.sh/uv/install.sh" -O install_uv.sh; \ + chmod +x install_uv.sh; ./install_uv.sh; \ + wget --tries=3 --waitretry=5 "https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh" -O install_nvm.sh; \ + chmod +x install_nvm.sh; ./install_nvm.sh; \ + export NVM_DIR="${HOME}/.nvm"; \ + . "$NVM_DIR/nvm.sh"; \ + nvm install --lts; \ + npm install -g "azurite@${AZURITE_VER}" + +# ---- Alias stages for the final COPY --from (var-expansion workaround, see above). +# abseil-src / grpc-src are already defined earlier and reused here. ---- +FROM ${ETCD_IMAGE} AS etcd-src +FROM ${LIBFABRIC_IMAGE} AS libfabric-src +FROM ${AWS_IMAGE} AS aws-src +FROM ${AZURE_IMAGE} AS azure-src +FROM ${MOONCAKE_IMAGE} AS mooncake-src +FROM ${GUSLI_IMAGE} AS gusli-src +FROM ${GTEST_IMAGE} AS gtest-src +FROM ${UCX_IMAGE} AS ucx-src +FROM ${TOOLCHAIN_IMAGE} AS toolchain-src + +# ============================================================================ +# final: deps + UCX ENVIRONMENT image. NIXL itself is NOT compiled here — this is an +# environment base against which NIXL/nixlbench are built later (build-rocm.sh run with +# PRE_INSTALLED_ENV=1 PRE_INSTALLED_UCX_ENV=1 from a mounted source tree). +# +# Base is `builder-base`, NOT bare rocm.base: build-rocm.sh's env expects the full +# apt/pip/torch toolchain present, so reusing builder-base here (a) keeps build==runtime +# parity, (b) gives Mooncake's runtime apt libs for free (no fragile hand-picked -lib +# package names), and (c) means a later MOONCAKE/AZURE/GUSLI-enabled NIXL build finds +# the -dev headers. +# +# COPY every dep's /out (merge-copy: each /out holds only that dep's files, so +# overlapping /opt/nixl/lib & /usr/local/lib merge, not clobber), relocate the +# toolchain home, set the runtime ENV build-rocm.sh:68-73,396 would export, ldconfig. +# ============================================================================ +FROM builder-base AS final + +ARG _UID=148069 +ARG _GID=30 +ARG _LOGIN=svc-nixl +ARG _GROUP=hardware +ARG _HOME=/home/$_LOGIN +ARG ARCH=x86_64 +ARG NIXL_INSTALL_DIR=/opt/nixl + +# User/group + sudo setup for the non-root service account. +RUN if ! getent group "${_GID}" > /dev/null 2>&1; then \ + groupadd -g "${_GID}" "${_GROUP}"; \ + fi && \ + useradd -l -u "${_UID}" -g "${_GID}" -m -s /bin/bash "${_LOGIN}" && \ + mkdir -p "${_HOME}" && \ + chown -R "${_UID}":"${_GID}" "${_HOME}" && \ + mkdir -p /etc/sudoers.d && \ + echo "${_LOGIN} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/${_LOGIN} && \ + chmod 440 /etc/sudoers.d/${_LOGIN} && \ + chown root:root /etc/sudoers.d/${_LOGIN} + +RUN mkdir -p ${NIXL_INSTALL_DIR} && chown -R "${_UID}":"${_GID}" ${NIXL_INSTALL_DIR} + +# ---- merge-copy each dep's staged /out into the real filesystem ---- +COPY --from=abseil-src /out/ / +COPY --from=grpc-src /out/ / +COPY --from=etcd-src /out/ / +COPY --from=libfabric-src /out/ / +COPY --from=aws-src /out/ / +COPY --from=azure-src /out/ / +COPY --from=mooncake-src /out/ / +COPY --from=gusli-src /out/ / +COPY --from=gtest-src /out/ / +COPY --from=ucx-src /out/ / + +# ---- relocate the toolchain home (installed under /root in toolchain-build) ---- +COPY --from=toolchain-src --chown=${_UID}:${_GID} /root/.cargo ${_HOME}/.cargo +COPY --from=toolchain-src --chown=${_UID}:${_GID} /root/.rustup ${_HOME}/.rustup +COPY --from=toolchain-src --chown=${_UID}:${_GID} /root/.local ${_HOME}/.local +COPY --from=toolchain-src --chown=${_UID}:${_GID} /root/.nvm ${_HOME}/.nvm + +RUN chown -R "${_UID}":"${_GID}" ${NIXL_INSTALL_DIR} && ldconfig + +# ---- runtime ENV (build-rocm.sh:68-73 exported these at build time; as an image we +# persist them so a later NIXL build against this base resolves the deps). ---- +ENV INSTALL_DIR=${NIXL_INSTALL_DIR} +ENV LIBFABRIC_INSTALL_DIR=${NIXL_INSTALL_DIR} +ENV UCX_INSTALL_DIR=${NIXL_INSTALL_DIR} +ENV LD_LIBRARY_PATH="${NIXL_INSTALL_DIR}/lib:${NIXL_INSTALL_DIR}/lib/${ARCH}-linux-gnu:${NIXL_INSTALL_DIR}/lib64:/usr/local/lib:/usr/local/lib/${ARCH}-linux-gnu" +ENV CPATH="${NIXL_INSTALL_DIR}/include:/usr/local/include" +ENV PATH="${NIXL_INSTALL_DIR}/bin:${_HOME}/.local/bin:${_HOME}/.cargo/bin:/usr/local/bin:${PATH}" +ENV PKG_CONFIG_PATH="${NIXL_INSTALL_DIR}/lib/pkgconfig:${NIXL_INSTALL_DIR}/lib64/pkgconfig:/usr/local/lib/pkgconfig" +ENV NIXL_PLUGIN_DIR="${NIXL_INSTALL_DIR}/lib/${ARCH}-linux-gnu/plugins" +ENV CMAKE_PREFIX_PATH="${NIXL_INSTALL_DIR}:/usr/local" +ENV UCX_TLS=^cuda_ipc +ENV NVM_DIR="${_HOME}/.nvm" + +USER ${_LOGIN} +ENV HOME=${_HOME} +ENV USER=${_LOGIN} +WORKDIR ${_HOME} + +ENTRYPOINT ["/bin/bash", "-c", "exec \"$@\"", "--"] +CMD ["/bin/bash"] diff --git a/.ci/dockerfiles/Dockerfile.rocm.base b/.ci/dockerfiles/Dockerfile.rocm.base new file mode 100644 index 0000000000..c41dbf827d --- /dev/null +++ b/.ci/dockerfiles/Dockerfile.rocm.base @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ROCm base image: Ubuntu + the AMD ROCm runtime/dev stack, nothing else. +# +# Usage: +# docker build -f .ci/dockerfiles/Dockerfile.rocm.base -t rocm-base-7.14-ubuntu24.04: . +# +# Key build args (defaults target Ubuntu 24.04 / ROCm 7.14.0): +# UBUNTU_CODENAME apt codename of the base image (default: noble) +# BASE_IMAGE OS image to build on (default: ubuntu:noble) +# UBUNTU_MAJOR_VERSION used to pick the ROCm repo (ubuntu2404) (default: 24) +# UBUNTU_MINOR_VERSION used to pick the ROCm repo (ubuntu2404) (default: 04) +# ROCM_VERSION ROCm release to install (default: 7.14.0) +# +# Tag the image to match ROCM_VERSION so Dockerfile.rocm can pin the base it uses. + +ARG UBUNTU_CODENAME=noble +ARG BASE_IMAGE=ubuntu:${UBUNTU_CODENAME} + +FROM ${BASE_IMAGE} +ARG UBUNTU_CODENAME +ARG UBUNTU_MAJOR_VERSION=24 +ARG UBUNTU_MINOR_VERSION=04 +ARG ROCM_VERSION=7.14.0 + +# Minimal prereqs to reach the ROCm prerelease repo over HTTPS and install from it. +RUN apt-get update && \ + apt-get install -y \ + ca-certificates \ + curl \ + gpg \ + python3 \ + python3-pip \ + sudo \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Optionally install a corporate/proxy root CA (for TLS-intercepting proxies) from +# a BuildKit secret, keeping it out of git and the build context. Supply it with: +# --secret id=proxy_ca,src= (omit for public builds; step no-ops). +# Must run after ca-certificates is installed but before any HTTPS fetch. The cert is +# baked into the trust store and inherited by images built FROM this one. +RUN --mount=type=secret,id=proxy_ca,required=false \ + if [ -s /run/secrets/proxy_ca ]; then \ + cp /run/secrets/proxy_ca /usr/local/share/ca-certificates/proxy-root-ca.crt && \ + update-ca-certificates; \ + else \ + echo "No proxy_ca secret provided; assuming public build with default CAs."; \ + fi + +# Install ROCm from the prerelease multi-arch repo (follows the "new" layout: +# install path /opt/rocm/core-., amdrocm-runtime-dev meta-package). +# /etc/rocm-build.env records the resolved layout for build-rocm.sh to read. +RUN ROCM_MAJOR=$(echo "${ROCM_VERSION}" | cut -d. -f1); \ + ROCM_MINOR=$(echo "${ROCM_VERSION}" | cut -d. -f2); \ + ROCM_INSTALL_PATH="/opt/rocm/core-${ROCM_MAJOR}.${ROCM_MINOR}"; \ + printf 'ROCM_MAJOR=%s\nROCM_MINOR=%s\nROCM_BUILD_TYPE=%s\nROCM_INSTALL_PATH=%s\n' \ + "${ROCM_MAJOR}" "${ROCM_MINOR}" "new" "${ROCM_INSTALL_PATH}" \ + > /etc/rocm-build.env +RUN . /etc/rocm-build.env && \ + wget https://rocm.prereleases.amd.com/packages/gpg/rocm.gpg -O - | \ + gpg --dearmor > /etc/apt/keyrings/amdrocm.gpg && \ + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/amdrocm.gpg] https://rocm.prereleases.amd.com/packages-multi-arch/ubuntu${UBUNTU_MAJOR_VERSION}${UBUNTU_MINOR_VERSION} stable main" \ + > /etc/apt/sources.list.d/amdrocm-prerelease.list + +RUN . /etc/rocm-build.env && \ + apt-get update && \ + apt-get install -y \ + amdrocm-runtime-dev${ROCM_MAJOR}.${ROCM_MINOR} && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/.ci/scripts/common.sh b/.ci/scripts/common.sh index 410c4310bc..c211734a35 100755 --- a/.ci/scripts/common.sh +++ b/.ci/scripts/common.sh @@ -69,7 +69,13 @@ max_gtest_port=$((tcp_port_max + gtest_offset)) # Check if a GPU is present if [ -z "${HAS_GPU}" ]; then - nvidia-smi -L | grep -q '^GPU' && HAS_GPU=true || HAS_GPU=false + if command -v nvidia-smi > /dev/null 2>&1; then + nvidia-smi -L | grep -q '^GPU' && HAS_GPU=true || HAS_GPU=false + elif command -v amd-smi > /dev/null 2>&1; then + amd-smi static -a | grep -q '^GPU' && HAS_GPU=true || HAS_GPU=false + else + HAS_GPU=false + fi fi # Ensure CUDA_HOME is set if CUDA is installed (cuda-dl-base images don't set it by default) diff --git a/.gitlab/build-rocm.sh b/.gitlab/build-rocm.sh new file mode 100755 index 0000000000..d4ebfcc9ee --- /dev/null +++ b/.gitlab/build-rocm.sh @@ -0,0 +1,447 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC1091 +. "$(dirname "$0")/../.ci/scripts/common.sh" +. /etc/rocm-build.env + +set -e +set -x +set -o pipefail + +# Parse commandline arguments with first argument being the install directory +# and second argument being the UCX installation directory. +# Source dependency options grouped with dep download/build/install steps. +INSTALL_DIR=$1 +UCX_INSTALL_DIR=$2 +EXTRA_BUILD_ARGS=${3:-""} +NIXL_BUILD_DIR=${NIXL_BUILD_DIR:-nixl_build} +NIXLBENCH_BUILD_DIR=${NIXLBENCH_BUILD_DIR:-nixlbench_build} +TMPDIR=$(mktemp -d) + +# DEPS_SANITIZE, when set (e.g. "address"), builds the C++ dependency stack that +# shares Abseil's ABI with NIXL (abseil, protobuf/gRPC, etcd-cpp) using the +# matching -fsanitize flags. Required for AddressSanitizer: Abseil changes its +# SwissTable layout under ASan, so a prebuilt non-instrumented Abseil would +# mismatch NIXL's instrumented one at runtime (new-delete-type-mismatch during +# gRPC static init). Only ASan changes ABI (UBSan/TSan do not), so callers pass +# DEPS_SANITIZE=address. The array expands to nothing when unset. +DEPS_SANITIZE=${DEPS_SANITIZE:-""} +DEPS_SANITIZE_CMAKE_ARGS=() +if [ -n "$DEPS_SANITIZE" ]; then + _deps_san_cxxflags="-fsanitize=${DEPS_SANITIZE}" + case ",${DEPS_SANITIZE}," in + # Abseil's headers hit a GCC constexpr bug under UBSan's null checks + # (GCC #71962); drop those sub-checks if undefined is requested. + *,undefined,*) _deps_san_cxxflags="${_deps_san_cxxflags} -fno-sanitize=null,nonnull-attribute,returns-nonnull-attribute" ;; + esac + DEPS_SANITIZE_CMAKE_ARGS=( + "-DCMAKE_C_FLAGS=-fsanitize=${DEPS_SANITIZE}" + "-DCMAKE_CXX_FLAGS=${_deps_san_cxxflags}" + "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" + "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" + ) +fi + +if [ -z "$INSTALL_DIR" ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ -z "$UCX_INSTALL_DIR" ]; then + UCX_INSTALL_DIR=$INSTALL_DIR +fi + + +# For running as user - check if running as root, if not set sudo variable +if [ "$(id -u)" -ne 0 ]; then + SUDO=sudo +else + SUDO="" +fi + +ARCH=$(uname -m) +[ "$ARCH" = "arm64" ] && ARCH="aarch64" + +LIBFABRIC_INSTALL_DIR=${LIBFABRIC_INSTALL_DIR:-$INSTALL_DIR} + +export LD_LIBRARY_PATH="${INSTALL_DIR}/lib:${INSTALL_DIR}/lib/$ARCH-linux-gnu:${INSTALL_DIR}/lib64:$LD_LIBRARY_PATH:${LIBFABRIC_INSTALL_DIR}/lib" +export CPATH="${INSTALL_DIR}/include:${LIBFABRIC_INSTALL_DIR}/include:$CPATH" +export PATH="${INSTALL_DIR}/bin:$HOME/.local/bin:/usr/local/bin:$HOME/.cargo/bin:$PATH" +export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig:${INSTALL_DIR}/lib64/pkgconfig:${INSTALL_DIR}:${LIBFABRIC_INSTALL_DIR}/lib/pkgconfig:$PKG_CONFIG_PATH" +export NIXL_PLUGIN_DIR="${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins" +export CMAKE_PREFIX_PATH="${INSTALL_DIR}:${CMAKE_PREFIX_PATH}" + +if [ -n "$PRE_INSTALLED_ENV" ]; then + echo "PRE_INSTALLED_ENV is set, skipping package installation" +else + # Some docker images are with broken installations: + $SUDO rm -rf /usr/lib/cmake/grpc /usr/lib/cmake/protobuf + + $SUDO apt-get -qq update + $SUDO apt-get -qq install -y \ + python3-dev \ + python3-pip \ + curl \ + wget \ + libnuma-dev \ + numactl \ + autotools-dev \ + automake \ + git \ + libtool \ + libz-dev \ + libiberty-dev \ + flex \ + build-essential \ + cmake \ + libgoogle-glog-dev \ + libgtest-dev \ + libgmock-dev \ + libjsoncpp-dev \ + libpython3-dev \ + libboost-all-dev \ + libssl-dev \ + libprotobuf-dev \ + libcpprest-dev \ + libaio-dev \ + libelf-dev \ + libgflags-dev \ + patchelf \ + meson \ + ninja-build \ + parallel \ + pkg-config \ + protobuf-compiler-grpc \ + pybind11-dev \ + etcd-server \ + net-tools \ + iproute2 \ + pciutils \ + libpci-dev \ + uuid-dev \ + libibmad-dev \ + doxygen \ + clang \ + hwloc \ + libhwloc-dev \ + libxml2-dev \ + libcurl4-openssl-dev \ + zlib1g-dev + $SUDO apt-mark hold \ + liburing2 \ + liburing-dev + + # Ubuntu 22.04 specific setup + if grep -q "Ubuntu 22.04" /etc/os-release 2>/dev/null; then + # Upgrade pip for '--break-system-packages' support + $SUDO pip3 install --upgrade pip + fi + + # Install python dependencies and upgrade to latest version + $SUDO pip3 --no-cache-dir install --break-system-packages \ + meson \ + meson-python \ + pybind11 \ + patchelf \ + click \ + tabulate \ + auditwheel \ + tomlkit \ + pytest \ + pytest-timeout \ + zmq \ + mpmath \ + typing-extensions \ + sympy \ + numpy \ + networkx \ + MarkupSafe \ + fsspec \ + filelock \ + jinja2 \ + nanobind + + # Install torch from the ROCm package repository + # (PyTorch has only published torch for ROCm 7.2.2 at this time) + # (Download torch 2.11 built against ROCm prerelease 7.14.0rc0) + ROCM_TORCH_VERSION="2.11.0" + ROCM_TORCH_INDEX_URL=${ROCM_WHL_INDEX_URL:-"https://rocm.prereleases.amd.com/whl-multi-arch/"} + ROCM_TORCH_SUFFIX=${ROCM_TORCH_SUFFIX:-".0rc0"} + $SUDO pip3 --no-cache-dir install --break-system-packages \ + --cert /etc/ssl/certs/ca-certificates.crt \ + --index-url ${ROCM_TORCH_INDEX_URL} \ + torch==${ROCM_TORCH_VERSION}+rocm${ROCM_MAJOR}.${ROCM_MINOR}${ROCM_TORCH_SUFFIX} + + # Skipping DOCA? + # Nvidia installs RDMA packages from DOCA, just install the default + # packages here for now + $SUDO apt-get -qq -y install --reinstall \ + libibverbs-dev \ + rdma-core \ + ibverbs-utils \ + libibumad-dev \ + libnuma-dev \ + librdmacm-dev \ + ibverbs-providers + + # All other source dependencies +# Rust + wget --tries=3 --waitretry=5 https://static.rust-lang.org/rustup/dist/${ARCH}-unknown-linux-gnu/rustup-init -O ${TMPDIR}/rustup-init + chmod +x ${TMPDIR}/rustup-init + ${TMPDIR}/rustup-init -y --default-toolchain 1.86.0 +# Astral + wget --tries=3 --waitretry=5 "https://astral.sh/uv/install.sh" -O ${TMPDIR}/install_uv.sh + chmod +x ${TMPDIR}/install_uv.sh + ${TMPDIR}/install_uv.sh +# Nodejs & Azurite + # Install Node Version Manager then Nodejs to install Azurite + AZURITE_VER="3.35.0" + wget --tries=3 --waitretry=5 "https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh" -O ${TMPDIR}/install_nvm.sh + chmod +x ${TMPDIR}/install_nvm.sh + ${TMPDIR}/install_nvm.sh + export NVM_DIR=${HOME}/.nvm + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + nvm install --lts # install nodejs + npm install -g azurite@${AZURITE_VER} +# Libfabric + LIBFABRIC_VERSION=${LIBFABRIC_VERSION:-v1.21.0} + wget --tries=3 --waitretry=5 -O "${TMPDIR}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" + tar xjf "${TMPDIR}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -C ${TMPDIR} + rm "${TMPDIR}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" + ( \ + cd ${TMPDIR}/libfabric-* && \ + ./autogen.sh && \ + ./configure \ + --prefix="${LIBFABRIC_INSTALL_DIR}" \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa && \ + make -j"$NPROC" && \ + make install && \ + $SUDO ldconfig && \ + cd .. && \ + rm -rf libfabric-* + ) +# Abseil-cpp + ABSL_TAG=${ABSL_TAG:-lts_2025_08_14} + ( \ + cd ${TMPDIR} && \ + git clone --depth 1 --branch "${ABSL_TAG}" https://github.com/abseil/abseil-cpp.git && \ + cd abseil-cpp && \ + mkdir -p build && cd build && \ + cmake .. \ + "${DEPS_SANITIZE_CMAKE_ARGS[@]}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON && \ + make -j"$NPROC" && \ + $SUDO make install && \ + $SUDO ldconfig && \ + cd ${TMPDIR} && \ + rm -rf abseil-cpp \ + ) +# gRPC + # Possible tied to Abseil-cpp version + GRPC_TAG=${GRPC_TAG:-v1.73.0} + ( \ + cd ${TMPDIR} && \ + git clone --recurse-submodules -b "${GRPC_TAG}" --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ + cd grpc && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake ../.. \ + "${DEPS_SANITIZE_CMAKE_ARGS[@]}" \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -Dabsl_DIR="${INSTALL_DIR}/lib/cmake/absl" \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=module \ + -DgRPC_ZLIB_PROVIDER=package && \ + make -j"$NPROC" && \ + $SUDO make install && \ + $SUDO ldconfig && \ + cd ${TMPDIR} && \ + rm -rf grpc \ + ) +# etcd + ( \ + cd ${TMPDIR} && \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ + cd etcd-cpp-apiv3 && \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ + mkdir build && cd build && \ + cmake .. \ + "${DEPS_SANITIZE_CMAKE_ARGS[@]}" \ + -DBUILD_ETCD_CORE_ONLY=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DETCD_CMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" && \ + make -j"$NPROC" && \ + $SUDO make install && \ + $SUDO ldconfig \ + ) +# AWS + ( \ + cd ${TMPDIR} && \ + git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \ + mkdir aws_sdk_build && \ + cd aws_sdk_build && \ + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_CXX_STANDARD=20 -DCMAKE_INSTALL_PREFIX=/usr/local && \ + make -j"$NPROC" && \ + $SUDO make install && \ + cd .. && \ + rm -rf aws_sdk_build aws-sdk-cpp + ) +# gusli + ( \ + cd ${TMPDIR} && \ + git clone --depth 1 https://github.com/nvidia/gusli.git && \ + cd gusli && \ + $SUDO make all CXX="g++ -std=c++20" BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ + $SUDO ldconfig && \ + cd .. && \ + $SUDO rm -rf gusli + ) +# Mookcake + MOONCAKE_VERSION="${MOONCAKE_VERSION:-v0.3.10.post1}" + ( \ + cd ${TMPDIR} && \ + echo "MOONCAKE_VERSION: ${MOONCAKE_VERSION}" && \ + git clone --depth 1 --branch "${MOONCAKE_VERSION}" https://github.com/kvcache-ai/Mooncake.git && \ + cd Mooncake && \ + sed -i '/liburing-dev/d' dependencies.sh && \ + $SUDO bash dependencies.sh -y && \ + mkdir build && cd build && \ + cmake .. -DBUILD_SHARED_LIBS=ON -DWITH_STORE=OFF -G Ninja && \ + ninja -j"$NPROC" && \ + $SUDO ninja install && \ + $SUDO ldconfig && \ + cd .. && \ + rm -rf Mooncake + ) +# gTest + ( \ + cd ${TMPDIR} && + git clone --depth 1 https://github.com/google/gtest-parallel.git && + mkdir -p ${INSTALL_DIR}/bin && + cp ${TMPDIR}/gtest-parallel/* ${INSTALL_DIR}/bin/ + ) +# Azure + ( \ + cd ${TMPDIR} && \ + df -h && \ + curl -sL https://aka.ms/InstallAzureCLIDeb | $SUDO bash && \ + git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ + cd azure-sdk-for-cpp/ && \ + mkdir build && cd build && \ + AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON -DCMAKE_CXX_STANDARD=20 && \ + cmake --build . --parallel "$NPROC" --target azure-storage-blobs azure-identity && \ + $SUDO cmake --install sdk/core && \ + $SUDO cmake --install sdk/storage/azure-storage-common && \ + $SUDO cmake --install sdk/storage/azure-storage-blobs && \ + $SUDO cmake --install sdk/identity + ) +fi # PRE_INSTALLED_ENV + +if [ -n "$PRE_INSTALLED_UCX_ENV" ]; then + echo "PRE_INSTALLED_UCX_ENV is set, skipping UCX compilation" +else + # UCCL is skipped if no Nvidia GPU is present. + # Skipping steps entirely for ROCm. + UCX_VERSION=${UCX_VERSION:-v1.21.x} + git clone --depth 1 --branch "${UCX_VERSION}" https://github.com/openucx/ucx.git ${TMPDIR}/ucx + ( \ + cd ${TMPDIR}/ucx && \ + ./autogen.sh && \ + ./contrib/configure-release-mt \ + --prefix="${UCX_INSTALL_DIR}" \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-verbs \ + --with-dm \ + --without-gdrcopy \ + ${UCX_CUDA_BUILD_ARGS} && \ + make -j"$NPROC" && \ + $SUDO make -j install-strip && \ + $SUDO ldconfig \ + ) +fi # PRE_INSTALLED_UCX_ENV end + +$SUDO rm -rf ${TMPDIR} + +export UCX_TLS=^cuda_ipc + +# Lastly build and install nixl + nixlbench +if [ -n "$PRE_INSTALLED_NIXL_ENV" ]; then + echo "PRE_INSTALLED_NIXL_ENV is set, skipping compilation" +else + if [ "${BUILD_NIXL_EP}" = "true" ]; then + EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS} -Dbuild_nixl_ep=true" + fi + # This image is CUDA-free. Restrict building to plugins that do not require cuFile. + # POSIX alone is enough to produce libnixl and nixlbench. + NIXL_ENABLE_PLUGINS="${NIXL_ENABLE_PLUGINS:-POSIX}" + # shellcheck disable=SC2086 + meson setup \ + ${NIXL_BUILD_DIR} \ + --prefix=${INSTALL_DIR} \ + -Ducx_path=${UCX_INSTALL_DIR} \ + -Dbuild_docs=false \ + -Drust=false \ + ${EXTRA_BUILD_ARGS} \ + -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" \ + --buildtype=debug \ + -Denable_plugins="${NIXL_ENABLE_PLUGINS}" + ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} + ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install + mkdir -p dist && cp ${NIXL_BUILD_DIR}/src/bindings/python/nixl-meta/nixl-*.whl dist/ + + # TODO(kapila): Copy the nixl.pc file to the install directory if needed. + # cp ${BUILD_DIR}/nixl.pc ${INSTALL_DIR}/lib/pkgconfig/nixl.pc + + cd benchmark/nixlbench + meson setup \ + ${NIXLBENCH_BUILD_DIR} \ + -Dnixl_path=${INSTALL_DIR} \ + -Dprefix=${INSTALL_DIR} \ + -Duse_rocm=true \ + -Drocm_path="${ROCM_INSTALL_PATH}" + ninja -j"$NPROC" -C ${NIXLBENCH_BUILD_DIR} + ninja -j"$NPROC" -C ${NIXLBENCH_BUILD_DIR} install +fi # PRE_INSTALLED_NIXL_ENV From 80989c70c76b33dc4f1f7281c265cd0a156b6b23 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 21 Jul 2026 17:03:59 -0600 Subject: [PATCH 2/3] CI: Use the official ROCm 7.14 base image This replaces the custom base image that we have with the official ROCm 7.14 base image for building the ROCm NIXL image. --- .ci/dockerfiles/Dockerfile.rocm | 41 ++++++++-------- .ci/dockerfiles/Dockerfile.rocm.base | 73 ---------------------------- 2 files changed, 20 insertions(+), 94 deletions(-) delete mode 100644 .ci/dockerfiles/Dockerfile.rocm.base diff --git a/.ci/dockerfiles/Dockerfile.rocm b/.ci/dockerfiles/Dockerfile.rocm index 81bfd95d8d..2b21b16c47 100644 --- a/.ci/dockerfiles/Dockerfile.rocm +++ b/.ci/dockerfiles/Dockerfile.rocm @@ -16,10 +16,7 @@ # DOCKER_BUILDKIT=1 docker build -f .ci/dockerfiles/Dockerfile.rocm \ # --build-arg ROCM_BASE_IMAGE= -t nixl-rocm . # -# ROCM_BASE_IMAGE should point towards an image that has ROCm already installed. -# Dockerfile.rocm.base may be used if a base image does not exist. Currently, this -# base image must also have a source-able /etc/rocm-build.env file that contains the -# installed ROCm version and install location. +# Built on top of a public ROCm image (e.g. rocm/dev-ubuntu-24.04:7.14.0-full). # # ============================================================================ # Dependency version refs (kept identical to build-rocm.sh; see mutability notes) @@ -34,7 +31,7 @@ # ============================================================================ # ---- Global build args (must be re-declared inside each stage that uses them) ---- -ARG ROCM_BASE_IMAGE +ARG ROCM_BASE_IMAGE=rocm/dev-ubuntu-24.04:7.14.0-full # Per-dep image indirection. Default = the in-graph stage name (self-contained mode). # Override to a published registry tag (e.g. reg/nixl-rocm-dep-grpc:v1.73.0) to consume @@ -147,6 +144,7 @@ RUN apt-get -qq update && \ pybind11-dev \ python3-dev \ python3-pip \ + sudo \ unzip \ uuid-dev \ wget \ @@ -183,16 +181,19 @@ RUN pip3 --no-cache-dir install --break-system-packages \ typing-extensions \ zmq -# torch from the ROCm package repo (matches build-rocm.sh:165-174). -# Currently targeting ROCm pre-releases newer than ROCm 7.2. -RUN . /etc/rocm-build.env && \ - ROCM_TORCH_VERSION="2.11.0" && \ - ROCM_TORCH_INDEX_URL="${ROCM_WHL_INDEX_URL:-https://rocm.prereleases.amd.com/whl-multi-arch/}" && \ - ROCM_TORCH_SUFFIX="${ROCM_TORCH_SUFFIX:-.0rc0}" && \ +# torch from the public ROCm wheel repo. +# The wheel's local-version tag (+rocmX.Y.Z) is version-embedded, so we derive X.Y.Z from +# the base image's ROCm install (/opt/rocm/core/.info/version, format "X.Y.Z-"). +ARG TORCH_VERSION=2.11.0 +ARG TORCH_INDEX_URL=https://repo.amd.com/rocm/whl-multi-arch/ +RUN set -eux; \ + ver_file=/opt/rocm/core/.info/version; \ + [ -f "$ver_file" ] || { echo "ERROR: $ver_file not found; cannot derive ROCm version for torch" >&2; exit 1; }; \ + rocm_ver="$(cut -d- -f1 "$ver_file")"; \ pip3 --no-cache-dir install --break-system-packages \ --cert /etc/ssl/certs/ca-certificates.crt \ - --index-url "${ROCM_TORCH_INDEX_URL}" \ - "torch==${ROCM_TORCH_VERSION}+rocm${ROCM_MAJOR}.${ROCM_MINOR}${ROCM_TORCH_SUFFIX}" + --index-url "${TORCH_INDEX_URL}" \ + "torch==${TORCH_VERSION}+rocm${rocm_ver}" # RDMA packages (ROCm has no DOCA; install the default verbs/rdma stack). RUN apt-get -qq update && \ @@ -468,8 +469,7 @@ RUN set -eux; \ FROM ${BUILDER_BASE_IMAGE} AS ucx-build ARG NIXL_INSTALL_DIR=/opt/nixl ARG UCX_VERSION=v1.21.x -RUN . /etc/rocm-build.env; \ - set -eux; \ +RUN set -eux; \ NPROC=$(nproc); \ UCX_INSTALL_DIR="${NIXL_INSTALL_DIR}"; \ cd /tmp; \ @@ -484,7 +484,7 @@ RUN . /etc/rocm-build.env; \ --enable-optimizations \ --enable-cma \ --enable-devel-headers \ - --with-rocm="${ROCM_INSTALL_PATH}" \ + --with-rocm="${ROCM_PATH}/core" \ --with-verbs \ --with-dm \ --without-gdrcopy; \ @@ -534,11 +534,10 @@ FROM ${TOOLCHAIN_IMAGE} AS toolchain-src # environment base against which NIXL/nixlbench are built later (build-rocm.sh run with # PRE_INSTALLED_ENV=1 PRE_INSTALLED_UCX_ENV=1 from a mounted source tree). # -# Base is `builder-base`, NOT bare rocm.base: build-rocm.sh's env expects the full -# apt/pip/torch toolchain present, so reusing builder-base here (a) keeps build==runtime -# parity, (b) gives Mooncake's runtime apt libs for free (no fragile hand-picked -lib -# package names), and (c) means a later MOONCAKE/AZURE/GUSLI-enabled NIXL build finds -# the -dev headers. +# Reusing `builder-base` (a) keeps build==runtime parity for build-rocm.sh, which +# expects the full apt/pip/torch toolchain present, (b) gives Mooncake's runtime apt +# libs for free (no fragile hand-picked -lib package names), and (c) means a later +# MOONCAKE/AZURE/GUSLI-enabled NIXL build finds the -dev headers. # # COPY every dep's /out (merge-copy: each /out holds only that dep's files, so # overlapping /opt/nixl/lib & /usr/local/lib merge, not clobber), relocate the diff --git a/.ci/dockerfiles/Dockerfile.rocm.base b/.ci/dockerfiles/Dockerfile.rocm.base deleted file mode 100644 index c41dbf827d..0000000000 --- a/.ci/dockerfiles/Dockerfile.rocm.base +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# ROCm base image: Ubuntu + the AMD ROCm runtime/dev stack, nothing else. -# -# Usage: -# docker build -f .ci/dockerfiles/Dockerfile.rocm.base -t rocm-base-7.14-ubuntu24.04: . -# -# Key build args (defaults target Ubuntu 24.04 / ROCm 7.14.0): -# UBUNTU_CODENAME apt codename of the base image (default: noble) -# BASE_IMAGE OS image to build on (default: ubuntu:noble) -# UBUNTU_MAJOR_VERSION used to pick the ROCm repo (ubuntu2404) (default: 24) -# UBUNTU_MINOR_VERSION used to pick the ROCm repo (ubuntu2404) (default: 04) -# ROCM_VERSION ROCm release to install (default: 7.14.0) -# -# Tag the image to match ROCM_VERSION so Dockerfile.rocm can pin the base it uses. - -ARG UBUNTU_CODENAME=noble -ARG BASE_IMAGE=ubuntu:${UBUNTU_CODENAME} - -FROM ${BASE_IMAGE} -ARG UBUNTU_CODENAME -ARG UBUNTU_MAJOR_VERSION=24 -ARG UBUNTU_MINOR_VERSION=04 -ARG ROCM_VERSION=7.14.0 - -# Minimal prereqs to reach the ROCm prerelease repo over HTTPS and install from it. -RUN apt-get update && \ - apt-get install -y \ - ca-certificates \ - curl \ - gpg \ - python3 \ - python3-pip \ - sudo \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Optionally install a corporate/proxy root CA (for TLS-intercepting proxies) from -# a BuildKit secret, keeping it out of git and the build context. Supply it with: -# --secret id=proxy_ca,src= (omit for public builds; step no-ops). -# Must run after ca-certificates is installed but before any HTTPS fetch. The cert is -# baked into the trust store and inherited by images built FROM this one. -RUN --mount=type=secret,id=proxy_ca,required=false \ - if [ -s /run/secrets/proxy_ca ]; then \ - cp /run/secrets/proxy_ca /usr/local/share/ca-certificates/proxy-root-ca.crt && \ - update-ca-certificates; \ - else \ - echo "No proxy_ca secret provided; assuming public build with default CAs."; \ - fi - -# Install ROCm from the prerelease multi-arch repo (follows the "new" layout: -# install path /opt/rocm/core-., amdrocm-runtime-dev meta-package). -# /etc/rocm-build.env records the resolved layout for build-rocm.sh to read. -RUN ROCM_MAJOR=$(echo "${ROCM_VERSION}" | cut -d. -f1); \ - ROCM_MINOR=$(echo "${ROCM_VERSION}" | cut -d. -f2); \ - ROCM_INSTALL_PATH="/opt/rocm/core-${ROCM_MAJOR}.${ROCM_MINOR}"; \ - printf 'ROCM_MAJOR=%s\nROCM_MINOR=%s\nROCM_BUILD_TYPE=%s\nROCM_INSTALL_PATH=%s\n' \ - "${ROCM_MAJOR}" "${ROCM_MINOR}" "new" "${ROCM_INSTALL_PATH}" \ - > /etc/rocm-build.env -RUN . /etc/rocm-build.env && \ - wget https://rocm.prereleases.amd.com/packages/gpg/rocm.gpg -O - | \ - gpg --dearmor > /etc/apt/keyrings/amdrocm.gpg && \ - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/amdrocm.gpg] https://rocm.prereleases.amd.com/packages-multi-arch/ubuntu${UBUNTU_MAJOR_VERSION}${UBUNTU_MINOR_VERSION} stable main" \ - > /etc/apt/sources.list.d/amdrocm-prerelease.list - -RUN . /etc/rocm-build.env && \ - apt-get update && \ - apt-get install -y \ - amdrocm-runtime-dev${ROCM_MAJOR}.${ROCM_MINOR} && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* From f8665f63025193dd9e479ae6b34cd7ac13e7b6e5 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Mon, 20 Jul 2026 19:53:08 -0600 Subject: [PATCH 3/3] CI: Allow approved external copyright attribution in SPDX check copyright-check.sh previously only accepted NVIDIA CORPORATION & AFFILIATES as a valid SPDX-FileCopyrightText author, blocking external contributors from attributing their own contributions. Introduces an AUTHORS allowlist so additional organizations can be accepted. Each entry must still follow NVIDIA's existing header pattern. Files may carry multiple accepted authors, in which case the year validation uses the most recent year across their lines. --- .github/workflows/copyright-check.sh | 47 +++++++++++++++++++--------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/workflows/copyright-check.sh b/.github/workflows/copyright-check.sh index 67eec295e3..a5ab2da244 100755 --- a/.github/workflows/copyright-check.sh +++ b/.github/workflows/copyright-check.sh @@ -1,10 +1,17 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail failures=() +# Accepted copyright holders. Add new orgs here — each must appear in a header line as: +# SPDX-FileCopyrightText: Copyright (c) YYYY(-YYYY) . All rights reserved. +AUTHORS=( + "NVIDIA CORPORATION & AFFILIATES" + "Advanced Micro Devices, Inc" +) + for f in $(git ls-files); do # Normalize path f=${f#./} @@ -28,8 +35,24 @@ for f in $(git ls-files); do header=$(head -n 20 "$f") - # Match SPDX-FileCopyrightText with NVIDIA and year(s) - if ! echo "$header" | grep -Eq 'SPDX-FileCopyrightText:\s*Copyright \(c\) [0-9]{4}(-[0-9]{4})? NVIDIA CORPORATION & AFFILIATES\. All rights reserved\.'; then + # Match SPDX-FileCopyrightText with year(s) + copyright_lines=$(echo "$header" | grep -E 'SPDX-FileCopyrightText:\s*Copyright \(c\) [0-9]{4}(-[0-9]{4})? .+\. All rights reserved\.' || true) + + # Keep only lines whose author field exactly matches an entry in AUTHORS. + # The author field is the text between the year and ". All rights reserved."; + # exact comparison rejects any extra text before/after an accepted author. + matched_lines="" + while IFS= read -r line; do + line_author=$(echo "$line" | sed -E 's/.*Copyright \(c\) [0-9]{4}(-[0-9]{4})? (.+)\. All rights reserved\..*/\2/') + for author in "${AUTHORS[@]}"; do + if [[ "$line_author" == "$author" ]]; then + matched_lines+="$line"$'\n' + break + fi + done + done <<< "$copyright_lines" + + if [[ -z "$matched_lines" ]]; then failures+=("$f (missing or incorrect copyright line)") continue fi @@ -37,18 +60,14 @@ for f in $(git ls-files); do # Extract last modification year from git last_modified=$(git log -1 --pretty="%cs" -- "$f" | cut -d- -f1) - # Extract copyright years (handles YYYY or YYYY-YYYY) - copyright_years=$(echo "$header" | \ - grep NVIDIA | grep -Eo 'Copyright \(c\) [0-9]{4}(-[0-9]{4})?' | \ - sed -E 's/.* ([0-9]{4})(-[0-9]{4})?/\1\2/' || true) - - if [[ -z "$copyright_years" ]]; then - failures+=("$f (missing copyright)") - continue - fi + # Extract copyright years (handles YYYY or YYYY-YYYY) from every matched line; + # a file may carry more than one accepted author (e.g. a dual-attributed derivative). + copyright_years=$(echo "$matched_lines" | \ + grep -Eo 'Copyright \(c\) [0-9]{4}(-[0-9]{4})?' | \ + sed -E 's/.* ([0-9]{4})(-[0-9]{4})?/\1\2/') - # Get last year (handles range) - end_year=$(echo "$copyright_years" | sed -E 's/.*-//' || true) + # Get the latest end year across all matched authors (handles ranges) + end_year=$(echo "$copyright_years" | sed -E 's/.*-//' | sort -n | tail -1) # Validate date if (( end_year < last_modified )); then