diff --git a/.ci/dockerfiles/Dockerfile.rocm b/.ci/dockerfiles/Dockerfile.rocm new file mode 100644 index 0000000000..0a03a75081 --- /dev/null +++ b/.ci/dockerfiles/Dockerfile.rocm @@ -0,0 +1,610 @@ +# SPDX-FileCopyrightText: Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# syntax=docker/dockerfile:1 +# +# Parallel multi-stage ROCm build for NIXL. +# +# This is a BuildKit-parallel refactor of the sequential .gitlab/build-rocm.sh flow. +# Where that script builds ~12 source deps sequentially inside one giant RUN, this +# fans each dep out into its own `FROM builder-base AS -build` stage so BuildKit's +# DAG scheduler runs them concurrently, then COPY --from= merges each dep's +# staged /out tree into a final deps+UCX ENVIRONMENT image. +# +# Usage (self-contained, local dev — single build, stages run in parallel): +# DOCKER_BUILDKIT=1 docker build -f .ci/dockerfiles/Dockerfile.rocm \ +# --build-arg ROCM_BASE_IMAGE= -t nixl-rocm . +# +# ROCM_BASE_IMAGE should point towards an image that has ROCm already installed. +# Dockerfile.rocm.base may be used if a base image does not exist. Currently, this +# base image must also have a source-able /etc/rocm-build.env file that contains the +# installed ROCm version and install location. +# +# ============================================================================ +# Dependency version refs (kept identical to build-rocm.sh; see mutability notes) +# ---------------------------------------------------------------------------- +# Pinned to IMMUTABLE tags (safe to cache; only audit when deliberately bumping): +# abseil lts_2025_08_14, gRPC v1.73.0, AWS 1.11.760, Mooncake v0.3.10.post1, +# Azure azure-storage-blobs_12.15.0, libfabric v1.21.0 +# MUTABLE ref (WILL serve stale layer cache after upstream moves — audit before trust): +# UCX v1.21.x (moving branch), Node.js `nvm install --lts` (floating) +# BARE HEAD, no ref at all — fill in the *_REF args below to make these cacheable: +# etcd-cpp-apiv3, gusli, gtest-parallel +# ============================================================================ + +# ---- Global build args (must be re-declared inside each stage that uses them) ---- +ARG ROCM_BASE_IMAGE + +# Per-dep image indirection. Default = the in-graph stage name (self-contained mode). +# Override to a published registry tag (e.g. reg/nixl-rocm-dep-grpc:v1.73.0) to consume +# a prebuilt dep image instead of rebuilding it (prebuilt / multi-version CI mode). +ARG BUILDER_BASE_IMAGE=builder-base +ARG ABSEIL_IMAGE=abseil-build +ARG GRPC_IMAGE=grpc-build +ARG ETCD_IMAGE=etcd-build +ARG LIBFABRIC_IMAGE=libfabric-build +ARG AWS_IMAGE=aws-build +ARG AZURE_IMAGE=azure-build +ARG MOONCAKE_IMAGE=mooncake-build +ARG GUSLI_IMAGE=gusli-build +ARG GTEST_IMAGE=gtest-build +ARG UCX_IMAGE=ucx-build +ARG TOOLCHAIN_IMAGE=toolchain-build + +# ============================================================================ +# builder-base: Ubuntu+ROCm + the apt/pip toolchain every dep C++ build needs. +# This is the shared parent of every *-build stage. Mirrors the top of the +# PRE_INSTALLED_ENV block in build-rocm.sh, plus Mooncake's runtime apt packages. +# ============================================================================ +FROM ${ROCM_BASE_IMAGE} AS builder-base + +ARG NIXL_INSTALL_DIR=/opt/nixl +ENV INSTALL_DIR=${NIXL_INSTALL_DIR} + +# ---- Point every package manager at the system CA store ---- +# Many tools ship their own trust bundle and ignore the system store, which breaks TLS +# behind a proxy that needs a corporate root cert (expected to be in the system store). +# Inherited by every *-build stage and `final`. +ENV SYSTEM_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +ENV PIP_CERT=${SYSTEM_CA_BUNDLE} \ + REQUESTS_CA_BUNDLE=${SYSTEM_CA_BUNDLE} \ + SSL_CERT_FILE=${SYSTEM_CA_BUNDLE} \ + CURL_CA_BUNDLE=${SYSTEM_CA_BUNDLE} \ + GIT_SSL_CAINFO=${SYSTEM_CA_BUNDLE} \ + NODE_EXTRA_CA_CERTS=${SYSTEM_CA_BUNDLE} \ + CARGO_HTTP_CAINFO=${SYSTEM_CA_BUNDLE} + +# Put the dep install prefixes on the loader search path, the same way build-rocm.sh +# does. Needed so an INSTALLED tool linked against a dep .so resolves at build time — +# e.g. etcd's build invokes /opt/nixl/bin/grpc_cpp_plugin, which needs +# libgrpc_plugin_support.so from /opt/nixl/lib. Every stage inherits this ENV. +ENV LD_LIBRARY_PATH="${NIXL_INSTALL_DIR}/lib:${NIXL_INSTALL_DIR}/lib64:/usr/local/lib:/usr/local/lib64" + +# Some ROCm base images ship broken cmake configs for these. +RUN rm -rf /usr/lib/cmake/grpc /usr/lib/cmake/protobuf + +# Core build toolchain (build-rocm.sh apt list) + Mooncake dependencies.sh +# SYSTEM_PACKAGES (Mooncake runtime deps: libunwind, libyaml-cpp, libhiredis, +# libjemalloc, libmsgpack, libzstd, libasio, libxxhash, unzip). These live here so +# both the mooncake-build stage AND the final image share them (build==runtime +# parity for transfer_engine.so's DT_NEEDED links). +# NOTE: bumping MOONCAKE_VERSION may change this apt set -> treat as invalidating +# builder-base. +RUN apt-get -qq update && \ + apt-get -qq install -y \ + automake \ + autotools-dev \ + build-essential \ + clang \ + cmake \ + curl \ + doxygen \ + etcd-server \ + flex \ + git \ + hwloc \ + iproute2 \ + libaio-dev \ + libasio-dev \ + libboost-all-dev \ + libcpprest-dev \ + libcurl4-openssl-dev \ + libelf-dev \ + libgflags-dev \ + libgmock-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libhiredis-dev \ + libhwloc-dev \ + libibmad-dev \ + libiberty-dev \ + libjemalloc-dev \ + libjsoncpp-dev \ + libmsgpack-dev \ + libnuma-dev \ + libpci-dev \ + libprotobuf-dev \ + libpython3-dev \ + libssl-dev \ + libtool \ + libunwind-dev \ + liburing-dev \ + libxml2-dev \ + libxxhash-dev \ + libyaml-cpp-dev \ + libz-dev \ + libzstd-dev \ + meson \ + net-tools \ + ninja-build \ + numactl \ + parallel \ + patchelf \ + pciutils \ + pkg-config \ + protobuf-compiler-grpc \ + pybind11-dev \ + python3-dev \ + python3-pip \ + unzip \ + uuid-dev \ + wget \ + zlib1g-dev && \ + apt-mark hold liburing2 liburing-dev && \ + rm -rf /var/lib/apt/lists/* + +# Ubuntu 22.04: upgrade pip for --break-system-packages support. From build.sh +RUN if grep -q "Ubuntu 22.04" /etc/os-release 2>/dev/null; then \ + pip3 install --upgrade pip; \ + fi + +# Python build/runtime deps (build-rocm.sh pip list). +RUN pip3 --no-cache-dir install --break-system-packages \ + auditwheel \ + click \ + filelock \ + fsspec \ + jinja2 \ + MarkupSafe \ + meson \ + meson-python \ + mpmath \ + nanobind \ + networkx \ + numpy \ + patchelf \ + pybind11 \ + pytest \ + pytest-timeout \ + sympy \ + tabulate \ + tomlkit \ + typing-extensions \ + zmq + +# torch from the ROCm package repo (matches build-rocm.sh:165-174). +# Currently targeting ROCm pre-releases newer than ROCm 7.2. +RUN . /etc/rocm-build.env && \ + ROCM_TORCH_VERSION="2.11.0" && \ + ROCM_TORCH_INDEX_URL="${ROCM_WHL_INDEX_URL:-https://rocm.prereleases.amd.com/whl-multi-arch/}" && \ + ROCM_TORCH_SUFFIX="${ROCM_TORCH_SUFFIX:-.0rc0}" && \ + pip3 --no-cache-dir install --break-system-packages \ + --cert /etc/ssl/certs/ca-certificates.crt \ + --index-url "${ROCM_TORCH_INDEX_URL}" \ + "torch==${ROCM_TORCH_VERSION}+rocm${ROCM_MAJOR}.${ROCM_MINOR}${ROCM_TORCH_SUFFIX}" + +# RDMA packages (ROCm has no DOCA; install the default verbs/rdma stack). +RUN apt-get -qq update && \ + apt-get -qq -y install --reinstall \ + libibverbs-dev \ + rdma-core \ + ibverbs-utils \ + libibumad-dev \ + libnuma-dev \ + librdmacm-dev \ + ibverbs-providers && \ + rm -rf /var/lib/apt/lists/* + +# ============================================================================ +# abseil-build (head of the abseil->grpc->etcd critical path) +# Staged install: DESTDIR=/out -> /out/opt/nixl/{lib,include,lib/cmake/absl} +# DEPS_SANITIZE (e.g. "address") builds the ABI-shared Abseil instrumented so ASan +# NIXL builds match (see build-rocm.sh:24-46). +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS abseil-build +SHELL ["/bin/bash", "-c"] +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG DEPS_SANITIZE="" +ARG ABSL_TAG=lts_2025_08_14 +RUN set -eux; \ + NPROC=$(nproc); \ + SAN_ARGS=(); \ + if [ -n "$DEPS_SANITIZE" ]; then \ + cxxflags="-fsanitize=${DEPS_SANITIZE}"; \ + case ",${DEPS_SANITIZE}," in \ + *,undefined,*) cxxflags="${cxxflags} -fno-sanitize=null,nonnull-attribute,returns-nonnull-attribute" ;; \ + esac; \ + SAN_ARGS=("-DCMAKE_C_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_CXX_FLAGS=${cxxflags}" "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}"); \ + fi; \ + cd /tmp; \ + git clone --depth 1 --branch "${ABSL_TAG}" https://github.com/abseil/abseil-cpp.git; \ + cd abseil-cpp; mkdir -p build; cd build; \ + cmake .. \ + "${SAN_ARGS[@]}" \ + -DCMAKE_INSTALL_PREFIX="${NIXL_INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +# ---- Alias stages: FROM supports var expansion, COPY --from does not. These let a +# COPY --from= use the static alias while the alias resolves ${*_IMAGE} to +# either the in-graph *-build stage (default) or a prebuilt registry tag. ---- +FROM ${ABSEIL_IMAGE} AS abseil-src + +# ============================================================================ +# grpc-build (longest single compile, on the critical path) +# Needs Abseil's cmake config at configure time -> COPY abseil's /out to the real +# prefix first, then point -Dabsl_DIR at it. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS grpc-build +SHELL ["/bin/bash", "-c"] +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG DEPS_SANITIZE="" +ARG GRPC_TAG=v1.73.0 +# Clone before the COPY so it runs concurrently with abseil's compile. +# gRPC's submodules make this clone particularly slow, so overlapping it helps. +RUN set -eux; \ + cd /tmp; \ + git clone --recurse-submodules -b "${GRPC_TAG}" --depth 1 --shallow-submodules https://github.com/grpc/grpc +COPY --from=abseil-src /out/ / +RUN set -eux; \ + NPROC=$(nproc); \ + SAN_ARGS=(); \ + if [ -n "$DEPS_SANITIZE" ]; then \ + cxxflags="-fsanitize=${DEPS_SANITIZE}"; \ + case ",${DEPS_SANITIZE}," in \ + *,undefined,*) cxxflags="${cxxflags} -fno-sanitize=null,nonnull-attribute,returns-nonnull-attribute" ;; \ + esac; \ + SAN_ARGS=("-DCMAKE_C_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_CXX_FLAGS=${cxxflags}" "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}"); \ + fi; \ + cd /tmp/grpc; mkdir -p cmake/build; cd cmake/build; \ + cmake ../.. \ + "${SAN_ARGS[@]}" \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${NIXL_INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="${NIXL_INSTALL_DIR}" \ + -Dabsl_DIR="${NIXL_INSTALL_DIR}/lib/cmake/absl" \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=module \ + -DgRPC_ZLIB_PROVIDER=package; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +FROM ${GRPC_IMAGE} AS grpc-src + +# ============================================================================ +# etcd-build (small codebase; tail of the critical path) +# Needs both abseil + grpc present at configure time (find_package(gRPC)). +# BARE HEAD clone today -> set ETCD_REF to pin+cache. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS etcd-build +SHELL ["/bin/bash", "-c"] +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG DEPS_SANITIZE="" +ARG ETCD_REF="" +COPY --from=abseil-src /out/ / +COPY --from=grpc-src /out/ / +RUN set -eux; \ + NPROC=$(nproc); \ + SAN_ARGS=(); \ + if [ -n "$DEPS_SANITIZE" ]; then \ + cxxflags="-fsanitize=${DEPS_SANITIZE}"; \ + case ",${DEPS_SANITIZE}," in \ + *,undefined,*) cxxflags="${cxxflags} -fno-sanitize=null,nonnull-attribute,returns-nonnull-attribute" ;; \ + esac; \ + SAN_ARGS=("-DCMAKE_C_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_CXX_FLAGS=${cxxflags}" "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}"); \ + fi; \ + cd /tmp; \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git; \ + cd etcd-cpp-apiv3; \ + if [ -n "$ETCD_REF" ]; then git fetch --depth 1 origin "$ETCD_REF" && git checkout FETCH_HEAD; fi; \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake; \ + mkdir build; cd build; \ + cmake .. \ + "${SAN_ARGS[@]}" \ + -DBUILD_ETCD_CORE_ONLY=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DETCD_CMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_INSTALL_PREFIX="${NIXL_INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="${NIXL_INSTALL_DIR}"; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +# ============================================================================ +# libfabric-build (independent, parallel with the abseil chain) +# Release tarball, not a clone. Installs to ${NIXL_INSTALL_DIR}. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS libfabric-build +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG LIBFABRIC_VERSION=v1.21.0 +RUN set -eux; \ + NPROC=$(nproc); \ + cd /tmp; \ + ver="${LIBFABRIC_VERSION#v}"; \ + wget --tries=3 --waitretry=5 -O "libfabric-${ver}.tar.bz2" "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${ver}.tar.bz2"; \ + tar xjf "libfabric-${ver}.tar.bz2"; \ + cd libfabric-*; \ + ./autogen.sh; \ + ./configure \ + --prefix="${NIXL_INSTALL_DIR}" \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +# ============================================================================ +# aws-build (s3 + s3-crt pulls the whole CRT stack) +# Installs to /usr/local -> /out/usr/local. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS aws-build +ARG AWS_SDK_TAG=1.11.760 +RUN set -eux; \ + NPROC=$(nproc); \ + cd /tmp; \ + git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch "${AWS_SDK_TAG}"; \ + mkdir aws_sdk_build; cd aws_sdk_build; \ + cmake ../aws-sdk-cpp/ \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_ONLY="s3;s3-crt" \ + -DENABLE_TESTING=OFF \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_INSTALL_PREFIX=/usr/local; \ + make -j"$NPROC"; \ + make install DESTDIR=/out + +# ============================================================================ +# azure-build +# Installs to /usr/local -> /out/usr/local. Builds only the targets NIXL needs. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS azure-build +ARG AZURE_SDK_TAG=azure-storage-blobs_12.15.0 +RUN set -eux; \ + NPROC=$(nproc); \ + cd /tmp; \ + git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch "${AZURE_SDK_TAG}"; \ + cd azure-sdk-for-cpp; mkdir build; cd build; \ + AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DDISABLE_AMQP=ON \ + -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DCMAKE_CXX_STANDARD=20; \ + cmake --build . --parallel "$NPROC" --target azure-storage-blobs azure-identity; \ + cmake --install sdk/core --prefix /out/usr/local; \ + cmake --install sdk/storage/azure-storage-common --prefix /out/usr/local; \ + cmake --install sdk/storage/azure-storage-blobs --prefix /out/usr/local; \ + cmake --install sdk/identity --prefix /out/usr/local + +# ============================================================================ +# mooncake-build +# dependencies.sh does 3 things: (1) apt SYSTEM_PACKAGES — already in builder-base, so +# this re-install is fast/redundant but harmless; the copies that matter for runtime +# live in builder-base (and thus final) for build==runtime parity; (2) yalantinglibs, +# header-only, built here for this compile only; (3) Go — installed by the script but +# unused here (WITH_STORE=OFF) and not propagated to final. +# Installs to /usr/local -> /out/usr/local. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS mooncake-build +ARG MOONCAKE_VERSION=v0.3.10.post1 +RUN set -eux; \ + NPROC=$(nproc); \ + cd /tmp; \ + git clone --depth 1 --branch "${MOONCAKE_VERSION}" https://github.com/kvcache-ai/Mooncake.git; \ + cd Mooncake; \ + sed -i '/liburing-dev/d' dependencies.sh; \ + bash dependencies.sh -y; \ + mkdir build; cd build; \ + cmake .. -DBUILD_SHARED_LIBS=ON -DWITH_STORE=OFF -G Ninja; \ + ninja -j"$NPROC"; \ + DESTDIR=/out ninja install + +# ============================================================================ +# gusli-build (tiny build) +# NOTE: build-rocm.sh runs `make all` which installs NOTHING (then rm -rf's it) — the +# monolithic image ships no gusli libs. We run `make install INSTALL_DIR=/out/usr` +# instead (its Makefile has a proper install target) so future GUSLI-plugin builds work. +# BARE HEAD clone today -> set GUSLI_REF to pin+cache. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS gusli-build +ARG GUSLI_REF="" +RUN set -eux; \ + cd /tmp; \ + git clone --depth 1 https://github.com/nvidia/gusli.git; \ + cd gusli; \ + if [ -n "$GUSLI_REF" ]; then git fetch --depth 1 origin "$GUSLI_REF" && git checkout FETCH_HEAD; fi; \ + make all CXX="g++ -std=c++20" BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0; \ + make install INSTALL_DIR=/out/usr + +# ============================================================================ +# gtest-build (just copies scripts) -> /out/opt/nixl/bin +# BARE HEAD clone today -> set GTEST_PARALLEL_REF to pin+cache. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS gtest-build +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG GTEST_PARALLEL_REF="" +RUN set -eux; \ + cd /tmp; \ + git clone --depth 1 https://github.com/google/gtest-parallel.git; \ + cd gtest-parallel; \ + if [ -n "$GTEST_PARALLEL_REF" ]; then git fetch --depth 1 origin "$GTEST_PARALLEL_REF" && git checkout FETCH_HEAD; fi; \ + mkdir -p "/out${NIXL_INSTALL_DIR}/bin"; \ + cp /tmp/gtest-parallel/* "/out${NIXL_INSTALL_DIR}/bin/" + +# ============================================================================ +# ucx-build (independent, parallel with everything) +# MUTABLE ref v1.21.x -> layer cache may go stale; audit on refresh. +# Installs to ${UCX_INSTALL_DIR} (defaults to ${NIXL_INSTALL_DIR}) -> /out/opt/nixl. +# ROCm path: no CUDA/gdrcopy/UCCL. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS ucx-build +ARG NIXL_INSTALL_DIR=/opt/nixl +ARG UCX_VERSION=v1.21.x +RUN . /etc/rocm-build.env; \ + set -eux; \ + NPROC=$(nproc); \ + UCX_INSTALL_DIR="${NIXL_INSTALL_DIR}"; \ + cd /tmp; \ + git clone --depth 1 --branch "${UCX_VERSION}" https://github.com/openucx/ucx.git ucx; \ + cd ucx; \ + ./autogen.sh; \ + ./contrib/configure-release-mt \ + --prefix="${UCX_INSTALL_DIR}" \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-rocm="${ROCM_INSTALL_PATH}" \ + --with-verbs \ + --with-dm \ + --without-gdrcopy; \ + make -j"$NPROC"; \ + make -j install-strip DESTDIR=/out + +# ============================================================================ +# toolchain-build (I/O-bound: installers to $HOME; run as root, copy tree in final) +# Rust 1.86.0, uv, nvm+node(--lts)+azurite. Installs under /root; final stage +# relocates into the svc-nixl home. Node.js --lts is a FLOATING ref. +# ============================================================================ +FROM ${BUILDER_BASE_IMAGE} AS toolchain-build +SHELL ["/bin/bash", "-c"] +# Force HOME so rustup/uv/nvm land in a known tree the final stage can COPY from +# (BuildKit RUN may leave HOME unset -> installers would scatter to /). +ENV HOME=/root +ARG ARCH=x86_64 +ARG AZURITE_VER=3.35.0 +RUN set -eux; \ + cd /tmp; \ + wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/dist/${ARCH}-unknown-linux-gnu/rustup-init" -O rustup-init; \ + chmod +x rustup-init; \ + ./rustup-init -y --default-toolchain 1.86.0; \ + wget --tries=3 --waitretry=5 "https://astral.sh/uv/install.sh" -O install_uv.sh; \ + chmod +x install_uv.sh; ./install_uv.sh; \ + wget --tries=3 --waitretry=5 "https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh" -O install_nvm.sh; \ + chmod +x install_nvm.sh; ./install_nvm.sh; \ + export NVM_DIR="${HOME}/.nvm"; \ + . "$NVM_DIR/nvm.sh"; \ + nvm install --lts; \ + npm install -g "azurite@${AZURITE_VER}" + +# ---- Alias stages for the final COPY --from (var-expansion workaround, see above). +# abseil-src / grpc-src are already defined earlier and reused here. ---- +FROM ${ETCD_IMAGE} AS etcd-src +FROM ${LIBFABRIC_IMAGE} AS libfabric-src +FROM ${AWS_IMAGE} AS aws-src +FROM ${AZURE_IMAGE} AS azure-src +FROM ${MOONCAKE_IMAGE} AS mooncake-src +FROM ${GUSLI_IMAGE} AS gusli-src +FROM ${GTEST_IMAGE} AS gtest-src +FROM ${UCX_IMAGE} AS ucx-src +FROM ${TOOLCHAIN_IMAGE} AS toolchain-src + +# ============================================================================ +# final: deps + UCX ENVIRONMENT image. NIXL itself is NOT compiled here — this is an +# environment base against which NIXL/nixlbench are built later (build-rocm.sh run with +# PRE_INSTALLED_ENV=1 PRE_INSTALLED_UCX_ENV=1 from a mounted source tree). +# +# Base is `builder-base`, NOT bare rocm.base: build-rocm.sh's env expects the full +# apt/pip/torch toolchain present, so reusing builder-base here (a) keeps build==runtime +# parity, (b) gives Mooncake's runtime apt libs for free (no fragile hand-picked -lib +# package names), and (c) means a later MOONCAKE/AZURE/GUSLI-enabled NIXL build finds +# the -dev headers. +# +# COPY every dep's /out (merge-copy: each /out holds only that dep's files, so +# overlapping /opt/nixl/lib & /usr/local/lib merge, not clobber), relocate the +# toolchain home, set the runtime ENV build-rocm.sh:68-73,396 would export, ldconfig. +# ============================================================================ +FROM builder-base AS final + +ARG _UID=148069 +ARG _GID=30 +ARG _LOGIN=svc-nixl +ARG _GROUP=hardware +ARG _HOME=/home/$_LOGIN +ARG ARCH=x86_64 +ARG NIXL_INSTALL_DIR=/opt/nixl + +# User/group + sudo setup for the non-root service account. +RUN if ! getent group "${_GID}" > /dev/null 2>&1; then \ + groupadd -g "${_GID}" "${_GROUP}"; \ + fi && \ + useradd -l -u "${_UID}" -g "${_GID}" -m -s /bin/bash "${_LOGIN}" && \ + mkdir -p "${_HOME}" && \ + chown -R "${_UID}":"${_GID}" "${_HOME}" && \ + mkdir -p /etc/sudoers.d && \ + echo "${_LOGIN} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/${_LOGIN} && \ + chmod 440 /etc/sudoers.d/${_LOGIN} && \ + chown root:root /etc/sudoers.d/${_LOGIN} + +RUN mkdir -p ${NIXL_INSTALL_DIR} && chown -R "${_UID}":"${_GID}" ${NIXL_INSTALL_DIR} + +# ---- merge-copy each dep's staged /out into the real filesystem ---- +COPY --from=abseil-src /out/ / +COPY --from=grpc-src /out/ / +COPY --from=etcd-src /out/ / +COPY --from=libfabric-src /out/ / +COPY --from=aws-src /out/ / +COPY --from=azure-src /out/ / +COPY --from=mooncake-src /out/ / +COPY --from=gusli-src /out/ / +COPY --from=gtest-src /out/ / +COPY --from=ucx-src /out/ / + +# ---- relocate the toolchain home (installed under /root in toolchain-build) ---- +COPY --from=toolchain-src --chown=${_UID}:${_GID} /root/.cargo ${_HOME}/.cargo +COPY --from=toolchain-src --chown=${_UID}:${_GID} /root/.rustup ${_HOME}/.rustup +COPY --from=toolchain-src --chown=${_UID}:${_GID} /root/.local ${_HOME}/.local +COPY --from=toolchain-src --chown=${_UID}:${_GID} /root/.nvm ${_HOME}/.nvm + +RUN chown -R "${_UID}":"${_GID}" ${NIXL_INSTALL_DIR} && ldconfig + +# ---- runtime ENV (build-rocm.sh:68-73 exported these at build time; as an image we +# persist them so a later NIXL build against this base resolves the deps). ---- +ENV INSTALL_DIR=${NIXL_INSTALL_DIR} +ENV LIBFABRIC_INSTALL_DIR=${NIXL_INSTALL_DIR} +ENV UCX_INSTALL_DIR=${NIXL_INSTALL_DIR} +ENV LD_LIBRARY_PATH="${NIXL_INSTALL_DIR}/lib:${NIXL_INSTALL_DIR}/lib/${ARCH}-linux-gnu:${NIXL_INSTALL_DIR}/lib64:/usr/local/lib:/usr/local/lib/${ARCH}-linux-gnu" +ENV CPATH="${NIXL_INSTALL_DIR}/include:/usr/local/include" +ENV PATH="${NIXL_INSTALL_DIR}/bin:${_HOME}/.local/bin:${_HOME}/.cargo/bin:/usr/local/bin:${PATH}" +ENV PKG_CONFIG_PATH="${NIXL_INSTALL_DIR}/lib/pkgconfig:${NIXL_INSTALL_DIR}/lib64/pkgconfig:/usr/local/lib/pkgconfig" +ENV NIXL_PLUGIN_DIR="${NIXL_INSTALL_DIR}/lib/${ARCH}-linux-gnu/plugins" +ENV CMAKE_PREFIX_PATH="${NIXL_INSTALL_DIR}:/usr/local" +ENV UCX_TLS=^cuda_ipc +ENV NVM_DIR="${_HOME}/.nvm" + +USER ${_LOGIN} +ENV HOME=${_HOME} +ENV USER=${_LOGIN} +WORKDIR ${_HOME} + +ENTRYPOINT ["/bin/bash", "-c", "exec \"$@\"", "--"] +CMD ["/bin/bash"] diff --git a/.ci/dockerfiles/Dockerfile.rocm.base b/.ci/dockerfiles/Dockerfile.rocm.base new file mode 100644 index 0000000000..c41dbf827d --- /dev/null +++ b/.ci/dockerfiles/Dockerfile.rocm.base @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ROCm base image: Ubuntu + the AMD ROCm runtime/dev stack, nothing else. +# +# Usage: +# docker build -f .ci/dockerfiles/Dockerfile.rocm.base -t rocm-base-7.14-ubuntu24.04: . +# +# Key build args (defaults target Ubuntu 24.04 / ROCm 7.14.0): +# UBUNTU_CODENAME apt codename of the base image (default: noble) +# BASE_IMAGE OS image to build on (default: ubuntu:noble) +# UBUNTU_MAJOR_VERSION used to pick the ROCm repo (ubuntu2404) (default: 24) +# UBUNTU_MINOR_VERSION used to pick the ROCm repo (ubuntu2404) (default: 04) +# ROCM_VERSION ROCm release to install (default: 7.14.0) +# +# Tag the image to match ROCM_VERSION so Dockerfile.rocm can pin the base it uses. + +ARG UBUNTU_CODENAME=noble +ARG BASE_IMAGE=ubuntu:${UBUNTU_CODENAME} + +FROM ${BASE_IMAGE} +ARG UBUNTU_CODENAME +ARG UBUNTU_MAJOR_VERSION=24 +ARG UBUNTU_MINOR_VERSION=04 +ARG ROCM_VERSION=7.14.0 + +# Minimal prereqs to reach the ROCm prerelease repo over HTTPS and install from it. +RUN apt-get update && \ + apt-get install -y \ + ca-certificates \ + curl \ + gpg \ + python3 \ + python3-pip \ + sudo \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Optionally install a corporate/proxy root CA (for TLS-intercepting proxies) from +# a BuildKit secret, keeping it out of git and the build context. Supply it with: +# --secret id=proxy_ca,src= (omit for public builds; step no-ops). +# Must run after ca-certificates is installed but before any HTTPS fetch. The cert is +# baked into the trust store and inherited by images built FROM this one. +RUN --mount=type=secret,id=proxy_ca,required=false \ + if [ -s /run/secrets/proxy_ca ]; then \ + cp /run/secrets/proxy_ca /usr/local/share/ca-certificates/proxy-root-ca.crt && \ + update-ca-certificates; \ + else \ + echo "No proxy_ca secret provided; assuming public build with default CAs."; \ + fi + +# Install ROCm from the prerelease multi-arch repo (follows the "new" layout: +# install path /opt/rocm/core-., amdrocm-runtime-dev meta-package). +# /etc/rocm-build.env records the resolved layout for build-rocm.sh to read. +RUN ROCM_MAJOR=$(echo "${ROCM_VERSION}" | cut -d. -f1); \ + ROCM_MINOR=$(echo "${ROCM_VERSION}" | cut -d. -f2); \ + ROCM_INSTALL_PATH="/opt/rocm/core-${ROCM_MAJOR}.${ROCM_MINOR}"; \ + printf 'ROCM_MAJOR=%s\nROCM_MINOR=%s\nROCM_BUILD_TYPE=%s\nROCM_INSTALL_PATH=%s\n' \ + "${ROCM_MAJOR}" "${ROCM_MINOR}" "new" "${ROCM_INSTALL_PATH}" \ + > /etc/rocm-build.env +RUN . /etc/rocm-build.env && \ + wget https://rocm.prereleases.amd.com/packages/gpg/rocm.gpg -O - | \ + gpg --dearmor > /etc/apt/keyrings/amdrocm.gpg && \ + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/amdrocm.gpg] https://rocm.prereleases.amd.com/packages-multi-arch/ubuntu${UBUNTU_MAJOR_VERSION}${UBUNTU_MINOR_VERSION} stable main" \ + > /etc/apt/sources.list.d/amdrocm-prerelease.list + +RUN . /etc/rocm-build.env && \ + apt-get update && \ + apt-get install -y \ + amdrocm-runtime-dev${ROCM_MAJOR}.${ROCM_MINOR} && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/.ci/scripts/common.sh b/.ci/scripts/common.sh index 410c4310bc..c211734a35 100755 --- a/.ci/scripts/common.sh +++ b/.ci/scripts/common.sh @@ -69,7 +69,13 @@ max_gtest_port=$((tcp_port_max + gtest_offset)) # Check if a GPU is present if [ -z "${HAS_GPU}" ]; then - nvidia-smi -L | grep -q '^GPU' && HAS_GPU=true || HAS_GPU=false + if command -v nvidia-smi > /dev/null 2>&1; then + nvidia-smi -L | grep -q '^GPU' && HAS_GPU=true || HAS_GPU=false + elif command -v amd-smi > /dev/null 2>&1; then + amd-smi static -a | grep -q '^GPU' && HAS_GPU=true || HAS_GPU=false + else + HAS_GPU=false + fi fi # Ensure CUDA_HOME is set if CUDA is installed (cuda-dl-base images don't set it by default) diff --git a/.gitlab/build-rocm.sh b/.gitlab/build-rocm.sh new file mode 100755 index 0000000000..07f9f4f958 --- /dev/null +++ b/.gitlab/build-rocm.sh @@ -0,0 +1,446 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC1091 +. "$(dirname "$0")/../.ci/scripts/common.sh" +. /etc/rocm-build.env + +set -e +set -x +set -o pipefail + +# Parse commandline arguments with first argument being the install directory +# and second argument being the UCX installation directory. +# Source dependency options grouped with dep download/build/install steps. +INSTALL_DIR=$1 +UCX_INSTALL_DIR=$2 +EXTRA_BUILD_ARGS=${3:-""} +NIXL_BUILD_DIR=${NIXL_BUILD_DIR:-nixl_build} +NIXLBENCH_BUILD_DIR=${NIXLBENCH_BUILD_DIR:-nixlbench_build} +TMPDIR=$(mktemp -d) + +# DEPS_SANITIZE, when set (e.g. "address"), builds the C++ dependency stack that +# shares Abseil's ABI with NIXL (abseil, protobuf/gRPC, etcd-cpp) using the +# matching -fsanitize flags. Required for AddressSanitizer: Abseil changes its +# SwissTable layout under ASan, so a prebuilt non-instrumented Abseil would +# mismatch NIXL's instrumented one at runtime (new-delete-type-mismatch during +# gRPC static init). Only ASan changes ABI (UBSan/TSan do not), so callers pass +# DEPS_SANITIZE=address. The array expands to nothing when unset. +DEPS_SANITIZE=${DEPS_SANITIZE:-""} +DEPS_SANITIZE_CMAKE_ARGS=() +if [ -n "$DEPS_SANITIZE" ]; then + _deps_san_cxxflags="-fsanitize=${DEPS_SANITIZE}" + case ",${DEPS_SANITIZE}," in + # Abseil's headers hit a GCC constexpr bug under UBSan's null checks + # (GCC #71962); drop those sub-checks if undefined is requested. + *,undefined,*) _deps_san_cxxflags="${_deps_san_cxxflags} -fno-sanitize=null,nonnull-attribute,returns-nonnull-attribute" ;; + esac + DEPS_SANITIZE_CMAKE_ARGS=( + "-DCMAKE_C_FLAGS=-fsanitize=${DEPS_SANITIZE}" + "-DCMAKE_CXX_FLAGS=${_deps_san_cxxflags}" + "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" + "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${DEPS_SANITIZE}" + ) +fi + +if [ -z "$INSTALL_DIR" ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ -z "$UCX_INSTALL_DIR" ]; then + UCX_INSTALL_DIR=$INSTALL_DIR +fi + + +# For running as user - check if running as root, if not set sudo variable +if [ "$(id -u)" -ne 0 ]; then + SUDO=sudo +else + SUDO="" +fi + +ARCH=$(uname -m) +[ "$ARCH" = "arm64" ] && ARCH="aarch64" + +LIBFABRIC_INSTALL_DIR=${LIBFABRIC_INSTALL_DIR:-$INSTALL_DIR} + +export LD_LIBRARY_PATH="${INSTALL_DIR}/lib:${INSTALL_DIR}/lib/$ARCH-linux-gnu:${INSTALL_DIR}/lib64:$LD_LIBRARY_PATH:${LIBFABRIC_INSTALL_DIR}/lib" +export CPATH="${INSTALL_DIR}/include:${LIBFABRIC_INSTALL_DIR}/include:$CPATH" +export PATH="${INSTALL_DIR}/bin:$HOME/.local/bin:/usr/local/bin:$HOME/.cargo/bin:$PATH" +export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig:${INSTALL_DIR}/lib64/pkgconfig:${INSTALL_DIR}:${LIBFABRIC_INSTALL_DIR}/lib/pkgconfig:$PKG_CONFIG_PATH" +export NIXL_PLUGIN_DIR="${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins" +export CMAKE_PREFIX_PATH="${INSTALL_DIR}:${CMAKE_PREFIX_PATH}" + +if [ -n "$PRE_INSTALLED_ENV" ]; then + echo "PRE_INSTALLED_ENV is set, skipping package installation" +else + # Some docker images are with broken installations: + $SUDO rm -rf /usr/lib/cmake/grpc /usr/lib/cmake/protobuf + + $SUDO apt-get -qq update + $SUDO apt-get -qq install -y \ + python3-dev \ + python3-pip \ + curl \ + wget \ + libnuma-dev \ + numactl \ + autotools-dev \ + automake \ + git \ + libtool \ + libz-dev \ + libiberty-dev \ + flex \ + build-essential \ + cmake \ + libgoogle-glog-dev \ + libgtest-dev \ + libgmock-dev \ + libjsoncpp-dev \ + libpython3-dev \ + libboost-all-dev \ + libssl-dev \ + libprotobuf-dev \ + libcpprest-dev \ + libaio-dev \ + libelf-dev \ + libgflags-dev \ + patchelf \ + meson \ + ninja-build \ + parallel \ + pkg-config \ + protobuf-compiler-grpc \ + pybind11-dev \ + etcd-server \ + net-tools \ + iproute2 \ + pciutils \ + libpci-dev \ + uuid-dev \ + libibmad-dev \ + doxygen \ + clang \ + hwloc \ + libhwloc-dev \ + libxml2-dev \ + libcurl4-openssl-dev \ + zlib1g-dev + $SUDO apt-mark hold \ + liburing2 \ + liburing-dev + + # Ubuntu 22.04 specific setup + if grep -q "Ubuntu 22.04" /etc/os-release 2>/dev/null; then + # Upgrade pip for '--break-system-packages' support + $SUDO pip3 install --upgrade pip + fi + + # Install python dependencies and upgrade to latest version + $SUDO pip3 --no-cache-dir install --break-system-packages \ + meson \ + meson-python \ + pybind11 \ + patchelf \ + click \ + tabulate \ + auditwheel \ + tomlkit \ + pytest \ + pytest-timeout \ + zmq \ + mpmath \ + typing-extensions \ + sympy \ + numpy \ + networkx \ + MarkupSafe \ + fsspec \ + filelock \ + jinja2 \ + nanobind + + # Install torch from the ROCm package repository + # (PyTorch has only published torch for ROCm 7.2.2 at this time) + # (Download torch 2.11 built against ROCm prerelease 7.14.0rc0) + ROCM_TORCH_VERSION="2.11.0" + ROCM_TORCH_INDEX_URL=${ROCM_WHL_INDEX_URL:-"https://rocm.prereleases.amd.com/whl-multi-arch/"} + ROCM_TORCH_SUFFIX=${ROCM_TORCH_SUFFIX:-".0rc0"} + $SUDO pip3 --no-cache-dir install --break-system-packages \ + --cert /etc/ssl/certs/ca-certificates.crt \ + --index-url ${ROCM_TORCH_INDEX_URL} \ + torch==${ROCM_TORCH_VERSION}+rocm${ROCM_MAJOR}.${ROCM_MINOR}${ROCM_TORCH_SUFFIX} + + # Skipping DOCA? + # Nvidia installs RDMA packages from DOCA, just install the default + # packages here for now + $SUDO apt-get -qq -y install --reinstall \ + libibverbs-dev \ + rdma-core \ + ibverbs-utils \ + libibumad-dev \ + libnuma-dev \ + librdmacm-dev \ + ibverbs-providers + + # All other source dependencies +# Rust + wget --tries=3 --waitretry=5 https://static.rust-lang.org/rustup/dist/${ARCH}-unknown-linux-gnu/rustup-init -O ${TMPDIR}/rustup-init + chmod +x ${TMPDIR}/rustup-init + ${TMPDIR}/rustup-init -y --default-toolchain 1.86.0 +# Astral + wget --tries=3 --waitretry=5 "https://astral.sh/uv/install.sh" -O ${TMPDIR}/install_uv.sh + chmod +x ${TMPDIR}/install_uv.sh + ${TMPDIR}/install_uv.sh +# Nodejs & Azurite + # Install Node Version Manager then Nodejs to install Azurite + AZURITE_VER="3.35.0" + wget --tries=3 --waitretry=5 "https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh" -O ${TMPDIR}/install_nvm.sh + chmod +x ${TMPDIR}/install_nvm.sh + ${TMPDIR}/install_nvm.sh + export NVM_DIR=${HOME}/.nvm + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + nvm install --lts # install nodejs + npm install -g azurite@${AZURITE_VER} +# Libfabric + LIBFABRIC_VERSION=${LIBFABRIC_VERSION:-v1.21.0} + wget --tries=3 --waitretry=5 -O "${TMPDIR}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" + tar xjf "${TMPDIR}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -C ${TMPDIR} + rm "${TMPDIR}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" + ( \ + cd ${TMPDIR}/libfabric-* && \ + ./autogen.sh && \ + ./configure \ + --prefix="${LIBFABRIC_INSTALL_DIR}" \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa && \ + make -j"$NPROC" && \ + make install && \ + $SUDO ldconfig && \ + cd .. && \ + rm -rf libfabric-* + ) +# Abseil-cpp + ABSL_TAG=${ABSL_TAG:-lts_2025_08_14} + ( \ + cd ${TMPDIR} && \ + git clone --depth 1 --branch "${ABSL_TAG}" https://github.com/abseil/abseil-cpp.git && \ + cd abseil-cpp && \ + mkdir -p build && cd build && \ + cmake .. \ + "${DEPS_SANITIZE_CMAKE_ARGS[@]}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON && \ + make -j"$NPROC" && \ + $SUDO make install && \ + $SUDO ldconfig && \ + cd ${TMPDIR} && \ + rm -rf abseil-cpp \ + ) +# gRPC + # Possible tied to Abseil-cpp version + GRPC_TAG=${GRPC_TAG:-v1.73.0} + ( \ + cd ${TMPDIR} && \ + git clone --recurse-submodules -b "${GRPC_TAG}" --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ + cd grpc && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake ../.. \ + "${DEPS_SANITIZE_CMAKE_ARGS[@]}" \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -Dabsl_DIR="${INSTALL_DIR}/lib/cmake/absl" \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=module \ + -DgRPC_ZLIB_PROVIDER=package && \ + make -j"$NPROC" && \ + $SUDO make install && \ + $SUDO ldconfig && \ + cd ${TMPDIR} && \ + rm -rf grpc \ + ) +# etcd + ( \ + cd ${TMPDIR} && \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ + cd etcd-cpp-apiv3 && \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ + mkdir build && cd build && \ + cmake .. \ + "${DEPS_SANITIZE_CMAKE_ARGS[@]}" \ + -DBUILD_ETCD_CORE_ONLY=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DETCD_CMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" && \ + make -j"$NPROC" && \ + $SUDO make install && \ + $SUDO ldconfig \ + ) +# AWS + ( \ + cd ${TMPDIR} && \ + git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 && \ + mkdir aws_sdk_build && \ + cd aws_sdk_build && \ + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_CXX_STANDARD=20 -DCMAKE_INSTALL_PREFIX=/usr/local && \ + make -j"$NPROC" && \ + $SUDO make install && \ + cd .. && \ + rm -rf aws_sdk_build aws-sdk-cpp + ) +# gusli + ( \ + cd ${TMPDIR} && \ + git clone --depth 1 https://github.com/nvidia/gusli.git && \ + cd gusli && \ + $SUDO make all CXX="g++ -std=c++20" BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ + $SUDO ldconfig && \ + cd .. && \ + $SUDO rm -rf gusli + ) +# Mookcake + MOONCAKE_VERSION="${MOONCAKE_VERSION:-v0.3.10.post1}" + ( \ + cd ${TMPDIR} && \ + echo "MOONCAKE_VERSION: ${MOONCAKE_VERSION}" && \ + git clone --depth 1 --branch "${MOONCAKE_VERSION}" https://github.com/kvcache-ai/Mooncake.git && \ + cd Mooncake && \ + sed -i '/liburing-dev/d' dependencies.sh && \ + $SUDO bash dependencies.sh -y && \ + mkdir build && cd build && \ + cmake .. -DBUILD_SHARED_LIBS=ON -DWITH_STORE=OFF -G Ninja && \ + ninja -j"$NPROC" && \ + $SUDO ninja install && \ + $SUDO ldconfig && \ + cd .. && \ + rm -rf Mooncake + ) +# gTest + ( \ + cd ${TMPDIR} && + git clone --depth 1 https://github.com/google/gtest-parallel.git && + mkdir -p ${INSTALL_DIR}/bin && + cp ${TMPDIR}/gtest-parallel/* ${INSTALL_DIR}/bin/ + ) +# Azure + ( \ + cd ${TMPDIR} && \ + df -h && \ + curl -sL https://aka.ms/InstallAzureCLIDeb | $SUDO bash && \ + git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ + cd azure-sdk-for-cpp/ && \ + mkdir build && cd build && \ + AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON -DCMAKE_CXX_STANDARD=20 && \ + cmake --build . --parallel "$NPROC" --target azure-storage-blobs azure-identity && \ + $SUDO cmake --install sdk/core && \ + $SUDO cmake --install sdk/storage/azure-storage-common && \ + $SUDO cmake --install sdk/storage/azure-storage-blobs && \ + $SUDO cmake --install sdk/identity + ) +fi # PRE_INSTALLED_ENV + +if [ -n "$PRE_INSTALLED_UCX_ENV" ]; then + echo "PRE_INSTALLED_UCX_ENV is set, skipping UCX compilation" +else + # UCCL is skipped if no Nvidia GPU is present. + # Skipping steps entirely for ROCm. + UCX_VERSION=${UCX_VERSION:-v1.21.x} + git clone --depth 1 --branch "${UCX_VERSION}" https://github.com/openucx/ucx.git ${TMPDIR}/ucx + ( \ + cd ${TMPDIR}/ucx && \ + ./autogen.sh && \ + ./contrib/configure-release-mt \ + --prefix="${UCX_INSTALL_DIR}" \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-verbs \ + --with-dm \ + --without-gdrcopy \ + ${UCX_CUDA_BUILD_ARGS} && \ + make -j"$NPROC" && \ + $SUDO make -j install-strip && \ + $SUDO ldconfig \ + ) +fi # PRE_INSTALLED_UCX_ENV end + +$SUDO rm -rf ${TMPDIR} + +export UCX_TLS=^cuda_ipc + +# Lastly build and install nixl + nixlbench +if [ -n "$PRE_INSTALLED_NIXL_ENV" ]; then + echo "PRE_INSTALLED_NIXL_ENV is set, skipping compilation" +else + if [ "${BUILD_NIXL_EP}" = "true" ]; then + EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS} -Dbuild_nixl_ep=true" + fi + # This image is CUDA-free. Restrict building to plugins that do not require cuFile. + # POSIX alone is enough to produce libnixl and nixlbench. + NIXL_ENABLE_PLUGINS="${NIXL_ENABLE_PLUGINS:-POSIX}" + # shellcheck disable=SC2086 + meson setup \ + ${NIXL_BUILD_DIR} \ + --prefix=${INSTALL_DIR} \ + -Ducx_path=${UCX_INSTALL_DIR} \ + -Dbuild_docs=false \ + -Drust=false \ + ${EXTRA_BUILD_ARGS} \ + -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" \ + --buildtype=debug \ + -Denable_plugins="${NIXL_ENABLE_PLUGINS}" + ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} + ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install + mkdir -p dist && cp ${NIXL_BUILD_DIR}/src/bindings/python/nixl-meta/nixl-*.whl dist/ + + # TODO(kapila): Copy the nixl.pc file to the install directory if needed. + # cp ${BUILD_DIR}/nixl.pc ${INSTALL_DIR}/lib/pkgconfig/nixl.pc + + cd benchmark/nixlbench + meson setup \ + ${NIXLBENCH_BUILD_DIR} \ + -Dnixl_path=${INSTALL_DIR} \ + -Dprefix=${INSTALL_DIR} \ + -Duse_rocm=true \ + -Drocm_path="${ROCM_INSTALL_PATH}" + ninja -j"$NPROC" -C ${NIXLBENCH_BUILD_DIR} + ninja -j"$NPROC" -C ${NIXLBENCH_BUILD_DIR} install +fi # PRE_INSTALLED_NIXL_ENV