nixl re-sync 4-23-2026

.ci/dockerfiles/Dockerfile.build_helper

@@ -11,6 +11,7 @@ ARG BASE_IMAGE=dockerhub.nvidia.com/ubuntu:24.04
 FROM ${BASE_IMAGE}
 
 # Build arguments
+ARG ARCH=x86_64
 ARG _UID=148069
 ARG _GID=30
 ARG _LOGIN=svc-nixl
@@ -24,12 +25,16 @@ LABEL description="NIXL development environment"
 LABEL version="1.0.0"
 
 RUN apt update && \
-    apt install -y sudo podman curl jq && \
+    apt install -y sudo podman curl jq openssh-client git && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Install scctl for slurm client management
-RUN curl -s https://urm.nvidia.com/artifactory/sw-swx-lab-infra-generic/scctl/get-app.sh | bash -s -- --output-dir /usr/local/bin
+# Install scctl for slurm client management (x86_64 only, not available for ARM64)
+RUN if [ "$(uname -m)" = "x86_64" ]; then \
+        curl -s https://urm.nvidia.com/artifactory/sw-swx-lab-infra-generic/scctl/get-app.sh | bash -s -- --output-dir /usr/local/bin || echo "INFO: scctl installation failed, continuing..."; \
+    else \
+        echo "INFO: Skipping scctl installation on $(uname -m) architecture (not available)"; \
+    fi
 
 # Create group and user in one RUN command to reduce layers
 RUN if ! getent group "${_GID}" > /dev/null 2>&1; then \

.ci/jenkins/lib/build-container-matrix.yaml

@@ -16,7 +16,7 @@ kubernetes:
   requests: "{memory: 8Gi, cpu: 4000m}"
 
 runs_on_dockers:
-  - { name: "podman-v5.7.1", url: "quay.io/podman/stable:v5.7.1", privileged: true }
+  - { name: "podman", url: "quay.io/podman/stable:v5.7.1", privileged: true }
 
 # Build matrix
 matrix:
@@ -27,12 +27,15 @@ matrix:
 
 # Configuration
 env:
-  REGISTRY_HOSTESS: "artifactory.nvidia.com"
-  REGISTRY_REPO: "sw-nbu-swx-nixl-docker-local/verification"
+  REGISTRY_HOST_NAME: "artifactory.nvidia.com"
+  REGISTRY_REPO_NAME: "sw-nbu-swx-nixl-docker-local"
+  REGISTRY_REPO_PATH: "verification"
+  ARTIFACTORY_REGISTRY_URL: "${REGISTRY_HOST_NAME}/${REGISTRY_REPO_NAME}/${REGISTRY_REPO_PATH}"
+  ARTIFACTORY_CLEANUP_AGE: "3m" # 3 months
   LOCAL_TAG_BASE: "nixl-ci:build-"
   MAIL_FROM: "jenkins@nvidia.com"
 
-taskName: "${BUILD_TARGET}/${arch}/${axis_index}"
+taskName: "${BUILD_TARGET}/${arch}"
 
 credentials:
   - credentialsId: 'svc-nixl-new-artifactory-token'
@@ -56,6 +59,7 @@ pipeline_start:
 # Build pipeline
 steps:
   - name: Prepare
+    containerSelector: "{name: 'podman'}"
     run: |
       # Setup podman and dependencies
       set -x
@@ -67,6 +71,7 @@ steps:
 
   - name: Build NIXLBench
     enable: ${ENABLE_NIXLBENCH_BUILD}
+    containerSelector: "{name: 'podman'}"
     run: |
       # Clone UCX source for nixlbench
       git clone https://github.com/openucx/ucx.git ucx-src
@@ -83,6 +88,7 @@ steps:
 
   - name: Build NIXL
     enable: ${ENABLE_NIXL_BUILD}
+    containerSelector: "{name: 'podman'}"
     run: |
       export UCX_REF="${UCX_VERSION}"
 
@@ -101,6 +107,7 @@ steps:
         ${NIXL_EP_FLAG}
 
   - name: Add Version Info
+    containerSelector: "{name: 'podman'}"
     run: |
       git config --global --add safe.directory '*'
       # Extract standardized 8-char commit hash for UCX version info:
@@ -138,19 +145,20 @@ steps:
       docker rm -f tempcontainer || true
 
   - name: Push
+    containerSelector: "{name: 'podman'}"
     credentialsId: 'svc-nixl-new-artifactory-token'
     run: |
       source version-info
-      ARTIFACTORY_REGISTRY="${REGISTRY_HOSTESS}/${REGISTRY_REPO}/${BUILD_TARGET}"
-      ARTIFACTORY_API="https://${REGISTRY_HOSTESS}/artifactory/api/storage/${REGISTRY_REPO}/${BUILD_TARGET}"
+      ARTIFACTORY_REGISTRY="${ARTIFACTORY_REGISTRY_URL}/${BUILD_TARGET}"
+      ARTIFACTORY_API="https://${REGISTRY_HOST_NAME}/artifactory/api/storage/${REGISTRY_REPO_NAME}/${REGISTRY_REPO_PATH}/${BUILD_TARGET}"
 
       # Prepare image properties
       IMAGE_PROPERTIES="BUILD_TARGET=${BUILD_TARGET};NIXL_VERSION=${NIXL_VERSION};UCX_VERSION=${UCX_VERSION};arch=${arch};"
       IMAGE_PROPERTIES+="BUILD_NUMBER=${BUILD_NUMBER};JOB_NAME=${JOB_NAME};BUILD_URL=${BUILD_URL};NODE_NAME=${NODE_NAME};"
       IMAGE_PROPERTIES+="BASE_IMAGE=${BASE_IMAGE};BASE_IMAGE_TAG=${BASE_IMAGE_TAG}"
 
       # Login to Artifactory
-      echo "$ARTIFACTORY_PASSWORD" | docker login "${REGISTRY_HOSTESS}" -u "$ARTIFACTORY_USERNAME" --password-stdin
+      echo "$ARTIFACTORY_PASSWORD" | docker login "${REGISTRY_HOST_NAME}" -u "$ARTIFACTORY_USERNAME" --password-stdin
 
       # Function to tag, push, and set properties
       tag_push_set_properties() {
@@ -174,9 +182,9 @@ steps:
     run: |
       source version-info
       echo "Image type built: ${BUILD_TARGET} (${arch})"
-      echo "Image pushed to: ${REGISTRY_HOSTESS}/${REGISTRY_REPO}/${BUILD_TARGET}:${TAG_NAME}"
+      echo "Image pushed to: ${ARTIFACTORY_REGISTRY_URL}/${BUILD_TARGET}:${TAG_NAME}"
       if [[ "${UPDATE_LATEST}" == "true" ]]; then
-        echo "Latest tag updated: ${REGISTRY_HOSTESS}/${REGISTRY_REPO}/${BUILD_TARGET}:${BASE_IMAGE_TAG}-${arch}-latest"
+        echo "Latest tag updated: ${ARTIFACTORY_REGISTRY_URL}/${BUILD_TARGET}:${BASE_IMAGE_TAG}-${arch}-latest"
       fi
 
       echo -e "\nBuild config for manual repro:"
@@ -195,6 +203,7 @@ steps:
 pipeline_stop:
   shell: action
   module: groovy
+  containerSelector: "{name: 'podman'}"
   run: |
     if (params.MAIL_TO) {
         def jobStatus = currentBuild.result ?: 'SUCCESS'
@@ -223,3 +232,6 @@ pipeline_stop:
             """
         )
     }
+    withCredentials([usernamePassword(credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'ARTIFACTORY_USER', passwordVariable: 'ARTIFACTORY_TOKEN')]) {
+      sh "yum install -y jq > /dev/null && .ci/scripts/artifactory-cleanup-by-time.sh --path ${REGISTRY_REPO_PATH}/${BUILD_TARGET} ${REGISTRY_REPO_NAME} ${ARTIFACTORY_CLEANUP_AGE}"
+    }

.ci/jenkins/lib/build-matrix.yaml

@@ -43,7 +43,7 @@ env:
   TEST_TIMEOUT: 30
   UCX_TLS: "^shm"
   STORAGE_DRIVER: 'overlay'
-  CI_IMAGE_TAG: "20260323-1"
+  CI_IMAGE_TAG: "20260414-3"
 
 
 runs_on_dockers:

.ci/jenkins/lib/test-dl-matrix.yaml

@@ -0,0 +1,194 @@
+---
+#
+# DLCluster GPU Test Matrix Configuration for dlcluster.nvidia.com
+#
+# Key Components:
+# - Job Configuration: Defines timeout, failure behavior, and server resources
+# - Docker Images: Specifies the container images used for different build stages
+# - Matrix Axes: Defines build variations for dlcluster testing (multi-node, multi-GPU)
+# - Run Steps: Sequential steps for running dlcluster GPU tests
+#
+# When Modified:
+# - Adding/removing Docker images: Affects available test environments
+# - Modifying matrix axes: Changes test variations (e.g., adding architectures)
+# - Adjusting resource limits: Impacts test performance and resource allocation
+# - Adding/removing steps: Changes the test pipeline sequence
+#
+# Note: Changes to this file are tested as part of the PR CI flow no need to test them manually.
+
+
+job: nixl-ci-dl-gpu
+
+# Fail job if one of the steps fails or continue
+failFast: false
+
+timeout_minutes: 240
+
+registry_host: harbor.mellanox.com
+registry_auth: nixl_harbor_credentials
+registry_path: /nixl/base
+
+kubernetes:
+  cloud: il-ipp-blossom-prod
+  namespace: nbu-swx-nixl
+  limits: "{memory: 16Gi, cpu: 16000m}"
+  requests: "{memory: 8Gi, cpu: 8000m}"
+  privileged: true
+
+credentials:
+  - {credentialsId: 'nixl_harbor_credentials', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'}
+
+env:
+  NIXL_INSTALL_DIR: /opt/nixl
+  NIXL_BUILD_DIR: nixl_build
+  SLURM_NODES: 1
+  SLURM_PARTITION: gb300nvl72_ci
+  SLURM_HEAD_NODE: dlcluster.nvidia.com
+  SLURM_HEAD_USER: svc-nixl
+  SLURM_ACCOUNT: 'blackwell-ci'
+  SLURM_JOB_TIMEOUT: '01:30:00'
+  TEST_TIMEOUT: 50
+  STORAGE_DRIVER: overlay
+  CI_IMAGE_TAG: "20260414-3"
+
+empty_volumes:
+  - {mountPath: /var/lib/containers/storage, memory: false}
+
+pvc_volumes:
+  - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false}
+
+volumes:
+  - { mountPath: "/home/svc-nixl", hostPath: "/labhome/svc-nixl" }
+
+# Docker images for DL testing
+runs_on_dockers:
+  - {
+     file: '.ci/dockerfiles/Dockerfile.base',
+     name: 'nixl-ci-dl-gpu-base-25.10-cuda13.0-ubuntu24.04',
+     tag: "${CI_IMAGE_TAG}",
+     arch: "aarch64",
+     build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR}  --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.10-cuda13.0-devel-ubuntu24.04 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache'
+    }
+
+  - {
+     file: '.ci/dockerfiles/Dockerfile.build_helper',
+     name: 'build_helper_dl',
+     arch: "aarch64",
+     tag: "${CI_IMAGE_TAG}",
+     build_args: '--build-arg BASE_IMAGE=dockerhub.nvidia.com/ubuntu:24.04 --build-arg ARCH=${arch}'
+    }
+
+matrix:
+  axes:
+    arch:
+      - aarch64
+    ucx_version:
+      - master
+      - v1.20.x
+
+taskName: "${name}/${arch}/ucx-${ucx_version}/${axis_index}"
+
+
+steps:
+  - name: Compiling NIXL Docker Image for DL
+    containerSelector: "{name: 'build_helper_dl'}"
+    credentialsId: "nixl_harbor_credentials"
+    parallel: false
+    run: |
+      set -x
+      export PR_IMAGE=${registry_host}/nixl/pr/${arch}/nixl-ci-dl-gpu-test-${ucx_version}:${BUILD_NUMBER}
+      rm -rf /etc/containers/storage.conf && rm -f /usr/share/containers/storage.conf
+      podman build --network host \
+      --build-arg UCX_VERSION=${ucx_version} \
+      --build-arg PRE_INSTALLED_ENV="true" \
+      --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} \
+      --build-arg NIXL_BUILD_DIR=${NIXL_BUILD_DIR} \
+      --build-arg HAS_GPU=true \
+      --build-arg BASE_IMAGE=${registry_host}${registry_path}/${arch}/nixl-ci-dl-gpu-base-25.10-cuda13.0-ubuntu24.04:${CI_IMAGE_TAG} \
+      --tag ${PR_IMAGE} \
+      -f .ci/dockerfiles/Dockerfile.gpu-test .
+      podman push --creds ${REPO_USER}:${REPO_PASS} ${PR_IMAGE}
+
+  - name: Allocate DL Environment
+    containerSelector: "{name: 'build_helper_dl'}"
+    parallel: false
+    run: |
+      sudo -E -u svc-nixl .ci/scripts/run_slurm_allocation.sh --slurm_partition=${SLURM_PARTITION} \
+      --slurm_nodes=${SLURM_NODES} \
+      --slurm_head_node=${SLURM_HEAD_NODE} \
+      --slurm_job_timeout=${SLURM_JOB_TIMEOUT} \
+      --slurm_gres=${SLURM_GRES} \
+      --slurm_mem=${SLURM_MEM} \
+      --slurm_mincpus=${SLURM_MINCPUS} \
+      --slurm_job_name=nixl-dl-${ucx_version}-${BUILD_NUMBER} \
+      --slurm_job_id_file=/mnt/pvc/dl_job_id_${ucx_version}_${BUILD_NUMBER}.txt
+
+  - name: Run DL Python tests
+    containerSelector: "{name: 'build_helper_dl'}"
+    timeout: "${TEST_TIMEOUT}"
+    parallel: false
+    run: |
+      set -x
+      sudo -E -u svc-nixl .ci/scripts/run_tests_slurm.sh --test_script_path=".gitlab/test_python.sh" \
+        --container_name=nixl-dl-${ucx_version}-${BUILD_NUMBER} \
+        --slurm_job_id=$(cat /mnt/pvc/dl_job_id_${ucx_version}_${BUILD_NUMBER}.txt) \
+        --docker_image="${registry_host}#nixl/pr/${arch}/nixl-ci-dl-gpu-test-${ucx_version}:${BUILD_NUMBER}"
+
+  - name: Run DL Rust tests
+    containerSelector: "{name: 'build_helper_dl'}"
+    timeout: "${TEST_TIMEOUT}"
+    parallel: false
+    run: |
+      set -x
+      sudo -E -u svc-nixl .ci/scripts/run_tests_slurm.sh --test_script_path=".gitlab/test_rust.sh" \
+        --container_name=nixl-dl-${ucx_version}-${BUILD_NUMBER} \
+        --slurm_job_id=$(cat /mnt/pvc/dl_job_id_${ucx_version}_${BUILD_NUMBER}.txt) \
+        --docker_image="${registry_host}#nixl/pr/${arch}/nixl-ci-dl-gpu-test-${ucx_version}:${BUILD_NUMBER}"
+
+  - name: Run DL CPP tests
+    containerSelector: "{name: 'build_helper_dl'}"
+    timeout: "${TEST_TIMEOUT}"
+    parallel: false
+    run: |
+      set -x
+      sudo -E -u svc-nixl .ci/scripts/run_tests_slurm.sh --test_script_path=".gitlab/test_cpp.sh" \
+        --container_name=nixl-dl-${ucx_version}-${BUILD_NUMBER} \
+        --slurm_job_id=$(cat /mnt/pvc/dl_job_id_${ucx_version}_${BUILD_NUMBER}.txt) \
+        --docker_image="${registry_host}#nixl/pr/${arch}/nixl-ci-dl-gpu-test-${ucx_version}:${BUILD_NUMBER}"
+
+  - name: Run DL Nixlbench tests
+    containerSelector: "{name: 'build_helper_dl'}"
+    timeout: "${TEST_TIMEOUT}"
+    parallel: false
+    run: |
+      set -x
+      sudo -E -u svc-nixl .ci/scripts/run_tests_slurm.sh --test_script_path="HAS_GPU=false .gitlab/test_nixlbench.sh" \
+        --container_name=nixl-dl-${ucx_version}-${BUILD_NUMBER} \
+        --slurm_job_id=$(cat /mnt/pvc/dl_job_id_${ucx_version}_${BUILD_NUMBER}.txt) \
+        --docker_image="${registry_host}#nixl/pr/${arch}/nixl-ci-dl-gpu-test-${ucx_version}:${BUILD_NUMBER}"
+
+
+pipeline_stop:
+  containerSelector:
+    - "{name: 'build_helper_dl'}"
+  parallel: false
+  run: |
+    set -x
+    # Find all dl_job_id_*.txt files and stop each allocation
+    shopt -s nullglob
+    job_files=(/mnt/pvc/dl_job_id_*_${BUILD_NUMBER}.txt)
+    shopt -u nullglob
+
+    if [ ${#job_files[@]} -eq 0 ]; then
+      echo "WARNING: No DL job ID files found in /mnt/pvc/"
+      exit 0
+    fi
+
+    echo "INFO: Found ${#job_files[@]} DL job ID file(s) to stop"
+    for job_file in "${job_files[@]}"; do
+      if [ -f "${job_file}" ]; then
+        echo "INFO: Stopping DL allocation from ${job_file}"
+        sudo -E -u svc-nixl .ci/scripts/stop_slurm_allocation.sh --slurm_job_id_file="${job_file}"
+        rm -f "${job_file}"
+      fi
+    done