ai-dynamo · NirWolfer · Jul 15, 2026 · Jul 13, 2026
@@ -1,6 +1,9 @@
 #!/bin/bash -eE
+set -o pipefail
 
-# Files to check for changes
+# CI source files whose last-touching commit determines CI_IMAGE_TAG.
+# When any of these files change in a commit, the derived tag changes
+# and the matrix jobs rebuild their base Docker images automatically.
 CI_FILES=(
     ".ci/dockerfiles/Dockerfile.base"
     ".ci/dockerfiles/Dockerfile.gpu-test"
@@ -10,136 +13,32 @@ CI_FILES=(
     "contrib/Dockerfile.manylinux"
 )
 
-# YAML files containing CI_IMAGE_TAG
-BUILD_MATRIX_YAML=".ci/jenkins/lib/build-matrix.yaml"
-TEST_MATRIX_YAML=".ci/jenkins/lib/test-matrix.yaml"
-TEST_DL_MATRIX_YAML=".ci/jenkins/lib/test-dl-matrix.yaml"
-TEST_DL_EP_MATRIX_YAML=".ci/jenkins/lib/test-dl-ep-matrix.yaml"
-SANITIZER_MATRIX_YAML=".ci/jenkins/lib/test-sanitizer-matrix.yaml"
-WHEEL_BUILD_MATRIX_YAML=".ci/jenkins/lib/build-wheel-matrix.yaml"
-
-# Function to extract CI_IMAGE_TAG from a YAML file
-get_ci_image_tag() {
-    local file=$1
-    local ref=$2  # git ref (HEAD, HEAD~1, etc.)
-
-    if [ -z "$ref" ]; then
-        # Current working tree
-        grep -E '^\s*CI_IMAGE_TAG:' "$file" | sed 's/.*CI_IMAGE_TAG:\s*"\(.*\)".*/\1/' | tr -d ' '
-    else
-        # Previous commit
-        git show "${ref}:${file}" 2>/dev/null | grep -E '^\s*CI_IMAGE_TAG:' | sed 's/.*CI_IMAGE_TAG:\s*"\(.*\)".*/\1/' | tr -d ' '
-    fi
-}
-
-# Check if we're in a git repository
-if ! git rev-parse --git-dir > /dev/null 2>&1; then
-    echo "Not a git repository. Skipping CI_IMAGE_TAG check."
-    echo "Ok"
-    exit 0
-fi
-
-# Check if any CI files were changed in the last commit or working tree
-files_changed=false
-for file in "${CI_FILES[@]}"; do
-    # Check if file was changed in HEAD commit or has uncommitted changes
-    if git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -q "^${file}$" || \
-       git diff --name-only 2>/dev/null | grep -q "^${file}$" || \
-       git diff --cached --name-only 2>/dev/null | grep -q "^${file}$"; then
-        echo "Detected changes in: $file"
-        files_changed=true
-    fi
-done
-
-if [ "$files_changed" = false ]; then
-    echo "No changes detected in CI files. Skipping CI_IMAGE_TAG check."
-    echo "Ok"
-    exit 0
-fi
-
-# Files were changed, now check if CI_IMAGE_TAG was increased
-echo "CI files were modified. Checking if CI_IMAGE_TAG was increased..."
-
-# Get current and previous CI_IMAGE_TAG values
-current_build_image_tag=$(get_ci_image_tag "$BUILD_MATRIX_YAML" "")
-current_test_image_tag=$(get_ci_image_tag "$TEST_MATRIX_YAML" "")
-current_test_dl_image_tag=$(get_ci_image_tag "$TEST_DL_MATRIX_YAML" "")
-current_test_dl_ep_image_tag=$(get_ci_image_tag "$TEST_DL_EP_MATRIX_YAML" "")
-current_sanitizer_image_tag=$(get_ci_image_tag "$SANITIZER_MATRIX_YAML" "")
-current_wheel_build_image_tag=$(get_ci_image_tag "$WHEEL_BUILD_MATRIX_YAML" "")
-
-previous_build_image_tag=$(get_ci_image_tag "$BUILD_MATRIX_YAML" "HEAD~1")
-previous_test_image_tag=$(get_ci_image_tag "$TEST_MATRIX_YAML" "HEAD~1")
-previous_test_dl_image_tag=$(get_ci_image_tag "$TEST_DL_MATRIX_YAML" "HEAD~1")
-previous_test_dl_ep_image_tag=$(get_ci_image_tag "$TEST_DL_EP_MATRIX_YAML" "HEAD~1")
-previous_sanitizer_image_tag=$(get_ci_image_tag "$SANITIZER_MATRIX_YAML" "HEAD~1")
-previous_wheel_build_image_tag=$(get_ci_image_tag "$WHEEL_BUILD_MATRIX_YAML" "HEAD~1")
-
-echo "Build Matrix CI_IMAGE_TAG:     $previous_build_image_tag -> $current_build_image_tag"
-echo "Test Matrix CI_IMAGE_TAG:      $previous_test_image_tag -> $current_test_image_tag"
-echo "Test DL Matrix CI_IMAGE_TAG:   $previous_test_dl_image_tag -> $current_test_dl_image_tag"
-echo "Test DL EP Matrix CI_IMAGE_TAG: $previous_test_dl_ep_image_tag -> $current_test_dl_ep_image_tag"
-echo "Sanitizer Matrix CI_IMAGE_TAG: $previous_sanitizer_image_tag -> $current_sanitizer_image_tag"
-echo "Wheel Build Matrix CI_IMAGE_TAG: $previous_wheel_build_image_tag -> $current_wheel_build_image_tag"
-
-# Check if CI_IMAGE_TAG was changed in all files
-build_tag_changed=false
-test_tag_changed=false
-test_dl_tag_changed=false
-test_dl_ep_tag_changed=false
-sanitizer_tag_changed=false
-wheel_build_tag_changed=false
-
-if [ "$current_build_image_tag" != "$previous_build_image_tag" ]; then
-    echo "✓ CI_IMAGE_TAG in build-matrix.yaml was updated"
-    build_tag_changed=true
-fi
-
-if [ "$current_test_image_tag" != "$previous_test_image_tag" ]; then
-    echo "✓ CI_IMAGE_TAG in test-matrix.yaml was updated"
-    test_tag_changed=true
-fi
-
-if [ "$current_test_dl_image_tag" != "$previous_test_dl_image_tag" ]; then
-    echo "✓ CI_IMAGE_TAG in test-dl-matrix.yaml was updated"
-    test_dl_tag_changed=true
-fi
+# Matrix YAML files that contain the CI_MANAGED placeholder.
+# These are patched in the Jenkins workspace before the matrix library
+# reads them — no commit or push is made.
+YAML_FILES=(
+    ".ci/jenkins/lib/build-matrix.yaml"
+    ".ci/jenkins/lib/test-matrix.yaml"
+    ".ci/jenkins/lib/test-dl-matrix.yaml"
+    ".ci/jenkins/lib/test-dl-ep-matrix.yaml"
+    ".ci/jenkins/lib/test-sanitizer-matrix.yaml"
+    ".ci/jenkins/lib/build-wheel-matrix.yaml"
+)
 
-if [ "$current_test_dl_ep_image_tag" != "$previous_test_dl_ep_image_tag" ]; then
-    echo "✓ CI_IMAGE_TAG in test-dl-ep-matrix.yaml was updated"
-    test_dl_ep_tag_changed=true
-fi
+# Derive the tag from the most recent commit that touched any CI file.
+NEW_TAG=$(git log -1 --format=%h -- "${CI_FILES[@]}")
 
-if [ "$current_sanitizer_image_tag" != "$previous_sanitizer_image_tag" ]; then
-    echo "✓ CI_IMAGE_TAG in test-sanitizer-matrix.yaml was updated"
-    sanitizer_tag_changed=true
+# Fallback: if no commit has ever touched those files (should not happen
+# in practice), use a sha256sum of their content truncated to 12 chars.
+if [ -z "$NEW_TAG" ]; then
+    echo "Warning: git log returned empty for CI files. Falling back to content hash."
+    NEW_TAG=$(cat "${CI_FILES[@]}" | sha256sum | cut -c1-12)
 fi
 
-if [ "$current_wheel_build_image_tag" != "$previous_wheel_build_image_tag" ]; then
-    echo "✓ CI_IMAGE_TAG in build-wheel-matrix.yaml was updated"
-    wheel_build_tag_changed=true
-fi
+echo "CI_IMAGE_TAG derived as: ${NEW_TAG}"
 
-if [ "$build_tag_changed" = false ] || [ "$test_tag_changed" = false ] || [ "$test_dl_tag_changed" = false ] || [ "$test_dl_ep_tag_changed" = false ] || [ "$sanitizer_tag_changed" = false ] || [ "$wheel_build_tag_changed" = false ]; then
-    echo ""
-    echo "❌ ERROR: You have changed CI files but forgot to increase CI_IMAGE_TAG!"
-    echo ""
-    echo "Changed files:"
-    for file in "${CI_FILES[@]}"; do
-        if git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -q "^${file}$" || \
-           git diff --name-only 2>/dev/null | grep -q "^${file}$" || \
-           git diff --cached --name-only 2>/dev/null | grep -q "^${file}$"; then
-            echo "  - $file"
-        fi
-    done
-    echo ""
-    echo "Please update CI_IMAGE_TAG in:"
-    echo "  - $BUILD_MATRIX_YAML (line 46)"
-    echo "  - $TEST_MATRIX_YAML (line 53)"
-    echo "  - $TEST_DL_MATRIX_YAML (line 52)"
-    echo "  - $TEST_DL_EP_MATRIX_YAML"
-    echo "  - $SANITIZER_MATRIX_YAML"
-    echo "  - $WHEEL_BUILD_MATRIX_YAML"
-    echo ""
-    exit 1
-fi
+for yaml in "${YAML_FILES[@]}"; do
+    grep -q 'CI_IMAGE_TAG: "CI_MANAGED"' "$yaml" || { echo "ERROR: CI_MANAGED placeholder missing in $yaml" >&2; exit 1; }
+    sed -i "s/CI_IMAGE_TAG: \"CI_MANAGED\"/CI_IMAGE_TAG: \"${NEW_TAG}\"/" "$yaml"
+    echo "Patched: $yaml"
+done
@@ -170,7 +170,7 @@ their own nightly/manual trigger. They split into two groups:
 - **Config:** `.ci/jenkins/lib/build-wheel-matrix.yaml`
 - **What it does:** Builds NIXL Python wheels for each Python version × architecture combination. Uses a two-stage `contrib/Dockerfile.manylinux` build: the `wheel_base` stage (all slow deps: UCX, gRPC, Rust, etc.) is pre-built and cached in Artifactory under `CI_IMAGE_TAG`; PR builds pull this pre-built image and run only the `wheel` stage (~16 steps). PR builds also pass `--torch-versions` (set via the `TORCH_VERSIONS` env in the matrix file) to limit the torch extension builds to the latest version. x86_64 wheels build in the `manylinux` (podman-in-container) runner.
 - **vLLM/SGLang NIXL sanity (aarch64):** the same job also gates a 1-prefill/1-decode NIXL KV-transfer sanity for vLLM and SGLang. Each framework runs as its own aarch64 branch (`build_helper_vllm` / `build_helper_sglang`): build the aarch64 wheels (reusing the cached `wheel_base`), layer them onto the pinned framework base image (`.ci/dockerfiles/Dockerfile.{vllm,sglang}-base`, built as `category: tool` by ci-demo), then run the sanity on the `gb200nvl72_ci` dlcluster SLURM partition over SSH (`.gitlab/test_vllm_sglang_sanity.sh`). SGLang additionally asserts a gsm8k accuracy floor on `Qwen/Qwen3-8B`. The aarch64 wheels are built once per framework branch (duplicated) so the two flows stay separate, labeled branches in the Jenkins UI.
-- **`CI_IMAGE_TAG`:** Same convention as the other five matrix files (also tags `build_helper_*` and the sanity `*-nixl-base` images). `contrib/Dockerfile.manylinux` is part of the `CI_FILES` list in `cidemo-init.sh`, so changing it (or any other CI file) without bumping `CI_IMAGE_TAG` in all six matrix YAMLs will fail the pre-commit check.
+- **`CI_IMAGE_TAG`:** Same convention as the other five matrix files (also tags `build_helper_*` and the sanity `*-nixl-base` images). `contrib/Dockerfile.manylinux` is part of the `CI_FILES` list in `cidemo-init.sh`, so changing it (or any other CI file) automatically derives a new `CI_IMAGE_TAG` and rebuilds the cached `wheel_base` image (see [CI_IMAGE_TAG management](#ci_image_tag-management)).
 
 ### `nixl-ci-build-container` (standalone)
 - **Trigger:** Nightly cron (builds `nixlbench` and `nixl` targets, on both the default CUDA base image and the DLFW PyTorch daily image, ~3–4 AM), or manual run with parameters (`BUILD_TARGET`, `NIXL_VERSION`, `UCX_VERSION`, base image overrides, etc.).
@@ -204,6 +204,35 @@ their own nightly/manual trigger. They split into two groups:
 - **Re-run a single Jenkins job with different parameters:** use `workflow_dispatch` on `blossom-ci.yml`, or trigger the Jenkins job directly if you have Jenkins access.
 - **Container/wheel builds outside the nightly schedule:** run `nixl-ci-build-container` or `nixl-ci-build-wheel-nightly` manually from the Jenkins UI with custom parameters.
 
+## CI_IMAGE_TAG management
+
+`CI_IMAGE_TAG` is the Docker image tag used by all matrix jobs to identify the
+base images they build and pull. It appears as `CI_IMAGE_TAG: "CI_MANAGED"` in
+the six matrix YAML files — this placeholder is intentional and must not be
+replaced with a static value.
+
+At the start of every Jenkins run, `cidemo-init.sh` derives the real tag
+automatically:
+
+```bash
+NEW_TAG=$(git log -1 --format=%h -- "${CI_FILES[@]}")
+```
+
+This returns the short git commit hash of the most recent commit that touched
+any of the CI source files (`Dockerfile.base`, `Dockerfile.gpu-test`,
+`Dockerfile.build_helper`, `build.sh`, `common.sh`, `Dockerfile.manylinux`). It
+then patches all six YAML files in the Jenkins workspace with `sed` before the
+matrix library reads them. No commit or push is made — the patch exists only in
+the workspace.
+
+**Caching behaviour:** the derived tag is stable as long as the CI source files
+are unchanged. Two PRs that both leave the CI files untouched get the same tag
+and share the cached Artifactory images. A PR that changes a Dockerfile gets a
+new tag and triggers a rebuild automatically.
+
+**You never need to manually update `CI_IMAGE_TAG`.** The `CI_MANAGED`
+placeholder signals this clearly.
+
 ## Related docs
 
 - [Build Wheel Matrix CI Job Documentation](build-wheel-matrix-ci.md) — deep dive into `nixl-ci-build-wheel`.

@@ -45,7 +45,9 @@ env:
   TEST_TIMEOUT: 30
   UCX_TLS: "^shm"
   STORAGE_DRIVER: 'overlay'
-  CI_IMAGE_TAG: "20260712-2"
+  # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files;
+  # the "CI_MANAGED" placeholder must not be hand-edited.
+  CI_IMAGE_TAG: "CI_MANAGED"
 
 runs_on_dockers:
   - {

@@ -184,8 +184,9 @@ env:
   GRPC_NPROC: 10
   BASE_IMAGE: "${registry_host}/sw-nbu-swx-nixl-docker-local/base/cuda"
   BASE_TAG: '13.0-devel-manylinux--25.09'
-  # CI image tag for wheel_base + build_helper + the sanity base images (bump to rebuild them)
-  CI_IMAGE_TAG: "20260709-1"
+  # CI image tag for wheel_base + build_helper + the sanity base images
+  # (auto-derived by cidemo-init.sh — the "CI_MANAGED" placeholder must not be hand-edited)
+  CI_IMAGE_TAG: "CI_MANAGED"
   WHL_BASE: "manylinux_2_28"
   TORCH_VERSIONS: "2.13"
   WHEEL_BASE_IMAGE_NAME: "nixl-wheel-base-manylinux_2_28"

@@ -60,7 +60,9 @@ env:
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
   TEST_TIMEOUT: 50
   STORAGE_DRIVER: overlay
-  CI_IMAGE_TAG: "20260712-2"
+  # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files;
+  # the "CI_MANAGED" placeholder must not be hand-edited.
+  CI_IMAGE_TAG: "CI_MANAGED"
 
 empty_volumes:
   - {mountPath: /var/lib/containers/storage, memory: false}

@@ -54,7 +54,9 @@ env:
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
   TEST_TIMEOUT: 50
   STORAGE_DRIVER: overlay
-  CI_IMAGE_TAG: "20260712-2"
+  # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files;
+  # the "CI_MANAGED" placeholder must not be hand-edited.
+  CI_IMAGE_TAG: "CI_MANAGED"
 
 empty_volumes:
   - {mountPath: /var/lib/containers/storage, memory: false}

@@ -54,7 +54,9 @@ env:
   SCCTL_CREDENTIALS_ID: 'svc-nixl-scctl'
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
   STORAGE_DRIVER: overlay
-  CI_IMAGE_TAG: "20260712-2"
+  # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files;
+  # the "CI_MANAGED" placeholder must not be hand-edited.
+  CI_IMAGE_TAG: "CI_MANAGED"
   ENROOT_MOUNT_PATH: "/enroot_images"  # NFS mount on mizu nodes; on failure the container is exported here as <containerName>.sqsh
   ENROOT_DATA_PATH: "/enroot-data/enroot-data-148069/user-148069"  # svc-nixl user on mizu nodes
 

@@ -15,8 +15,8 @@
 #    prebuilt, non-instrumented abseil, TSan misses the lock and every
 #    nixlLock-protected access shows up as a false race.
 #
-# Note: changes to .ci/dockerfiles/Dockerfile.base, .gitlab/build.sh or
-# .ci/scripts/common.sh require bumping CI_IMAGE_TAG here too (see cidemo-init.sh).
+# Note: CI_IMAGE_TAG is auto-derived by cidemo-init.sh from the CI source files
+# and patched in at CI time — the "CI_MANAGED" placeholder must not be hand-edited.
 
 ---
 job: nixl-ci-test-sanitizers
@@ -42,7 +42,7 @@ env:
   TEST_TIMEOUT: 40
   UCX_TLS: "^shm"
   STORAGE_DRIVER: 'overlay'
-  CI_IMAGE_TAG: "20260712-2"
+  CI_IMAGE_TAG: "CI_MANAGED"
 
 runs_on_dockers:
   # TSan-instrumented dependency image for the tsan build (DEPS_SANITIZE=thread