diff --git a/.ci/cidemo-init.sh b/.ci/cidemo-init.sh index 5399be278d..6f879dacf9 100755 --- a/.ci/cidemo-init.sh +++ b/.ci/cidemo-init.sh @@ -1,6 +1,9 @@ #!/bin/bash -eE +set -o pipefail -# Files to check for changes +# CI source files whose last-touching commit determines CI_IMAGE_TAG. +# When any of these files change in a commit, the derived tag changes +# and the matrix jobs rebuild their base Docker images automatically. CI_FILES=( ".ci/dockerfiles/Dockerfile.base" ".ci/dockerfiles/Dockerfile.gpu-test" @@ -10,136 +13,32 @@ CI_FILES=( "contrib/Dockerfile.manylinux" ) -# YAML files containing CI_IMAGE_TAG -BUILD_MATRIX_YAML=".ci/jenkins/lib/build-matrix.yaml" -TEST_MATRIX_YAML=".ci/jenkins/lib/test-matrix.yaml" -TEST_DL_MATRIX_YAML=".ci/jenkins/lib/test-dl-matrix.yaml" -TEST_DL_EP_MATRIX_YAML=".ci/jenkins/lib/test-dl-ep-matrix.yaml" -SANITIZER_MATRIX_YAML=".ci/jenkins/lib/test-sanitizer-matrix.yaml" -WHEEL_BUILD_MATRIX_YAML=".ci/jenkins/lib/build-wheel-matrix.yaml" - -# Function to extract CI_IMAGE_TAG from a YAML file -get_ci_image_tag() { - local file=$1 - local ref=$2 # git ref (HEAD, HEAD~1, etc.) - - if [ -z "$ref" ]; then - # Current working tree - grep -E '^\s*CI_IMAGE_TAG:' "$file" | sed 's/.*CI_IMAGE_TAG:\s*"\(.*\)".*/\1/' | tr -d ' ' - else - # Previous commit - git show "${ref}:${file}" 2>/dev/null | grep -E '^\s*CI_IMAGE_TAG:' | sed 's/.*CI_IMAGE_TAG:\s*"\(.*\)".*/\1/' | tr -d ' ' - fi -} - -# Check if we're in a git repository -if ! git rev-parse --git-dir > /dev/null 2>&1; then - echo "Not a git repository. Skipping CI_IMAGE_TAG check." - echo "Ok" - exit 0 -fi - -# Check if any CI files were changed in the last commit or working tree -files_changed=false -for file in "${CI_FILES[@]}"; do - # Check if file was changed in HEAD commit or has uncommitted changes - if git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -q "^${file}$" || \ - git diff --name-only 2>/dev/null | grep -q "^${file}$" || \ - git diff --cached --name-only 2>/dev/null | grep -q "^${file}$"; then - echo "Detected changes in: $file" - files_changed=true - fi -done - -if [ "$files_changed" = false ]; then - echo "No changes detected in CI files. Skipping CI_IMAGE_TAG check." - echo "Ok" - exit 0 -fi - -# Files were changed, now check if CI_IMAGE_TAG was increased -echo "CI files were modified. Checking if CI_IMAGE_TAG was increased..." - -# Get current and previous CI_IMAGE_TAG values -current_build_image_tag=$(get_ci_image_tag "$BUILD_MATRIX_YAML" "") -current_test_image_tag=$(get_ci_image_tag "$TEST_MATRIX_YAML" "") -current_test_dl_image_tag=$(get_ci_image_tag "$TEST_DL_MATRIX_YAML" "") -current_test_dl_ep_image_tag=$(get_ci_image_tag "$TEST_DL_EP_MATRIX_YAML" "") -current_sanitizer_image_tag=$(get_ci_image_tag "$SANITIZER_MATRIX_YAML" "") -current_wheel_build_image_tag=$(get_ci_image_tag "$WHEEL_BUILD_MATRIX_YAML" "") - -previous_build_image_tag=$(get_ci_image_tag "$BUILD_MATRIX_YAML" "HEAD~1") -previous_test_image_tag=$(get_ci_image_tag "$TEST_MATRIX_YAML" "HEAD~1") -previous_test_dl_image_tag=$(get_ci_image_tag "$TEST_DL_MATRIX_YAML" "HEAD~1") -previous_test_dl_ep_image_tag=$(get_ci_image_tag "$TEST_DL_EP_MATRIX_YAML" "HEAD~1") -previous_sanitizer_image_tag=$(get_ci_image_tag "$SANITIZER_MATRIX_YAML" "HEAD~1") -previous_wheel_build_image_tag=$(get_ci_image_tag "$WHEEL_BUILD_MATRIX_YAML" "HEAD~1") - -echo "Build Matrix CI_IMAGE_TAG: $previous_build_image_tag -> $current_build_image_tag" -echo "Test Matrix CI_IMAGE_TAG: $previous_test_image_tag -> $current_test_image_tag" -echo "Test DL Matrix CI_IMAGE_TAG: $previous_test_dl_image_tag -> $current_test_dl_image_tag" -echo "Test DL EP Matrix CI_IMAGE_TAG: $previous_test_dl_ep_image_tag -> $current_test_dl_ep_image_tag" -echo "Sanitizer Matrix CI_IMAGE_TAG: $previous_sanitizer_image_tag -> $current_sanitizer_image_tag" -echo "Wheel Build Matrix CI_IMAGE_TAG: $previous_wheel_build_image_tag -> $current_wheel_build_image_tag" - -# Check if CI_IMAGE_TAG was changed in all files -build_tag_changed=false -test_tag_changed=false -test_dl_tag_changed=false -test_dl_ep_tag_changed=false -sanitizer_tag_changed=false -wheel_build_tag_changed=false - -if [ "$current_build_image_tag" != "$previous_build_image_tag" ]; then - echo "✓ CI_IMAGE_TAG in build-matrix.yaml was updated" - build_tag_changed=true -fi - -if [ "$current_test_image_tag" != "$previous_test_image_tag" ]; then - echo "✓ CI_IMAGE_TAG in test-matrix.yaml was updated" - test_tag_changed=true -fi - -if [ "$current_test_dl_image_tag" != "$previous_test_dl_image_tag" ]; then - echo "✓ CI_IMAGE_TAG in test-dl-matrix.yaml was updated" - test_dl_tag_changed=true -fi +# Matrix YAML files that contain the CI_MANAGED placeholder. +# These are patched in the Jenkins workspace before the matrix library +# reads them — no commit or push is made. +YAML_FILES=( + ".ci/jenkins/lib/build-matrix.yaml" + ".ci/jenkins/lib/test-matrix.yaml" + ".ci/jenkins/lib/test-dl-matrix.yaml" + ".ci/jenkins/lib/test-dl-ep-matrix.yaml" + ".ci/jenkins/lib/test-sanitizer-matrix.yaml" + ".ci/jenkins/lib/build-wheel-matrix.yaml" +) -if [ "$current_test_dl_ep_image_tag" != "$previous_test_dl_ep_image_tag" ]; then - echo "✓ CI_IMAGE_TAG in test-dl-ep-matrix.yaml was updated" - test_dl_ep_tag_changed=true -fi +# Derive the tag from the most recent commit that touched any CI file. +NEW_TAG=$(git log -1 --format=%h -- "${CI_FILES[@]}") -if [ "$current_sanitizer_image_tag" != "$previous_sanitizer_image_tag" ]; then - echo "✓ CI_IMAGE_TAG in test-sanitizer-matrix.yaml was updated" - sanitizer_tag_changed=true +# Fallback: if no commit has ever touched those files (should not happen +# in practice), use a sha256sum of their content truncated to 12 chars. +if [ -z "$NEW_TAG" ]; then + echo "Warning: git log returned empty for CI files. Falling back to content hash." + NEW_TAG=$(cat "${CI_FILES[@]}" | sha256sum | cut -c1-12) fi -if [ "$current_wheel_build_image_tag" != "$previous_wheel_build_image_tag" ]; then - echo "✓ CI_IMAGE_TAG in build-wheel-matrix.yaml was updated" - wheel_build_tag_changed=true -fi +echo "CI_IMAGE_TAG derived as: ${NEW_TAG}" -if [ "$build_tag_changed" = false ] || [ "$test_tag_changed" = false ] || [ "$test_dl_tag_changed" = false ] || [ "$test_dl_ep_tag_changed" = false ] || [ "$sanitizer_tag_changed" = false ] || [ "$wheel_build_tag_changed" = false ]; then - echo "" - echo "❌ ERROR: You have changed CI files but forgot to increase CI_IMAGE_TAG!" - echo "" - echo "Changed files:" - for file in "${CI_FILES[@]}"; do - if git diff --name-only HEAD~1 HEAD 2>/dev/null | grep -q "^${file}$" || \ - git diff --name-only 2>/dev/null | grep -q "^${file}$" || \ - git diff --cached --name-only 2>/dev/null | grep -q "^${file}$"; then - echo " - $file" - fi - done - echo "" - echo "Please update CI_IMAGE_TAG in:" - echo " - $BUILD_MATRIX_YAML (line 46)" - echo " - $TEST_MATRIX_YAML (line 53)" - echo " - $TEST_DL_MATRIX_YAML (line 52)" - echo " - $TEST_DL_EP_MATRIX_YAML" - echo " - $SANITIZER_MATRIX_YAML" - echo " - $WHEEL_BUILD_MATRIX_YAML" - echo "" - exit 1 -fi +for yaml in "${YAML_FILES[@]}"; do + grep -q 'CI_IMAGE_TAG: "CI_MANAGED"' "$yaml" || { echo "ERROR: CI_MANAGED placeholder missing in $yaml" >&2; exit 1; } + sed -i "s/CI_IMAGE_TAG: \"CI_MANAGED\"/CI_IMAGE_TAG: \"${NEW_TAG}\"/" "$yaml" + echo "Patched: $yaml" +done diff --git a/.ci/docs/ci-overview.md b/.ci/docs/ci-overview.md index ba85f98781..4ff693d179 100644 --- a/.ci/docs/ci-overview.md +++ b/.ci/docs/ci-overview.md @@ -170,7 +170,7 @@ their own nightly/manual trigger. They split into two groups: - **Config:** `.ci/jenkins/lib/build-wheel-matrix.yaml` - **What it does:** Builds NIXL Python wheels for each Python version × architecture combination. Uses a two-stage `contrib/Dockerfile.manylinux` build: the `wheel_base` stage (all slow deps: UCX, gRPC, Rust, etc.) is pre-built and cached in Artifactory under `CI_IMAGE_TAG`; PR builds pull this pre-built image and run only the `wheel` stage (~16 steps). PR builds also pass `--torch-versions` (set via the `TORCH_VERSIONS` env in the matrix file) to limit the torch extension builds to the latest version. x86_64 wheels build in the `manylinux` (podman-in-container) runner. - **vLLM/SGLang NIXL sanity (aarch64):** the same job also gates a 1-prefill/1-decode NIXL KV-transfer sanity for vLLM and SGLang. Each framework runs as its own aarch64 branch (`build_helper_vllm` / `build_helper_sglang`): build the aarch64 wheels (reusing the cached `wheel_base`), layer them onto the pinned framework base image (`.ci/dockerfiles/Dockerfile.{vllm,sglang}-base`, built as `category: tool` by ci-demo), then run the sanity on the `gb200nvl72_ci` dlcluster SLURM partition over SSH (`.gitlab/test_vllm_sglang_sanity.sh`). SGLang additionally asserts a gsm8k accuracy floor on `Qwen/Qwen3-8B`. The aarch64 wheels are built once per framework branch (duplicated) so the two flows stay separate, labeled branches in the Jenkins UI. -- **`CI_IMAGE_TAG`:** Same convention as the other five matrix files (also tags `build_helper_*` and the sanity `*-nixl-base` images). `contrib/Dockerfile.manylinux` is part of the `CI_FILES` list in `cidemo-init.sh`, so changing it (or any other CI file) without bumping `CI_IMAGE_TAG` in all six matrix YAMLs will fail the pre-commit check. +- **`CI_IMAGE_TAG`:** Same convention as the other five matrix files (also tags `build_helper_*` and the sanity `*-nixl-base` images). `contrib/Dockerfile.manylinux` is part of the `CI_FILES` list in `cidemo-init.sh`, so changing it (or any other CI file) automatically derives a new `CI_IMAGE_TAG` and rebuilds the cached `wheel_base` image (see [CI_IMAGE_TAG management](#ci_image_tag-management)). ### `nixl-ci-build-container` (standalone) - **Trigger:** Nightly cron (builds `nixlbench` and `nixl` targets, on both the default CUDA base image and the DLFW PyTorch daily image, ~3–4 AM), or manual run with parameters (`BUILD_TARGET`, `NIXL_VERSION`, `UCX_VERSION`, base image overrides, etc.). @@ -204,6 +204,35 @@ their own nightly/manual trigger. They split into two groups: - **Re-run a single Jenkins job with different parameters:** use `workflow_dispatch` on `blossom-ci.yml`, or trigger the Jenkins job directly if you have Jenkins access. - **Container/wheel builds outside the nightly schedule:** run `nixl-ci-build-container` or `nixl-ci-build-wheel-nightly` manually from the Jenkins UI with custom parameters. +## CI_IMAGE_TAG management + +`CI_IMAGE_TAG` is the Docker image tag used by all matrix jobs to identify the +base images they build and pull. It appears as `CI_IMAGE_TAG: "CI_MANAGED"` in +the six matrix YAML files — this placeholder is intentional and must not be +replaced with a static value. + +At the start of every Jenkins run, `cidemo-init.sh` derives the real tag +automatically: + +```bash +NEW_TAG=$(git log -1 --format=%h -- "${CI_FILES[@]}") +``` + +This returns the short git commit hash of the most recent commit that touched +any of the CI source files (`Dockerfile.base`, `Dockerfile.gpu-test`, +`Dockerfile.build_helper`, `build.sh`, `common.sh`, `Dockerfile.manylinux`). It +then patches all six YAML files in the Jenkins workspace with `sed` before the +matrix library reads them. No commit or push is made — the patch exists only in +the workspace. + +**Caching behaviour:** the derived tag is stable as long as the CI source files +are unchanged. Two PRs that both leave the CI files untouched get the same tag +and share the cached Artifactory images. A PR that changes a Dockerfile gets a +new tag and triggers a rebuild automatically. + +**You never need to manually update `CI_IMAGE_TAG`.** The `CI_MANAGED` +placeholder signals this clearly. + ## Related docs - [Build Wheel Matrix CI Job Documentation](build-wheel-matrix-ci.md) — deep dive into `nixl-ci-build-wheel`. diff --git a/.ci/jenkins/lib/build-matrix.yaml b/.ci/jenkins/lib/build-matrix.yaml index e0bfc2cf17..4e3858225e 100644 --- a/.ci/jenkins/lib/build-matrix.yaml +++ b/.ci/jenkins/lib/build-matrix.yaml @@ -45,7 +45,9 @@ env: TEST_TIMEOUT: 30 UCX_TLS: "^shm" STORAGE_DRIVER: 'overlay' - CI_IMAGE_TAG: "20260712-2" + # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files; + # the "CI_MANAGED" placeholder must not be hand-edited. + CI_IMAGE_TAG: "CI_MANAGED" runs_on_dockers: - { diff --git a/.ci/jenkins/lib/build-wheel-matrix.yaml b/.ci/jenkins/lib/build-wheel-matrix.yaml index e7cc761ec5..9ea96edf61 100644 --- a/.ci/jenkins/lib/build-wheel-matrix.yaml +++ b/.ci/jenkins/lib/build-wheel-matrix.yaml @@ -184,8 +184,9 @@ env: GRPC_NPROC: 10 BASE_IMAGE: "${registry_host}/sw-nbu-swx-nixl-docker-local/base/cuda" BASE_TAG: '13.0-devel-manylinux--25.09' - # CI image tag for wheel_base + build_helper + the sanity base images (bump to rebuild them) - CI_IMAGE_TAG: "20260709-1" + # CI image tag for wheel_base + build_helper + the sanity base images + # (auto-derived by cidemo-init.sh — the "CI_MANAGED" placeholder must not be hand-edited) + CI_IMAGE_TAG: "CI_MANAGED" WHL_BASE: "manylinux_2_28" TORCH_VERSIONS: "2.13" WHEEL_BASE_IMAGE_NAME: "nixl-wheel-base-manylinux_2_28" diff --git a/.ci/jenkins/lib/test-dl-ep-matrix.yaml b/.ci/jenkins/lib/test-dl-ep-matrix.yaml index 25a7397c22..6454f45c3f 100644 --- a/.ci/jenkins/lib/test-dl-ep-matrix.yaml +++ b/.ci/jenkins/lib/test-dl-ep-matrix.yaml @@ -60,7 +60,9 @@ env: JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}" TEST_TIMEOUT: 50 STORAGE_DRIVER: overlay - CI_IMAGE_TAG: "20260712-2" + # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files; + # the "CI_MANAGED" placeholder must not be hand-edited. + CI_IMAGE_TAG: "CI_MANAGED" empty_volumes: - {mountPath: /var/lib/containers/storage, memory: false} diff --git a/.ci/jenkins/lib/test-dl-matrix.yaml b/.ci/jenkins/lib/test-dl-matrix.yaml index 7b25f2bc67..444880c230 100644 --- a/.ci/jenkins/lib/test-dl-matrix.yaml +++ b/.ci/jenkins/lib/test-dl-matrix.yaml @@ -54,7 +54,9 @@ env: JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}" TEST_TIMEOUT: 50 STORAGE_DRIVER: overlay - CI_IMAGE_TAG: "20260712-2" + # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files; + # the "CI_MANAGED" placeholder must not be hand-edited. + CI_IMAGE_TAG: "CI_MANAGED" empty_volumes: - {mountPath: /var/lib/containers/storage, memory: false} diff --git a/.ci/jenkins/lib/test-matrix.yaml b/.ci/jenkins/lib/test-matrix.yaml index f07769f48f..60ad768fbb 100644 --- a/.ci/jenkins/lib/test-matrix.yaml +++ b/.ci/jenkins/lib/test-matrix.yaml @@ -54,7 +54,9 @@ env: SCCTL_CREDENTIALS_ID: 'svc-nixl-scctl' JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}" STORAGE_DRIVER: overlay - CI_IMAGE_TAG: "20260712-2" + # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files; + # the "CI_MANAGED" placeholder must not be hand-edited. + CI_IMAGE_TAG: "CI_MANAGED" ENROOT_MOUNT_PATH: "/enroot_images" # NFS mount on mizu nodes; on failure the container is exported here as .sqsh ENROOT_DATA_PATH: "/enroot-data/enroot-data-148069/user-148069" # svc-nixl user on mizu nodes diff --git a/.ci/jenkins/lib/test-sanitizer-matrix.yaml b/.ci/jenkins/lib/test-sanitizer-matrix.yaml index 29f082579c..7acbad630f 100644 --- a/.ci/jenkins/lib/test-sanitizer-matrix.yaml +++ b/.ci/jenkins/lib/test-sanitizer-matrix.yaml @@ -15,8 +15,8 @@ # prebuilt, non-instrumented abseil, TSan misses the lock and every # nixlLock-protected access shows up as a false race. # -# Note: changes to .ci/dockerfiles/Dockerfile.base, .gitlab/build.sh or -# .ci/scripts/common.sh require bumping CI_IMAGE_TAG here too (see cidemo-init.sh). +# Note: CI_IMAGE_TAG is auto-derived by cidemo-init.sh from the CI source files +# and patched in at CI time — the "CI_MANAGED" placeholder must not be hand-edited. --- job: nixl-ci-test-sanitizers @@ -42,7 +42,7 @@ env: TEST_TIMEOUT: 40 UCX_TLS: "^shm" STORAGE_DRIVER: 'overlay' - CI_IMAGE_TAG: "20260712-2" + CI_IMAGE_TAG: "CI_MANAGED" runs_on_dockers: # TSan-instrumented dependency image for the tsan build (DEPS_SANITIZE=thread