From 81accab65fd0df0ed37de2683e65ccdf84321c0f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 17 Jul 2025 10:34:09 -0500 Subject: [PATCH 1/5] test pinning metapackage generated by conda-lock --- ci/build_cpp.sh | 1 + ci/build_python.sh | 1 + conda/recipes/cuml/recipe.yaml | 1 + conda/recipes/libcuml/recipe.yaml | 3 +++ 4 files changed, 6 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index cbba7c7845..f204840e4a 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,6 +20,7 @@ export RAPIDS_PACKAGE_VERSION # populates `RATTLER_CHANNELS` array and `RATTLER_ARGS` array source rapids-rattler-channel-string +RATTLER_CHANNELS+=("-c" "msarahan") # --no-build-id allows for caching with `sccache` # more info is available at diff --git a/ci/build_python.sh b/ci/build_python.sh index ade594dcab..8b2e901e8e 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -26,6 +26,7 @@ source rapids-rattler-channel-string rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS" RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}") +RATTLER_CHANNELS+=("-c" "msarahan") sccache --zero-stats diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index 178239675b..a915a8c223 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -71,6 +71,7 @@ requirements: - treelite ${{ treelite_version }} - cuda-cudart-dev - cuda-python >=12.6.2,<13.0a0 + - rapids-pin run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - cudf =${{ minor_version }} diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 5b9d137188..d0b7ca0c38 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -68,6 +68,7 @@ cache: - libcurand-dev - libcusolver-dev - libcusparse-dev + - rapids-pin outputs: - package: @@ -102,6 +103,7 @@ outputs: - libcurand - libcusolver - libcusparse + - rapids-pin run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - cuda-cudart @@ -148,6 +150,7 @@ outputs: - rapids-logger =0.1 - ${{ pin_subpackage("libcuml", exact=True) }} - cuda-cudart-dev + - rapids-pin run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - ${{ pin_subpackage("libcuml", exact=True) }} From f742c7b0bfa8db717d1adbb88bbc678a9ecfabe4 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 18 Jul 2025 15:08:11 -0500 Subject: [PATCH 2/5] add rapids-pin to testing env with dependencies.yaml --- conda/environments/all_cuda-129_arch-aarch64.yaml | 2 ++ conda/environments/all_cuda-129_arch-x86_64.yaml | 2 ++ conda/environments/clang_tidy_cuda-129_arch-x86_64.yaml | 1 + conda/environments/cpp_all_cuda-129_arch-x86_64.yaml | 1 + dependencies.yaml | 8 ++++++++ 5 files changed, 14 insertions(+) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 3a81f1f067..4f507abe8f 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -4,6 +4,7 @@ channels: - rapidsai - rapidsai-nightly - conda-forge +- msarahan dependencies: - c-compiler - certifi @@ -61,6 +62,7 @@ dependencies: - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==25.10.*,>=0.0.0a0 - rapids-logger==0.1.*,>=0.0.0a0 +- rapids-pin - recommonmark - rich - rmm==25.10.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index ed9e1ff2d4..cc3fbfccab 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -4,6 +4,7 @@ channels: - rapidsai - rapidsai-nightly - conda-forge +- msarahan dependencies: - c-compiler - certifi @@ -61,6 +62,7 @@ dependencies: - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==25.10.*,>=0.0.0a0 - rapids-logger==0.1.*,>=0.0.0a0 +- rapids-pin - recommonmark - rich - rmm==25.10.*,>=0.0.0a0 diff --git a/conda/environments/clang_tidy_cuda-129_arch-x86_64.yaml b/conda/environments/clang_tidy_cuda-129_arch-x86_64.yaml index 1d1071d1d3..33746ec1d2 100644 --- a/conda/environments/clang_tidy_cuda-129_arch-x86_64.yaml +++ b/conda/environments/clang_tidy_cuda-129_arch-x86_64.yaml @@ -4,6 +4,7 @@ channels: - rapidsai - rapidsai-nightly - conda-forge +- msarahan dependencies: - c-compiler - clang-tools==15.0.7 diff --git a/conda/environments/cpp_all_cuda-129_arch-x86_64.yaml b/conda/environments/cpp_all_cuda-129_arch-x86_64.yaml index e09c6b761a..09a20c3f46 100644 --- a/conda/environments/cpp_all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/cpp_all_cuda-129_arch-x86_64.yaml @@ -4,6 +4,7 @@ channels: - rapidsai - rapidsai-nightly - conda-forge +- msarahan dependencies: - c-compiler - cmake>=3.30.4 diff --git a/dependencies.yaml b/dependencies.yaml index 51d75f668d..648ac2b0cc 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -29,6 +29,7 @@ files: - rapids_build_backend - test_python - test_python_xgboost + - rapids_conda_pinning devcontainers: output: none includes: @@ -55,6 +56,7 @@ files: - py_version - rapids_build_backend - test_python + - rapids_conda_pinning cpp_all: output: conda matrix: @@ -208,6 +210,7 @@ channels: - rapidsai - rapidsai-nightly - conda-forge + - msarahan dependencies: rapids_build_backend: common: @@ -295,6 +298,11 @@ dependencies: # pip recognizes the index as a global option for the requirements.txt file - --extra-index-url=https://pypi.nvidia.com - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + rapids_conda_pinning: + common: + - output_types: [conda] + packages: + - rapids-pin cuda_version: specific: - output_types: conda From 98b27de0cfccf986e8dc8d2184c0cf0be66c0ac3 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 4 Aug 2025 16:10:07 -0500 Subject: [PATCH 3/5] use section-specific pin packages --- conda/recipes/cuml/recipe.yaml | 5 ++++- conda/recipes/libcuml/recipe.yaml | 12 +++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index a915a8c223..fc0ca1cdb9 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -10,6 +10,8 @@ context: py_version: ${{ env.get("RAPIDS_PY_VERSION") }} py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: '${{ git.head_rev(".")[:8] }}' + pin_version: ${{ minor_version }} + pin_build_wildcard: '*_0' package: name: cuml @@ -55,6 +57,7 @@ requirements: - ${{ compiler("cuda") }} - cuda-version =${{ cuda_version }} - ${{ stdlib("c") }} + - cuml-rapids-pins-build ${{ pin_version }} ${{ pin_build_wildcard }} host: - cuda-version =${{ cuda_version }} - cudf =${{ minor_version }} @@ -71,7 +74,7 @@ requirements: - treelite ${{ treelite_version }} - cuda-cudart-dev - cuda-python >=12.6.2,<13.0a0 - - rapids-pin + - cuml-rapids-pins-host ${{ pin_version }} ${{ pin_build_wildcard }} run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - cudf =${{ minor_version }} diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index d0b7ca0c38..1abace6533 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -8,6 +8,8 @@ context: cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' head_rev: '${{ git.head_rev(".")[:8] }}' + pin_version: {{ minor_version }} + pin_build_wildcard: '*_0' recipe: name: libcuml-split @@ -54,6 +56,7 @@ cache: - cmake ${{ cmake_version }} - ninja - ${{ stdlib("c") }} + - cuml-rapids-pins-build ${{ pin_version }} ${{ pin_build_wildcard }} host: - cuda-version =${{ cuda_version }} - libcumlprims =${{ minor_version }} @@ -68,7 +71,7 @@ cache: - libcurand-dev - libcusolver-dev - libcusparse-dev - - rapids-pin + - cuml-rapids-pins-host ${{ pin_version }} ${{ pin_build_wildcard }} outputs: - package: @@ -90,6 +93,7 @@ outputs: build: - cmake ${{ cmake_version }} - ${{ stdlib("c") }} + - cuml-rapids-pins-build ${{ pin_version }} ${{ pin_build_wildcard }} host: - cuda-version =${{ cuda_version }} - rapids-logger =0.1 @@ -103,7 +107,7 @@ outputs: - libcurand - libcusolver - libcusparse - - rapids-pin + - cuml-rapids-pins-host ${{ pin_version }} ${{ pin_build_wildcard }} run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - cuda-cudart @@ -117,6 +121,7 @@ outputs: - librmm =${{ minor_version }} - rapids-logger =0.1 - treelite ${{ treelite_version }} + ignore_run_exports: by_name: - cuda-cudart @@ -145,12 +150,13 @@ outputs: build: - cmake ${{ cmake_version }} - ${{ stdlib("c") }} + - cuml-rapids-pins-build ${{ pin_version }} ${{ pin_build_wildcard }} host: - cuda-version =${{ cuda_version }} - rapids-logger =0.1 - ${{ pin_subpackage("libcuml", exact=True) }} - cuda-cudart-dev - - rapids-pin + - cuml-rapids-pins-host ${{ pin_version }} ${{ pin_build_wildcard }} run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - ${{ pin_subpackage("libcuml", exact=True) }} From 8e9b1381571f57d7b708430347bb0ed6d198b40a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 4 Aug 2025 16:22:42 -0500 Subject: [PATCH 4/5] comment devcontainer and wheel builds for efficiency; fix variable usage --- .github/workflows/pr.yaml | 118 +++++++++++++++--------------- conda/recipes/cuml/recipe.yaml | 2 +- conda/recipes/libcuml/recipe.yaml | 2 +- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 24f9dec1f9..6397c6ae69 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,11 +25,11 @@ jobs: - conda-notebook-tests - docs-build - telemetry-setup - - wheel-build-libcuml - - wheel-build-cuml - - wheel-tests-cuml - - wheel-tests-cuml-dask - - devcontainer + # - wheel-build-libcuml + # - wheel-build-cuml + # - wheel-tests-cuml + # - wheel-tests-cuml-dask + # - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.10 if: always() @@ -215,60 +215,60 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:25.10-latest" script: "ci/build_docs.sh" - wheel-build-libcuml: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.10 - with: - build_type: pull-request - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - script: ci/build_wheel_libcuml.sh - extra-repo: rapidsai/cumlprims_mg - extra-repo-sha: branch-25.10 - extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY - # build for every combination of arch and CUDA version, but only for the latest Python - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) - package-name: libcuml - package-type: cpp - wheel-build-cuml: - needs: [checks, wheel-build-libcuml] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.10 - with: - build_type: pull-request - script: ci/build_wheel_cuml.sh - package-name: cuml - package-type: python - wheel-tests-cuml: - needs: [wheel-build-cuml, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.10 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel.sh - wheel-tests-cuml-dask: - needs: [wheel-build-cuml, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.10 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_dask.sh - devcontainer: - needs: telemetry-setup - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.10 - with: - arch: '["amd64"]' - cuda: '["12.9"]' - extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY - build_command: | - sccache -z; - build-all --verbose -DBUILD_TESTS=ON -DBUILD_BENCHMARKS=ON; - sccache -s; + # wheel-build-libcuml: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.10 + # with: + # build_type: pull-request + # branch: ${{ inputs.branch }} + # sha: ${{ inputs.sha }} + # date: ${{ inputs.date }} + # script: ci/build_wheel_libcuml.sh + # extra-repo: rapidsai/cumlprims_mg + # extra-repo-sha: branch-25.10 + # extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + # # build for every combination of arch and CUDA version, but only for the latest Python + # matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + # package-name: libcuml + # package-type: cpp + # wheel-build-cuml: + # needs: [checks, wheel-build-libcuml] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.10 + # with: + # build_type: pull-request + # script: ci/build_wheel_cuml.sh + # package-name: cuml + # package-type: python + # wheel-tests-cuml: + # needs: [wheel-build-cuml, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.10 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # script: ci/test_wheel.sh + # wheel-tests-cuml-dask: + # needs: [wheel-build-cuml, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.10 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # script: ci/test_wheel_dask.sh + # devcontainer: + # needs: telemetry-setup + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.10 + # with: + # arch: '["amd64"]' + # cuda: '["12.9"]' + # extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + # build_command: | + # sccache -z; + # build-all --verbose -DBUILD_TESTS=ON -DBUILD_BENCHMARKS=ON; + # sccache -s; telemetry-summarize: # This job must use a self-hosted runner to record telemetry traces. runs-on: linux-amd64-cpu4 diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index fc0ca1cdb9..c6626fccb2 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -10,7 +10,7 @@ context: py_version: ${{ env.get("RAPIDS_PY_VERSION") }} py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: '${{ git.head_rev(".")[:8] }}' - pin_version: ${{ minor_version }} + pin_version: '${{ minor_version }}' pin_build_wildcard: '*_0' package: diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 1abace6533..350bbede5e 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -8,7 +8,7 @@ context: cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' head_rev: '${{ git.head_rev(".")[:8] }}' - pin_version: {{ minor_version }} + pin_version: '${{ minor_version }}' pin_build_wildcard: '*_0' recipe: From f32b2434de32bfb5b17f542577b77006ca4bc1bb Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 4 Aug 2025 16:31:17 -0500 Subject: [PATCH 5/5] disambiguated pin package version --- conda/recipes/cuml/recipe.yaml | 2 +- conda/recipes/libcuml/recipe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cuml/recipe.yaml b/conda/recipes/cuml/recipe.yaml index c6626fccb2..3f5e3bee01 100644 --- a/conda/recipes/cuml/recipe.yaml +++ b/conda/recipes/cuml/recipe.yaml @@ -10,7 +10,7 @@ context: py_version: ${{ env.get("RAPIDS_PY_VERSION") }} py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: '${{ git.head_rev(".")[:8] }}' - pin_version: '${{ minor_version }}' + pin_version: '${{ minor_version }}.*' pin_build_wildcard: '*_0' package: diff --git a/conda/recipes/libcuml/recipe.yaml b/conda/recipes/libcuml/recipe.yaml index 350bbede5e..c9a34061f6 100644 --- a/conda/recipes/libcuml/recipe.yaml +++ b/conda/recipes/libcuml/recipe.yaml @@ -8,7 +8,7 @@ context: cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' head_rev: '${{ git.head_rev(".")[:8] }}' - pin_version: '${{ minor_version }}' + pin_version: '${{ minor_version }}.*' pin_build_wildcard: '*_0' recipe: