diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9c6c8556..5b1b9e7e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,6 +18,7 @@ jobs: - devcontainer - detect-changes - test_benchmark_data_tools + - test_sccache_setup permissions: contents: read uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main # zizmor: ignore[unpinned-uses] @@ -65,6 +66,7 @@ jobs: runs-on: ubuntu-latest outputs: benchmark_data_tools: ${{ steps.filter.outputs.benchmark_data_tools }} + sccache: ${{ steps.filter.outputs.sccache }} steps: - name: Checkout velox-testing uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -78,6 +80,11 @@ jobs: filters: | benchmark_data_tools: - "benchmark_data_tools/**" + sccache: + - "scripts/sccache/sccache_setup.sh" + - "scripts/sccache/sccache_setup_test.sh" + - "presto/docker/native_build.dockerfile" + - "velox/docker/adapters_build.dockerfile" test_benchmark_data_tools: needs: detect-changes @@ -93,3 +100,18 @@ jobs: - name: Test benchmark_data_tools uses: ./.github/actions/test-benchmark-data-tools + + test_sccache_setup: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.sccache == 'true' }} + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Checkout velox-testing + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Test sccache setup + run: bash scripts/sccache/sccache_setup_test.sh diff --git a/presto/docker/native_build.dockerfile b/presto/docker/native_build.dockerfile index a64f68a2..db237691 100644 --- a/presto/docker/native_build.dockerfile +++ b/presto/docker/native_build.dockerfile @@ -89,7 +89,8 @@ if [ "$ENABLE_SCCACHE" = "ON" ]; then if [ -n "${SCCACHE_NO_DIST_COMPILE:-}" ]; then export SCCACHE_NO_DIST_COMPILE=1; fi - bash /sccache_setup.sh; + # Source the setup so its automatic local-fallback export reaches the build. + source /sccache_setup.sh; EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache"; export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=100"; fi diff --git a/scripts/sccache/sccache_setup.sh b/scripts/sccache/sccache_setup.sh index b73e3210..f8f35dfe 100755 --- a/scripts/sccache/sccache_setup.sh +++ b/scripts/sccache/sccache_setup.sh @@ -18,7 +18,7 @@ chmod +x /usr/bin/sccache sccache --version # Increase file descriptor limit for high parallelism (as done in rapids-configure-sccache-dist) -ulimit -n $(ulimit -Hn) || echo "Could not increase file descriptor limit" +ulimit -n "$(ulimit -Hn)" || echo "Could not increase file descriptor limit" # Ensure sccache logfile directory exists mkdir -p "$(dirname "${SCCACHE_ERROR_LOG:-/tmp/sccache.log}")" @@ -45,25 +45,38 @@ if ! test -v SCCACHE_NO_DIST_COMPILE; then echo "=== end diagnostics ===" fi -sccache --zero-stats - if test -v SCCACHE_NO_DIST_COMPILE; then echo "Distributed compilation is DISABLED - using local compilation with remote S3 caching" else - if sccache --dist-status 2>/dev/null | jq -er '.SchedulerStatus? != null' >/dev/null 2>&1; then + dist_status="" + server_count="" + if dist_status="$(sccache --dist-status 2>/dev/null)" \ + && server_count="$(jq -er '(.SchedulerStatus?[1].servers? // []) | length' <<<"${dist_status}")" \ + && (( server_count > 0 )); then echo "Distributed compilation is available:" - sccache --dist-status | jq -r '["scheduler URL: " + .SchedulerStatus[0], "server count: " + (.SchedulerStatus[1].servers | length | tostring)][]'; + jq -r '["scheduler URL: " + .SchedulerStatus[0], "server count: " + (.SchedulerStatus[1].servers | length | tostring)][]' <<<"${dist_status}" else - echo "Error: Distributed compilation not available, check connectivity" + if [[ "${server_count}" == "0" ]]; then + echo "Error: Distributed compilation scheduler has no available servers" + else + echo "Error: Distributed compilation status unavailable, check connectivity" + fi + if [[ -n "${server_count}" ]]; then + echo "server count: ${server_count}" + fi if [[ -f "${SCCACHE_ERROR_LOG:-}" ]]; then echo "sccache error log:" cat "$SCCACHE_ERROR_LOG"; fi if [[ "${SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE:-false}" == "true" ]]; then echo "SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=true, continuing with local compilation" + # The Docker build callers source this script so this setting is + # inherited by every compiler request in the subsequent build. export SCCACHE_NO_DIST_COMPILE=1 else exit 1 fi fi fi + +sccache --zero-stats diff --git a/scripts/sccache/sccache_setup_test.sh b/scripts/sccache/sccache_setup_test.sh new file mode 100644 index 00000000..22bd623f --- /dev/null +++ b/scripts/sccache/sccache_setup_test.sh @@ -0,0 +1,139 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPOSITORY_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +SETUP_SCRIPT="${1:-${SCRIPT_DIR}/sccache_setup.sh}" +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "${TEST_ROOT}"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local file="$1" + local expected="$2" + grep -Fq -- "${expected}" "${file}" \ + || fail "Expected ${file} to contain: ${expected}" +} + +run_setup_case() { + local name="$1" + local dist_status="$2" + local dist_status_exit_code="$3" + local fallback_enabled="$4" + local initially_disabled="$5" + local expected_exit_code="$6" + local expected_no_dist="$7" + local expected_status_calls="$8" + + local case_dir="${TEST_ROOT}/${name}" + local output="${case_dir}/output" + local trace="${case_dir}/trace" + mkdir -p "${case_dir}" + + local actual_exit_code=0 + ( + export SCCACHE_VERSION=test + export SCCACHE_ERROR_LOG="${case_dir}/sccache.log" + export SCCACHE_DIST_AUTH_TOKEN=test-token + export SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE="${fallback_enabled}" + export TEST_DIST_STATUS="${dist_status}" + export TEST_DIST_STATUS_EXIT_CODE="${dist_status_exit_code}" + export TEST_SCCACHE_TRACE="${trace}" + + if [[ "${initially_disabled}" == "true" ]]; then + export SCCACHE_NO_DIST_COMPILE=1 + else + unset SCCACHE_NO_DIST_COMPILE + fi + + # These mocks are called indirectly by the sourced setup script. + # shellcheck disable=SC2317 + wget() { + return 0 + } + + # shellcheck disable=SC2317 + tar() { + return 0 + } + + # shellcheck disable=SC2317 + chmod() { + return 0 + } + + # shellcheck disable=SC2317 + sccache() { + printf '%s|no_dist=%s\n' "$*" "${SCCACHE_NO_DIST_COMPILE:-}" \ + >>"${TEST_SCCACHE_TRACE}" + case "${1:-}" in + --version) + echo "sccache test" + ;; + --stop-server | --zero-stats) + ;; + --dist-status) + if (( TEST_DIST_STATUS_EXIT_CODE != 0 )); then + return "${TEST_DIST_STATUS_EXIT_CODE}" + fi + printf '%s\n' "${TEST_DIST_STATUS}" + ;; + *) + fail "Unexpected sccache arguments: $*" + ;; + esac + } + + # shellcheck disable=SC1090 + source "${SETUP_SCRIPT}" + printf 'final_no_dist=%s\n' "${SCCACHE_NO_DIST_COMPILE:-unset}" + ) >"${output}" 2>&1 || actual_exit_code=$? + + [[ "${actual_exit_code}" == "${expected_exit_code}" ]] \ + || fail "${name}: expected exit ${expected_exit_code}, got ${actual_exit_code}" + + local actual_status_calls + actual_status_calls="$(grep -c '^--dist-status|' "${trace}" || true)" + [[ "${actual_status_calls}" == "${expected_status_calls}" ]] \ + || fail "${name}: expected ${expected_status_calls} status calls, got ${actual_status_calls}" + + if [[ "${expected_exit_code}" == "0" ]]; then + assert_contains "${output}" "final_no_dist=${expected_no_dist}" + fi +} + +healthy_status='{"SchedulerStatus":["https://arm64.example",{"servers":[{"id":"worker-1"}]}]}' +empty_status='{"SchedulerStatus":["https://arm64.example",{"servers":[]}]}' + +run_setup_case healthy "${healthy_status}" 0 true false 0 unset 1 +assert_contains "${TEST_ROOT}/healthy/output" "server count: 1" + +run_setup_case zero-workers "${empty_status}" 0 true false 0 1 1 +assert_contains "${TEST_ROOT}/zero-workers/output" "scheduler has no available servers" +assert_contains "${TEST_ROOT}/zero-workers/output" "server count: 0" +assert_contains "${TEST_ROOT}/zero-workers/trace" "--zero-stats|no_dist=1" + +run_setup_case status-error '{}' 1 true false 0 1 1 +run_setup_case missing-status '{}' 0 true false 0 1 1 +run_setup_case invalid-status 'not-json' 0 true false 0 1 1 +run_setup_case fallback-disabled "${empty_status}" 0 false false 1 unset 1 +run_setup_case explicitly-disabled "${empty_status}" 0 true true 0 1 0 + +for dockerfile in \ + "${REPOSITORY_ROOT}/presto/docker/native_build.dockerfile" \ + "${REPOSITORY_ROOT}/velox/docker/adapters_build.dockerfile" +do + assert_contains "${dockerfile}" "source /sccache_setup.sh;" + if grep -Fq "bash /sccache_setup.sh;" "${dockerfile}"; then + fail "${dockerfile} runs sccache setup in a child shell" + fi +done + +echo "All sccache setup tests passed" diff --git a/velox/docker/adapters_build.dockerfile b/velox/docker/adapters_build.dockerfile index 635bdc52..3b480fd7 100644 --- a/velox/docker/adapters_build.dockerfile +++ b/velox/docker/adapters_build.dockerfile @@ -163,7 +163,8 @@ if [ "$ENABLE_SCCACHE" = "ON" ]; then if [ -n "${SCCACHE_NO_DIST_COMPILE:-}" ]; then export SCCACHE_NO_DIST_COMPILE=1; fi - bash /sccache_setup.sh; + # Source the setup so its automatic local-fallback export reaches the build. + source /sccache_setup.sh; EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache"; export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=100"; fi