From 341869d6eda8246139383213f95d3ab17a1f44a8 Mon Sep 17 00:00:00 2001 From: Kyle Hubert Date: Sun, 26 Jul 2026 22:44:24 +0100 Subject: [PATCH] Fall back to local compilation when sccache has no workers The distributed scheduler can remain reachable while reporting zero compilation servers. The setup check treated any SchedulerStatus as available, so cache misses waited on 7,140-second requests and retries until GitHub Actions canceled the ARM jobs at six hours. The existing local fallback was also exported from a child shell, so it could not reach compiler requests in the subsequent Docker build. Require a nonempty server list, reuse one status snapshot, and source the setup script from both Presto and Velox image builds. This preserves S3 caching while bypassing distributed compilation whenever the scheduler is unavailable. Add regression coverage for healthy, empty, missing, malformed, failed, explicitly disabled, and fallback-disabled scheduler states, and run it from PR checks. --- .github/workflows/pr.yaml | 22 ++++ presto/docker/native_build.dockerfile | 3 +- scripts/sccache/sccache_setup.sh | 25 +++-- scripts/sccache/sccache_setup_test.sh | 139 +++++++++++++++++++++++++ velox/docker/adapters_build.dockerfile | 3 +- 5 files changed, 184 insertions(+), 8 deletions(-) create mode 100644 scripts/sccache/sccache_setup_test.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9c6c8556..5b1b9e7e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,6 +18,7 @@ jobs: - devcontainer - detect-changes - test_benchmark_data_tools + - test_sccache_setup permissions: contents: read uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main # zizmor: ignore[unpinned-uses] @@ -65,6 +66,7 @@ jobs: runs-on: ubuntu-latest outputs: benchmark_data_tools: ${{ steps.filter.outputs.benchmark_data_tools }} + sccache: ${{ steps.filter.outputs.sccache }} steps: - name: Checkout velox-testing uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -78,6 +80,11 @@ jobs: filters: | benchmark_data_tools: - "benchmark_data_tools/**" + sccache: + - "scripts/sccache/sccache_setup.sh" + - "scripts/sccache/sccache_setup_test.sh" + - "presto/docker/native_build.dockerfile" + - "velox/docker/adapters_build.dockerfile" test_benchmark_data_tools: needs: detect-changes @@ -93,3 +100,18 @@ jobs: - name: Test benchmark_data_tools uses: ./.github/actions/test-benchmark-data-tools + + test_sccache_setup: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.sccache == 'true' }} + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Checkout velox-testing + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Test sccache setup + run: bash scripts/sccache/sccache_setup_test.sh diff --git a/presto/docker/native_build.dockerfile b/presto/docker/native_build.dockerfile index a64f68a2..db237691 100644 --- a/presto/docker/native_build.dockerfile +++ b/presto/docker/native_build.dockerfile @@ -89,7 +89,8 @@ if [ "$ENABLE_SCCACHE" = "ON" ]; then if [ -n "${SCCACHE_NO_DIST_COMPILE:-}" ]; then export SCCACHE_NO_DIST_COMPILE=1; fi - bash /sccache_setup.sh; + # Source the setup so its automatic local-fallback export reaches the build. + source /sccache_setup.sh; EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache"; export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=100"; fi diff --git a/scripts/sccache/sccache_setup.sh b/scripts/sccache/sccache_setup.sh index b73e3210..f8f35dfe 100755 --- a/scripts/sccache/sccache_setup.sh +++ b/scripts/sccache/sccache_setup.sh @@ -18,7 +18,7 @@ chmod +x /usr/bin/sccache sccache --version # Increase file descriptor limit for high parallelism (as done in rapids-configure-sccache-dist) -ulimit -n $(ulimit -Hn) || echo "Could not increase file descriptor limit" +ulimit -n "$(ulimit -Hn)" || echo "Could not increase file descriptor limit" # Ensure sccache logfile directory exists mkdir -p "$(dirname "${SCCACHE_ERROR_LOG:-/tmp/sccache.log}")" @@ -45,25 +45,38 @@ if ! test -v SCCACHE_NO_DIST_COMPILE; then echo "=== end diagnostics ===" fi -sccache --zero-stats - if test -v SCCACHE_NO_DIST_COMPILE; then echo "Distributed compilation is DISABLED - using local compilation with remote S3 caching" else - if sccache --dist-status 2>/dev/null | jq -er '.SchedulerStatus? != null' >/dev/null 2>&1; then + dist_status="" + server_count="" + if dist_status="$(sccache --dist-status 2>/dev/null)" \ + && server_count="$(jq -er '(.SchedulerStatus?[1].servers? // []) | length' <<<"${dist_status}")" \ + && (( server_count > 0 )); then echo "Distributed compilation is available:" - sccache --dist-status | jq -r '["scheduler URL: " + .SchedulerStatus[0], "server count: " + (.SchedulerStatus[1].servers | length | tostring)][]'; + jq -r '["scheduler URL: " + .SchedulerStatus[0], "server count: " + (.SchedulerStatus[1].servers | length | tostring)][]' <<<"${dist_status}" else - echo "Error: Distributed compilation not available, check connectivity" + if [[ "${server_count}" == "0" ]]; then + echo "Error: Distributed compilation scheduler has no available servers" + else + echo "Error: Distributed compilation status unavailable, check connectivity" + fi + if [[ -n "${server_count}" ]]; then + echo "server count: ${server_count}" + fi if [[ -f "${SCCACHE_ERROR_LOG:-}" ]]; then echo "sccache error log:" cat "$SCCACHE_ERROR_LOG"; fi if [[ "${SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE:-false}" == "true" ]]; then echo "SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=true, continuing with local compilation" + # The Docker build callers source this script so this setting is + # inherited by every compiler request in the subsequent build. export SCCACHE_NO_DIST_COMPILE=1 else exit 1 fi fi fi + +sccache --zero-stats diff --git a/scripts/sccache/sccache_setup_test.sh b/scripts/sccache/sccache_setup_test.sh new file mode 100644 index 00000000..22bd623f --- /dev/null +++ b/scripts/sccache/sccache_setup_test.sh @@ -0,0 +1,139 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPOSITORY_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +SETUP_SCRIPT="${1:-${SCRIPT_DIR}/sccache_setup.sh}" +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "${TEST_ROOT}"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local file="$1" + local expected="$2" + grep -Fq -- "${expected}" "${file}" \ + || fail "Expected ${file} to contain: ${expected}" +} + +run_setup_case() { + local name="$1" + local dist_status="$2" + local dist_status_exit_code="$3" + local fallback_enabled="$4" + local initially_disabled="$5" + local expected_exit_code="$6" + local expected_no_dist="$7" + local expected_status_calls="$8" + + local case_dir="${TEST_ROOT}/${name}" + local output="${case_dir}/output" + local trace="${case_dir}/trace" + mkdir -p "${case_dir}" + + local actual_exit_code=0 + ( + export SCCACHE_VERSION=test + export SCCACHE_ERROR_LOG="${case_dir}/sccache.log" + export SCCACHE_DIST_AUTH_TOKEN=test-token + export SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE="${fallback_enabled}" + export TEST_DIST_STATUS="${dist_status}" + export TEST_DIST_STATUS_EXIT_CODE="${dist_status_exit_code}" + export TEST_SCCACHE_TRACE="${trace}" + + if [[ "${initially_disabled}" == "true" ]]; then + export SCCACHE_NO_DIST_COMPILE=1 + else + unset SCCACHE_NO_DIST_COMPILE + fi + + # These mocks are called indirectly by the sourced setup script. + # shellcheck disable=SC2317 + wget() { + return 0 + } + + # shellcheck disable=SC2317 + tar() { + return 0 + } + + # shellcheck disable=SC2317 + chmod() { + return 0 + } + + # shellcheck disable=SC2317 + sccache() { + printf '%s|no_dist=%s\n' "$*" "${SCCACHE_NO_DIST_COMPILE:-}" \ + >>"${TEST_SCCACHE_TRACE}" + case "${1:-}" in + --version) + echo "sccache test" + ;; + --stop-server | --zero-stats) + ;; + --dist-status) + if (( TEST_DIST_STATUS_EXIT_CODE != 0 )); then + return "${TEST_DIST_STATUS_EXIT_CODE}" + fi + printf '%s\n' "${TEST_DIST_STATUS}" + ;; + *) + fail "Unexpected sccache arguments: $*" + ;; + esac + } + + # shellcheck disable=SC1090 + source "${SETUP_SCRIPT}" + printf 'final_no_dist=%s\n' "${SCCACHE_NO_DIST_COMPILE:-unset}" + ) >"${output}" 2>&1 || actual_exit_code=$? + + [[ "${actual_exit_code}" == "${expected_exit_code}" ]] \ + || fail "${name}: expected exit ${expected_exit_code}, got ${actual_exit_code}" + + local actual_status_calls + actual_status_calls="$(grep -c '^--dist-status|' "${trace}" || true)" + [[ "${actual_status_calls}" == "${expected_status_calls}" ]] \ + || fail "${name}: expected ${expected_status_calls} status calls, got ${actual_status_calls}" + + if [[ "${expected_exit_code}" == "0" ]]; then + assert_contains "${output}" "final_no_dist=${expected_no_dist}" + fi +} + +healthy_status='{"SchedulerStatus":["https://arm64.example",{"servers":[{"id":"worker-1"}]}]}' +empty_status='{"SchedulerStatus":["https://arm64.example",{"servers":[]}]}' + +run_setup_case healthy "${healthy_status}" 0 true false 0 unset 1 +assert_contains "${TEST_ROOT}/healthy/output" "server count: 1" + +run_setup_case zero-workers "${empty_status}" 0 true false 0 1 1 +assert_contains "${TEST_ROOT}/zero-workers/output" "scheduler has no available servers" +assert_contains "${TEST_ROOT}/zero-workers/output" "server count: 0" +assert_contains "${TEST_ROOT}/zero-workers/trace" "--zero-stats|no_dist=1" + +run_setup_case status-error '{}' 1 true false 0 1 1 +run_setup_case missing-status '{}' 0 true false 0 1 1 +run_setup_case invalid-status 'not-json' 0 true false 0 1 1 +run_setup_case fallback-disabled "${empty_status}" 0 false false 1 unset 1 +run_setup_case explicitly-disabled "${empty_status}" 0 true true 0 1 0 + +for dockerfile in \ + "${REPOSITORY_ROOT}/presto/docker/native_build.dockerfile" \ + "${REPOSITORY_ROOT}/velox/docker/adapters_build.dockerfile" +do + assert_contains "${dockerfile}" "source /sccache_setup.sh;" + if grep -Fq "bash /sccache_setup.sh;" "${dockerfile}"; then + fail "${dockerfile} runs sccache setup in a child shell" + fi +done + +echo "All sccache setup tests passed" diff --git a/velox/docker/adapters_build.dockerfile b/velox/docker/adapters_build.dockerfile index 635bdc52..3b480fd7 100644 --- a/velox/docker/adapters_build.dockerfile +++ b/velox/docker/adapters_build.dockerfile @@ -163,7 +163,8 @@ if [ "$ENABLE_SCCACHE" = "ON" ]; then if [ -n "${SCCACHE_NO_DIST_COMPILE:-}" ]; then export SCCACHE_NO_DIST_COMPILE=1; fi - bash /sccache_setup.sh; + # Source the setup so its automatic local-fallback export reaches the build. + source /sccache_setup.sh; EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache"; export NVCC_APPEND_FLAGS="${NVCC_APPEND_FLAGS:+$NVCC_APPEND_FLAGS }-t=100"; fi