diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index d06b0e94..e7a431b9 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -86,6 +86,7 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/stdlib_load_time_regression.yml # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml diff --git a/pipelines/main/misc/stdlib_load_time_regression.yml b/pipelines/main/misc/stdlib_load_time_regression.yml new file mode 100644 index 00000000..5f2785d4 --- /dev/null +++ b/pipelines/main/misc/stdlib_load_time_regression.yml @@ -0,0 +1,20 @@ +steps: + - group: "Check" + steps: + - label: ":macos: stdlib load-time regression (aarch64)" + key: "stdlib-load-time-regression" + depends_on: + - "build_aarch64-apple-darwin" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: "bash .buildkite/utilities/stdlib_load_time_regression.sh" + timeout_in_minutes: 120 + agents: + queue: "test" + os: "macos" + arch: "aarch64" + env: + TRIPLET: "aarch64-apple-darwin" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/utilities/stdlib_load_time_regression.sh b/utilities/stdlib_load_time_regression.sh new file mode 100644 index 00000000..4d831b20 --- /dev/null +++ b/utilities/stdlib_load_time_regression.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# This script runs a stdlib precompile/load-time regression check, comparing: +# +# A = the Julia built by this CI pipeline (the "candidate") +# B = the Julia nightly associated with the merge-base commit (the "baseline") +# +# It downloads/extracts both binaries (building the baseline from source if the +# merge-base nightly is missing), then hands them to +# `compare_stdlib_load_times.jl`, which re-precompiles every stdlib for both +# (timing the precompile) and measures their load times across several A/B/B/A/B/A +# rounds (to control for noise), then flags clear precompile- or load-time +# regressions. +# +# This check is only meaningful on macOS aarch64. + +set -euo pipefail + +# First, get things like `OS`, `ARCH`, `TRIPLET`, `JULIA_INSTALL_DIR`, +# `UPLOAD_FILENAME`, `JULIA_BINARY`, `JULIA_CPU_TARGET`, etc... +# shellcheck source=SCRIPTDIR/build_envs.sh +source .buildkite/utilities/build_envs.sh + +if [[ "${OS}" != "macos" ]] || [[ "${ARCH}" != "aarch64" ]]; then + echo "This check only runs on macOS aarch64 (got ${OS} ${ARCH}); skipping." + exit 0 +fi + +# Default the number of build threads if the agent did not provide it. +: "${JULIA_CPU_THREADS:=$(sysctl -n hw.ncpu)}" + +# --- Candidate (A): the Julia built by this pipeline ------------------------- + +echo "--- Download candidate build artifact" +buildkite-agent artifact download --step "build_${TRIPLET}" "${UPLOAD_FILENAME}.tar.gz" . + +echo "--- Extract candidate build artifact" +tar xzf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}/" + +# Codesign so the binary runs and JLL artifact dylibs load on aarch64. We don't +# update the bundled pkgimage checksums here: this check recompiles every stdlib +# into a throwaway depot, so the bundled pkgimages are never loaded. +echo "--- [mac] Codesign candidate" +.buildkite/utilities/macos/codesign.sh "${JULIA_INSTALL_DIR}" + +JULIA_A="$(pwd)/${JULIA_BINARY}" + +# --- Baseline (B): the merge-base nightly ----------------------------------- + +MERGE_BASE="$(git merge-base HEAD origin/master)" +SHORT_MERGE_BASE="$(echo "${MERGE_BASE}" | cut -c1-10)" +# The baseline may live on a different version line than HEAD, so read its VERSION. +BASE_VERSION="$(git show "${MERGE_BASE}:VERSION")" +BASE_MAJMIN="$(cut -d. -f1-2 <<<"${BASE_VERSION}")" + +echo "--- Resolve merge-base nightly" +echo "Merge base: ${MERGE_BASE} (julia ${BASE_VERSION})" + +NIGHTLY_HOST="https://julialangnightlies-s3.julialang.org" +NIGHTLY_URL="${NIGHTLY_HOST}/bin/${OS}/${ARCH}/${BASE_MAJMIN}/julia-${SHORT_MERGE_BASE}-${OS}-${ARCH}.tar.gz" + +BASELINE_DIR="$(pwd)/baseline" +rm -rf "${BASELINE_DIR}" +mkdir -p "${BASELINE_DIR}" + +BASELINE_WORKTREE="$(pwd)/.stdlib-loadtime-baseline-src" +cleanup() { + if [[ -d "${BASELINE_WORKTREE}" ]]; then + git worktree remove --force "${BASELINE_WORKTREE}" || true + fi +} +trap cleanup EXIT + +if curl -fL --retry 3 -o baseline.tar.gz "${NIGHTLY_URL}"; then + echo "--- Extract merge-base nightly" + tar -C "${BASELINE_DIR}" --strip-components=1 -xzf baseline.tar.gz + + echo "--- [mac] Codesign baseline nightly" + .buildkite/utilities/macos/codesign.sh "${BASELINE_DIR}" + + JULIA_B="${BASELINE_DIR}/bin/julia" +else + echo "--- Merge-base nightly not found; building julia from source at ${SHORT_MERGE_BASE}" + rm -rf "${BASELINE_WORKTREE}" + git worktree add --detach "${BASELINE_WORKTREE}" "${MERGE_BASE}" + ( + cd "${BASELINE_WORKTREE}" + make -j"${JULIA_CPU_THREADS}" \ + VERBOSE=1 \ + "JULIA_CPU_TARGET=${JULIA_CPU_TARGET}" + ) + JULIA_B="${BASELINE_WORKTREE}/usr/bin/julia" +fi + +# --- Compare ---------------------------------------------------------------- + +echo "--- Compare stdlib load times" +"${JULIA_A}" .buildkite/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl \ + --a "${JULIA_A}" \ + --b "${JULIA_B}" diff --git a/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl b/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl new file mode 100644 index 00000000..73e65d64 --- /dev/null +++ b/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl @@ -0,0 +1,212 @@ +# Orchestrates a stdlib precompile/load-time regression comparison between two +# Julia binaries: +# +# A = the Julia built by this CI pipeline (the "candidate") +# B = the Julia nightly associated with the merge-base commit (the "baseline") +# +# For each binary, across several rounds, we: +# 1. Force a fresh re-precompilation of every loadable stdlib into an isolated, +# writable depot (so we never reuse the bundled pkgimages), timing the whole +# precompile. +# 2. Measure the time to load each stdlib from that freshly-populated depot. +# +# The work runs entirely against a throwaway depot via `Base.compilecache` / +# `Base.require`, so it needs no registry, manifest, or network. +# +# To control for machine noise we run the rounds in an A/B/B/A/B/A order and take +# the per-binary minimum of each metric (the least noise-perturbed sample) before +# comparing. +# +# We flag two kinds of "clear regression", each requiring the candidate to be both +# relatively and absolutely slower than the baseline: +# * total precompile time -- relative `PRECOMPILE_REL_THRESHOLD` (default 1.5x) +# and absolute `PRECOMPILE_ABS_THRESHOLD_S` (default 10s). +# * per-stdlib (or aggregate) load time -- relative `LOADTIME_REL_THRESHOLD` +# (default 1.5x) and absolute `LOADTIME_ABS_THRESHOLD_MS` (default 100ms). +# If any regression is found, the script exits non-zero. +# +# Usage: julia compare_stdlib_load_times.jl --a --b + +const MEASURE_SCRIPT = joinpath(@__DIR__, "measure_stdlib_load_times.jl") + +function parse_args(args) + a = b = nothing + i = 1 + while i <= length(args) + if args[i] == "--a" + a = args[i + 1]; i += 2 + elseif args[i] == "--b" + b = args[i + 1]; i += 2 + else + error("Unknown argument: $(args[i])") + end + end + (a === nothing || b === nothing) && error("Both --a and --b must be provided") + return abspath(a), abspath(b) +end + +# Read a TSV of `nameseconds` into a Dict. +function read_timings(path) + timings = Dict{String,Float64}() + for line in eachline(path) + isempty(strip(line)) && continue + name, t = split(line, '\t') + timings[String(name)] = parse(Float64, t) + end + return timings +end + +# Create an isolated, initially-empty depot for `julia`. It contains no compiled +# cache, so stdlibs are forced to (re)precompile into it; we symlink the binary's +# bundled `artifacts` directory so JLL stdlibs can still resolve their artifacts. +function setup_depot(julia::String) + depot = mktempdir(; prefix = "stdlib-loadtime-") + bundled = readchomp(`$(julia) --startup-file=no -e 'print(DEPOT_PATH[end])'`) + artifacts = joinpath(bundled, "artifacts") + if isdir(artifacts) + symlink(artifacts, joinpath(depot, "artifacts")) + end + return depot +end + +# Run the measure helper with `julia` against the isolated `depot` as the *only* +# depot (a single path, no trailing `:`), so cache lookups never fall through to +# the bundled pkgimages. Returns the path of the output file it wrote. +function run_measure(julia::String, depot::String, mode::String, args...) + out = tempname() + env = copy(ENV) + env["JULIA_DEPOT_PATH"] = depot + run(setenv(`$(julia) --startup-file=no $(MEASURE_SCRIPT) $(mode) $(out) $(args...)`, env)) + return out +end + +# Per-key minimum across repeated samples (each a Dict). The min is the least +# noise-perturbed sample, so it's the most robust estimate of true load time. +function min_load(samples) + keysets = (Set(keys(s)) for s in samples) + common = reduce(intersect, keysets) + return Dict(k => minimum(s[k] for s in samples) for k in common) +end + +# Load time of every stdlib in `names`, each in its own fresh process (loading one +# stdlib pulls in its deps, so a shared session would report later ones as already +# resident). Returns a `name => seconds` Dict for this sample. +function measure_sample(julia::String, depot::String, names) + timings = Dict{String,Float64}() + for name in names + merge!(timings, read_timings(run_measure(julia, depot, "measure", name))) + end + return timings +end + +# One measurement round for `julia`: create a fresh, isolated depot, precompile +# every stdlib into it (timing it), then measure each stdlib's warm-cache load +# time. The depot is removed afterward so repeated rounds don't pile up compiled +# caches. Returns `(precompile_seconds, name => load_seconds Dict)`. +function measure_round(julia::String) + depot = setup_depot(julia) + try + lines = readlines(run_measure(julia, depot, "precompile")) + precompile_s = parse(Float64, lines[1]) + names = lines[2:end] + return precompile_s, measure_sample(julia, depot, names) + finally + rm(depot; recursive = true, force = true) + end +end + +function main() + julia_a, julia_b = parse_args(ARGS) + + rel_threshold = parse(Float64, get(ENV, "LOADTIME_REL_THRESHOLD", "1.5")) + abs_threshold = parse(Float64, get(ENV, "LOADTIME_ABS_THRESHOLD_MS", "100")) / 1000 + pre_rel_threshold = parse(Float64, get(ENV, "PRECOMPILE_REL_THRESHOLD", "1.5")) + pre_abs_threshold = parse(Float64, get(ENV, "PRECOMPILE_ABS_THRESHOLD_S", "10")) + + # Run the precompile+load rounds in A/B/B/A/B/A order so machine drift/noise is + # spread across both binaries, then take the per-binary minimum of each metric. + println("--- Round 1: precompile + load (A)"); pa1, a1 = measure_round(julia_a) + println("--- Round 1: precompile + load (B)"); pb1, b1 = measure_round(julia_b) + println("--- Round 2: precompile + load (B)"); pb2, b2 = measure_round(julia_b) + println("--- Round 2: precompile + load (A)"); pa2, a2 = measure_round(julia_a) + println("--- Round 3: precompile + load (B)"); pb3, b3 = measure_round(julia_b) + println("--- Round 3: precompile + load (A)"); pa3, a3 = measure_round(julia_a) + + pre_a = minimum((pa1, pa2, pa3)) + pre_b = minimum((pb1, pb2, pb3)) + a_load = min_load((a1, a2, a3)) + b_load = min_load((b1, b2, b3)) + + common = sort!(collect(intersect(keys(a_load), keys(b_load)))) + + regressions = Tuple{String,Float64,Float64,Float64}[] + rows = Tuple{String,Float64,Float64,Float64,Bool}[] + total_a = total_b = 0.0 + for name in common + ta, tb = a_load[name], b_load[name] + total_a += ta + total_b += tb + ratio = tb > 0 ? ta / tb : 1.0 + is_reg = (ta > tb * rel_threshold) && (ta - tb > abs_threshold) + push!(rows, (name, ta, tb, ratio, is_reg)) + is_reg && push!(regressions, (name, ta, tb, ratio)) + end + + sort!(rows; by = r -> r[4], rev = true) + + println("+++ Stdlib load-time comparison (A = candidate, B = merge-base nightly)") + println(rpad("stdlib", 32), lpad("A (ms)", 12), lpad("B (ms)", 12), lpad("A/B", 10)) + for (name, ta, tb, ratio, is_reg) in rows + println( + rpad(name, 32), + lpad(round(ta * 1000; digits = 2), 12), + lpad(round(tb * 1000; digits = 2), 12), + lpad(round(ratio; digits = 3), 10), + is_reg ? " <-- REGRESSION" : "", + ) + end + + total_ratio = total_b > 0 ? total_a / total_b : 1.0 + pre_ratio = pre_b > 0 ? pre_a / pre_b : 1.0 + precompile_regression = (pre_a > pre_b * pre_rel_threshold) && + (pre_a - pre_b > pre_abs_threshold) + total_regression = (total_a > total_b * rel_threshold) && + (total_a - total_b > abs_threshold) + + println() + println("Precompile total: A = ", round(pre_a; digits = 2), "s, ", + "B = ", round(pre_b; digits = 2), "s, ", + "A/B = ", round(pre_ratio; digits = 3), + precompile_regression ? " <-- REGRESSION" : "") + println("Load total: A = ", round(total_a * 1000; digits = 2), "ms, ", + "B = ", round(total_b * 1000; digits = 2), "ms, ", + "A/B = ", round(total_ratio; digits = 3), + total_regression ? " <-- REGRESSION" : "") + println("Precompile thresholds: relative > ", pre_rel_threshold, + "x and absolute > ", round(pre_abs_threshold; digits = 1), "s") + println("Load thresholds: relative > ", rel_threshold, + "x and absolute > ", round(abs_threshold * 1000; digits = 1), "ms") + + if isempty(regressions) && !total_regression && !precompile_regression + println("\n✓ No clear stdlib precompile- or load-time regressions detected.") + return 0 + end + + println("\n✗ Clear stdlib precompile/load-time regression(s) detected:") + if precompile_regression + println(" - total precompile time: ", round(pre_a; digits = 2), "s vs ", + round(pre_b; digits = 2), "s (", round(pre_ratio; digits = 3), "x)") + end + for (name, ta, tb, ratio) in regressions + println(" - ", name, ": ", round(ta * 1000; digits = 2), "ms vs ", + round(tb * 1000; digits = 2), "ms (", round(ratio; digits = 3), "x)") + end + if total_regression + println(" - aggregate load time: ", round(total_a * 1000; digits = 2), + "ms vs ", round(total_b * 1000; digits = 2), "ms (", + round(total_ratio; digits = 3), "x)") + end + return 1 +end + +exit(main()) diff --git a/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl b/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl new file mode 100644 index 00000000..cfbafa53 --- /dev/null +++ b/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl @@ -0,0 +1,112 @@ +# Helper invoked (in separate processes) by the orchestrator +# `compare_stdlib_load_times.jl`, once per Julia binary, against a fresh isolated +# depot. It has two modes: +# +# precompile Precompile every loadable stdlib (dependencies first, so each is +# compiled exactly once) into the depot with `Base.compilecache`, and +# record the total elapsed time. No Pkg/registry/manifest is involved, +# so this never touches the network or the agent's depot. Writes the +# elapsed seconds on the first line of , then the names of +# the stdlibs that were compiled, one per line. +# +# measure Load the single named stdlib with `Base.require` (from the warm +# cache the precompile mode just built) and record its time. A +# separate process per stdlib is required because loading one stdlib +# pulls in its dependencies, so a shared session would report later +# stdlibs as already-resident (~0s). Writes a `nameseconds` line. +# +# Usage: +# julia measure_stdlib_load_times.jl precompile +# julia measure_stdlib_load_times.jl measure + +# Parse a stdlib's `Project.toml` for its uuid and the names listed under `[deps]`. +# We scan the TOML by hand to avoid loading the `TOML` stdlib (which would itself +# perturb the timings). +function read_project(proj) + uuid = nothing + deps = String[] + in_deps = false + for line in eachline(proj) + s = strip(line) + if s == "[deps]" + in_deps = true + elseif startswith(s, "[") + in_deps = false + elseif in_deps + m = match(r"^(\S+)\s*=", s) + m === nothing || push!(deps, String(m.captures[1])) + else + m = match(r"^uuid\s*=\s*\"([^\"]+)\"", s) + m === nothing || (uuid = String(m.captures[1])) + end + end + return uuid, deps +end + +# Every loadable stdlib (one that exposes a uuid) as `name => (uuid, dep_names)`. +function loadable_stdlibs() + stdlibs = Dict{String,Tuple{String,Vector{String}}}() + for name in readdir(Sys.STDLIB) + proj = joinpath(Sys.STDLIB, name, "Project.toml") + isfile(proj) || continue + uuid, deps = read_project(proj) + uuid === nothing || (stdlibs[name] = (uuid, deps)) + end + return stdlibs +end + +# Precompile every stdlib into the (fresh) depot, dependencies first so that each +# is compiled exactly once. Returns `(elapsed_seconds, sorted_names)`. +function precompile_all() + stdlibs = loadable_stdlibs() + done = Set{String}() + function precompile_one(name) + (name in done || !haskey(stdlibs, name)) && return + push!(done, name) + uuid, deps = stdlibs[name] + foreach(precompile_one, deps) + Base.compilecache(Base.PkgId(Base.UUID(uuid), name), devnull, devnull) + end + names = sort!(collect(keys(stdlibs))) + elapsed = @elapsed foreach(precompile_one, names) + return elapsed, names +end + +# Load the single named stdlib, timing it from the warm cache. +function measure_one(name) + stdlibs = loadable_stdlibs() + haskey(stdlibs, name) || error("Unknown stdlib: $(name)") + uuid, _ = stdlibs[name] + return @elapsed Base.require(Base.PkgId(Base.UUID(uuid), name)) +end + +function usage_error() + println(stderr, """ + Usage: + julia measure_stdlib_load_times.jl precompile + julia measure_stdlib_load_times.jl measure """) + exit(2) +end + +function main() + length(ARGS) >= 2 || usage_error() + mode, out_file = ARGS[1], ARGS[2] + + if mode == "precompile" + elapsed, names = precompile_all() + open(out_file, "w") do io + println(io, elapsed) + foreach(n -> println(io, n), names) + end + elseif mode == "measure" + length(ARGS) == 3 || usage_error() + t = measure_one(ARGS[3]) + open(out_file, "w") do io + println(io, ARGS[3], '\t', t) + end + else + usage_error() + end +end + +main()