From 82e6009177261b1f6fbbcd87529b742172c909ba Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 19 Jun 2026 10:53:39 -0400 Subject: [PATCH 1/2] Add stdlib load-time regression CI check (macOS aarch64) Compares the Julia built by the pipeline (candidate) against the nightly associated with the merge-base commit (baseline). For each binary it re-precompiles every stdlib into an isolated, cache-free depot and measures load times, flagging clear regressions (relative and absolute thresholds). - utilities/stdlib_load_time_regression.sh: driver that downloads/extracts both binaries, building the baseline from source if the merge-base nightly is missing, then runs the comparison. Only runs on macOS aarch64. - compare_stdlib_load_times.jl: orchestrator. Per binary, generates a resolved stdlib Project/Manifest with the standard depot, precompiles the whole environment at once into a fresh depot, then measures load times in A/B/B/A/B/A order taking the per-stdlib minimum across repeats. - measure_stdlib_load_times.jl: per-binary helper with generate/precompile/ measure modes; each measure runs one stdlib in its own process so previously-loaded deps don't skew timings. - pipelines/main/misc/stdlib_load_time_regression.yml + launcher upload. --- pipelines/main/launch_unsigned_jobs.yml | 1 + .../main/misc/stdlib_load_time_regression.yml | 20 ++ utilities/stdlib_load_time_regression.sh | 99 +++++++ .../compare_stdlib_load_times.jl | 262 ++++++++++++++++++ .../measure_stdlib_load_times.jl | 143 ++++++++++ 5 files changed, 525 insertions(+) create mode 100644 pipelines/main/misc/stdlib_load_time_regression.yml create mode 100644 utilities/stdlib_load_time_regression.sh create mode 100644 utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl create mode 100644 utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index d06b0e94..e7a431b9 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -86,6 +86,7 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/stdlib_load_time_regression.yml # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml diff --git a/pipelines/main/misc/stdlib_load_time_regression.yml b/pipelines/main/misc/stdlib_load_time_regression.yml new file mode 100644 index 00000000..5f2785d4 --- /dev/null +++ b/pipelines/main/misc/stdlib_load_time_regression.yml @@ -0,0 +1,20 @@ +steps: + - group: "Check" + steps: + - label: ":macos: stdlib load-time regression (aarch64)" + key: "stdlib-load-time-regression" + depends_on: + - "build_aarch64-apple-darwin" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + commands: "bash .buildkite/utilities/stdlib_load_time_regression.sh" + timeout_in_minutes: 120 + agents: + queue: "test" + os: "macos" + arch: "aarch64" + env: + TRIPLET: "aarch64-apple-darwin" + BUILDKITE_CANCEL_SIGNAL: "SIGQUIT" diff --git a/utilities/stdlib_load_time_regression.sh b/utilities/stdlib_load_time_regression.sh new file mode 100644 index 00000000..a75b14de --- /dev/null +++ b/utilities/stdlib_load_time_regression.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +# This script runs a stdlib precompile/load-time regression check, comparing: +# +# A = the Julia built by this CI pipeline (the "candidate") +# B = the Julia nightly associated with the merge-base commit (the "baseline") +# +# It downloads/extracts both binaries (building the baseline from source if the +# merge-base nightly is missing), then hands them to +# `compare_stdlib_load_times.jl`, which re-precompiles every stdlib for both +# (timing the precompile) and measures their load times across several A/B/B/A/B/A +# rounds (to control for noise), then flags clear precompile- or load-time +# regressions. +# +# This check is only meaningful on macOS aarch64. + +set -euo pipefail + +# First, get things like `OS`, `ARCH`, `TRIPLET`, `JULIA_INSTALL_DIR`, +# `UPLOAD_FILENAME`, `JULIA_BINARY`, `JULIA_CPU_TARGET`, etc... +# shellcheck source=SCRIPTDIR/build_envs.sh +source .buildkite/utilities/build_envs.sh + +if [[ "${OS}" != "macos" ]] || [[ "${ARCH}" != "aarch64" ]]; then + echo "This check only runs on macOS aarch64 (got ${OS} ${ARCH}); skipping." + exit 0 +fi + +# Default the number of build threads if the agent did not provide it. +: "${JULIA_CPU_THREADS:=$(sysctl -n hw.ncpu)}" + +# --- Candidate (A): the Julia built by this pipeline ------------------------- + +echo "--- Download candidate build artifact" +buildkite-agent artifact download --step "build_${TRIPLET}" "${UPLOAD_FILENAME}.tar.gz" . + +echo "--- Extract candidate build artifact" +tar xzf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}/" + +echo "--- [mac] Codesign candidate" +.buildkite/utilities/macos/codesign.sh "${JULIA_INSTALL_DIR}" +"${JULIA_INSTALL_DIR}/bin/julia" .buildkite/utilities/update_stdlib_pkgimage_checksums.jl + +JULIA_A="$(pwd)/${JULIA_BINARY}" + +# --- Baseline (B): the merge-base nightly ----------------------------------- + +MERGE_BASE="$(git merge-base HEAD origin/master)" +SHORT_MERGE_BASE="$(echo "${MERGE_BASE}" | cut -c1-10)" +# The baseline may live on a different version line than HEAD, so read its VERSION. +BASE_VERSION="$(git show "${MERGE_BASE}:VERSION")" +BASE_MAJMIN="$(cut -d. -f1-2 <<<"${BASE_VERSION}")" + +echo "--- Resolve merge-base nightly" +echo "Merge base: ${MERGE_BASE} (julia ${BASE_VERSION})" + +NIGHTLY_HOST="https://julialangnightlies-s3.julialang.org" +NIGHTLY_URL="${NIGHTLY_HOST}/bin/${OS}/${ARCH}/${BASE_MAJMIN}/julia-${SHORT_MERGE_BASE}-${OS}-${ARCH}.tar.gz" + +BASELINE_DIR="$(pwd)/baseline" +rm -rf "${BASELINE_DIR}" +mkdir -p "${BASELINE_DIR}" + +BASELINE_WORKTREE="$(pwd)/.stdlib-loadtime-baseline-src" +cleanup() { + if [[ -d "${BASELINE_WORKTREE}" ]]; then + git worktree remove --force "${BASELINE_WORKTREE}" || true + fi +} +trap cleanup EXIT + +if curl -fL --retry 3 -o baseline.tar.gz "${NIGHTLY_URL}"; then + echo "--- Extract merge-base nightly" + tar -C "${BASELINE_DIR}" --strip-components=1 -xzf baseline.tar.gz + + echo "--- [mac] Codesign baseline nightly" + .buildkite/utilities/macos/codesign.sh "${BASELINE_DIR}" + "${BASELINE_DIR}/bin/julia" .buildkite/utilities/update_stdlib_pkgimage_checksums.jl + + JULIA_B="${BASELINE_DIR}/bin/julia" +else + echo "--- Merge-base nightly not found; building julia from source at ${SHORT_MERGE_BASE}" + rm -rf "${BASELINE_WORKTREE}" + git worktree add --detach "${BASELINE_WORKTREE}" "${MERGE_BASE}" + ( + cd "${BASELINE_WORKTREE}" + make -j"${JULIA_CPU_THREADS}" \ + VERBOSE=1 \ + "JULIA_CPU_TARGET=${JULIA_CPU_TARGET}" + ) + JULIA_B="${BASELINE_WORKTREE}/usr/bin/julia" +fi + +# --- Compare ---------------------------------------------------------------- + +echo "--- Compare stdlib load times" +"${JULIA_A}" .buildkite/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl \ + --a "${JULIA_A}" \ + --b "${JULIA_B}" diff --git a/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl b/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl new file mode 100644 index 00000000..6c76b173 --- /dev/null +++ b/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl @@ -0,0 +1,262 @@ +# Orchestrates a stdlib precompile/load-time regression comparison between two +# Julia binaries: +# +# A = the Julia built by this CI pipeline (the "candidate") +# B = the Julia nightly associated with the merge-base commit (the "baseline") +# +# For each binary, across several rounds, we: +# 1. Force a fresh re-precompilation of every loadable stdlib into an isolated, +# writable depot (so we never reuse the bundled pkgimages), timing the whole +# parallel precompile batch. +# 2. Measure the time to load each stdlib from that freshly-populated depot. +# +# To control for machine noise we run the rounds in an A/B/B/A/B/A order and take +# the per-binary minimum of each metric (the least noise-perturbed sample) before +# comparing. +# +# We flag two kinds of "clear regression", each requiring the candidate to be both +# relatively and absolutely slower than the baseline: +# * total precompile time -- relative `PRECOMPILE_REL_THRESHOLD` (default 1.5x) +# and absolute `PRECOMPILE_ABS_THRESHOLD_S` (default 10s). +# * per-stdlib (or aggregate) load time -- relative `LOADTIME_REL_THRESHOLD` +# (default 1.5x) and absolute `LOADTIME_ABS_THRESHOLD_MS` (default 100ms). +# If any regression is found, the script exits non-zero. +# +# Usage: julia compare_stdlib_load_times.jl --a --b + +const MEASURE_SCRIPT = joinpath(@__DIR__, "measure_stdlib_load_times.jl") + +function parse_args(args) + a = b = nothing + i = 1 + while i <= length(args) + if args[i] == "--a" + a = args[i + 1]; i += 2 + elseif args[i] == "--b" + b = args[i + 1]; i += 2 + else + error("Unknown argument: $(args[i])") + end + end + (a === nothing || b === nothing) && error("Both --a and --b must be provided") + return abspath(a), abspath(b) +end + +# Read a TSV of `nameseconds` into a Dict. +function read_timings(path) + timings = Dict{String,Float64}() + for line in eachline(path) + isempty(strip(line)) && continue + name, t = split(line, '\t') + timings[String(name)] = parse(Float64, t) + end + return timings +end + +# Create an isolated, initially-empty depot for `julia`. It contains no compiled +# cache, so stdlibs are forced to (re)precompile into it; we symlink the binary's +# bundled `artifacts` directory so JLL stdlibs can still resolve their artifacts. +function setup_depot(julia::String) + depot = mktempdir(; prefix = "stdlib-loadtime-") + bundled = readchomp(`$(julia) --startup-file=no -e 'print(DEPOT_PATH[end])'`) + artifacts = joinpath(bundled, "artifacts") + if isdir(artifacts) + symlink(artifacts, joinpath(depot, "artifacts")) + end + return depot +end + +# Run the measure script with `julia`, using the isolated depot as the *only* +# depot (a single path, no trailing `:`), so compiled-cache lookups never fall +# through to the bundled pkgimages. `project` is the shared dir whose generated +# `Project.toml`/`Manifest.toml` lists the stdlibs as deps. +function run_phase(julia::String, depot::String, project::String, mode::String) + out = tempname() + env = copy(ENV) + env["JULIA_DEPOT_PATH"] = depot + cmd = `$(julia) --startup-file=no --color=yes $(MEASURE_SCRIPT) $(mode) $(out) $(project)` + run(setenv(cmd, env)) + return read_timings(out) +end + +# Generate the `Project.toml`/`Manifest.toml` for `julia` using its *standard* +# depot, so resolution can consult the registry. The resolved environment is +# reused (against the fresh depot) by the precompile and measure phases. +function generate_project(julia::String, project::String) + cmd = `$(julia) --startup-file=no --color=yes $(MEASURE_SCRIPT) generate /dev/null $(project)` + run(cmd) + return nothing +end + +# Names of the stdlibs listed under `[deps]` in the generated `Project.toml`. +function stdlib_names(project::String) + names = String[] + in_deps = false + for line in eachline(joinpath(project, "Project.toml")) + s = strip(line) + if s == "[deps]" + in_deps = true + elseif startswith(s, "[") + in_deps = false + elseif in_deps + m = match(r"^(\S+)\s*=", s) + m === nothing || push!(names, String(m.captures[1])) + end + end + return names +end + +# Measure the load time of every stdlib, *each in its own fresh process* (loading +# one stdlib pulls in its deps, so a shared session would report later stdlibs as +# already-resident). Returns a `name => seconds` Dict for this sample. +function measure_sample(julia::String, depot::String, project::String, names) + env = copy(ENV) + env["JULIA_DEPOT_PATH"] = depot + timings = Dict{String,Float64}() + for name in names + out = tempname() + cmd = `$(julia) --startup-file=no $(MEASURE_SCRIPT) measure $(out) $(project) $(name)` + run(setenv(cmd, env)) + merge!(timings, read_timings(out)) + end + return timings +end + +# Per-key minimum across repeated samples (each a Dict). The min is the least +# noise-perturbed sample, so it's the most robust estimate of true load time. +function min_load(samples) + keysets = (Set(keys(s)) for s in samples) + common = reduce(intersect, keysets) + return Dict(k => minimum(s[k] for s in samples) for k in common) +end + +# One measurement round for `julia`: create a fresh, isolated depot, precompile the +# whole stdlib environment into it (timing the parallel batch), then measure each +# stdlib's load time against that warm depot. The depot is removed afterward so that +# repeated rounds don't accumulate compiled caches on disk. Returns +# `(precompile_seconds, name => load_seconds Dict)`. +function measure_round(julia::String, project::String, names) + depot = setup_depot(julia) + try + precompile_s = run_phase(julia, depot, project, "precompile")["__precompile__"] + load = measure_sample(julia, depot, project, names) + return precompile_s, load + finally + rm(depot; recursive = true, force = true) + end +end + +function main() + julia_a, julia_b = parse_args(ARGS) + + rel_threshold = parse(Float64, get(ENV, "LOADTIME_REL_THRESHOLD", "1.5")) + abs_threshold = parse(Float64, get(ENV, "LOADTIME_ABS_THRESHOLD_MS", "100")) / 1000 + pre_rel_threshold = parse(Float64, get(ENV, "PRECOMPILE_REL_THRESHOLD", "1.5")) + pre_abs_threshold = parse(Float64, get(ENV, "PRECOMPILE_ABS_THRESHOLD_S", "10")) + + project_a = mktempdir(; prefix = "stdlib-loadtime-proj-a-") + project_b = mktempdir(; prefix = "stdlib-loadtime-proj-b-") + + # Resolve each environment with the standard depot first, so the fresh-depot + # precompile phase has a manifest and never needs the registry. + println("--- Generate stdlib environment (candidate A)") + generate_project(julia_a, project_a) + println("--- Generate stdlib environment (baseline B)") + generate_project(julia_b, project_b) + + names_a = stdlib_names(project_a) + names_b = stdlib_names(project_b) + + # Run the precompile+load rounds in A/B/B/A/B/A order so machine drift/noise is + # spread across both binaries, then take the per-binary minimum of each metric. + println("--- Round 1: precompile + load (A)") + pa1, a1 = measure_round(julia_a, project_a, names_a) + println("--- Round 1: precompile + load (B)") + pb1, b1 = measure_round(julia_b, project_b, names_b) + println("--- Round 2: precompile + load (B)") + pb2, b2 = measure_round(julia_b, project_b, names_b) + println("--- Round 2: precompile + load (A)") + pa2, a2 = measure_round(julia_a, project_a, names_a) + println("--- Round 3: precompile + load (B)") + pb3, b3 = measure_round(julia_b, project_b, names_b) + println("--- Round 3: precompile + load (A)") + pa3, a3 = measure_round(julia_a, project_a, names_a) + + pre_a = minimum((pa1, pa2, pa3)) + pre_b = minimum((pb1, pb2, pb3)) + a_load = min_load((a1, a2, a3)) + b_load = min_load((b1, b2, b3)) + + common = sort!(collect(intersect(keys(a_load), keys(b_load)))) + + regressions = Tuple{String,Float64,Float64,Float64}[] + rows = Tuple{String,Float64,Float64,Float64,Bool}[] + total_a = total_b = 0.0 + for name in common + ta, tb = a_load[name], b_load[name] + total_a += ta + total_b += tb + ratio = tb > 0 ? ta / tb : 1.0 + is_reg = (ta > tb * rel_threshold) && (ta - tb > abs_threshold) + push!(rows, (name, ta, tb, ratio, is_reg)) + is_reg && push!(regressions, (name, ta, tb, ratio)) + end + + sort!(rows; by = r -> r[4], rev = true) + + println("+++ Stdlib load-time comparison (A = candidate, B = merge-base nightly)") + println(rpad("stdlib", 32), lpad("A (ms)", 12), lpad("B (ms)", 12), lpad("A/B", 10)) + for (name, ta, tb, ratio, is_reg) in rows + println( + rpad(name, 32), + lpad(round(ta * 1000; digits = 2), 12), + lpad(round(tb * 1000; digits = 2), 12), + lpad(round(ratio; digits = 3), 10), + is_reg ? " <-- REGRESSION" : "", + ) + end + + total_ratio = total_b > 0 ? total_a / total_b : 1.0 + pre_ratio = pre_b > 0 ? pre_a / pre_b : 1.0 + precompile_regression = (pre_a > pre_b * pre_rel_threshold) && + (pre_a - pre_b > pre_abs_threshold) + total_regression = (total_a > total_b * rel_threshold) && + (total_a - total_b > abs_threshold) + + println() + println("Precompile total: A = ", round(pre_a; digits = 2), "s, ", + "B = ", round(pre_b; digits = 2), "s, ", + "A/B = ", round(pre_ratio; digits = 3), + precompile_regression ? " <-- REGRESSION" : "") + println("Load total: A = ", round(total_a * 1000; digits = 2), "ms, ", + "B = ", round(total_b * 1000; digits = 2), "ms, ", + "A/B = ", round(total_ratio; digits = 3), + total_regression ? " <-- REGRESSION" : "") + println("Precompile thresholds: relative > ", pre_rel_threshold, + "x and absolute > ", round(pre_abs_threshold; digits = 1), "s") + println("Load thresholds: relative > ", rel_threshold, + "x and absolute > ", round(abs_threshold * 1000; digits = 1), "ms") + + if isempty(regressions) && !total_regression && !precompile_regression + println("\nāœ“ No clear stdlib precompile- or load-time regressions detected.") + return 0 + end + + println("\nāœ— Clear stdlib precompile/load-time regression(s) detected:") + if precompile_regression + println(" - total precompile time: ", round(pre_a; digits = 2), "s vs ", + round(pre_b; digits = 2), "s (", round(pre_ratio; digits = 3), "x)") + end + for (name, ta, tb, ratio) in regressions + println(" - ", name, ": ", round(ta * 1000; digits = 2), "ms vs ", + round(tb * 1000; digits = 2), "ms (", round(ratio; digits = 3), "x)") + end + if total_regression + println(" - aggregate load time: ", round(total_a * 1000; digits = 2), + "ms vs ", round(total_b * 1000; digits = 2), "ms (", + round(total_ratio; digits = 3), "x)") + end + return 1 +end + +exit(main()) diff --git a/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl b/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl new file mode 100644 index 00000000..0c633ddc --- /dev/null +++ b/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl @@ -0,0 +1,143 @@ +# Helper invoked (in separate processes) by the orchestrator +# `compare_stdlib_load_times.jl` for one Julia binary at a time. It has three +# modes, run in this order: +# +# generate Run with the binary's *standard* depot. Write a `Project.toml` +# that lists every loadable stdlib as a dependency and `Pkg.resolve` +# it to a `Manifest.toml`. Resolution needs the standard depot (it +# may consult the registry), which is why this is its own phase. +# +# precompile Run with a *fresh, isolated* depot (a single, writable path with +# no compiled cache). With the manifest already present, precompile +# the *entire* environment in one parallel batch via +# `Base.Precompilation.precompilepkgs`, building every stdlib's +# pkgimage into the fresh depot. No resolve happens here, so no +# registry is needed. The aggregate precompile time is recorded. +# +# measure Run with the same fresh depot, *one stdlib per process*. Load the +# single named stdlib with `Base.require` and record its time. A +# separate process per stdlib is required because loading one +# stdlib pulls in its dependencies, so measuring many in a single +# session would report later ones as already-resident (ā‰ˆ0s). +# +# Generating the Project/Manifest with the standard depot but precompiling and +# loading against the fresh depot is what forces real (re)precompilation instead +# of reusing the binary's bundled pkgimages. +# +# The measured time is written to the output file as a tab-separated +# `nameseconds` line (`generate` writes nothing). +# +# Usage: +# julia measure_stdlib_load_times.jl generate +# julia measure_stdlib_load_times.jl precompile +# julia measure_stdlib_load_times.jl measure + +const PKG_UUID = Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") + + +# Return `name => uuid` for a stdlib that can be added as a project dependency, +# or `nothing` if it has no `Project.toml`/`uuid` (and so can't be precompiled as +# an ordinary package). We scan the TOML by hand to avoid loading the `TOML` +# stdlib (which would itself perturb the timings). +function stdlib_name_uuid(dir, name) + proj = joinpath(dir, name, "Project.toml") + isfile(proj) || return nothing + uuid = nothing + for line in eachline(proj) + m = match(r"^\s*uuid\s*=\s*\"([^\"]+)\"", line) + m === nothing || (uuid = m.captures[1]) + end + uuid === nothing && return nothing + return name => uuid +end + +# All stdlibs (sorted) that expose a `uuid`, as `name => uuid` pairs. +function loadable_stdlibs() + dir = Sys.STDLIB + deps = Pair{String,String}[] + for name in sort(readdir(dir)) + isdir(joinpath(dir, name)) || continue + nu = stdlib_name_uuid(dir, name) + nu === nothing || push!(deps, nu) + end + return deps +end + +# Write a `Project.toml` whose `[deps]` lists every loadable stdlib, so that +# activating it turns the stdlibs into ordinary, precompilable dependencies. +function write_project(project_dir, deps) + mkpath(project_dir) + open(joinpath(project_dir, "Project.toml"), "w") do io + println(io, "[deps]") + for (name, uuid) in deps + println(io, name, " = \"", uuid, "\"") + end + end + return joinpath(project_dir, "Project.toml") +end + +# generate phase: write the Project.toml and resolve it to a Manifest.toml using +# the (standard) depot this process was started with. +function generate(project_dir) + deps = loadable_stdlibs() + write_project(project_dir, deps) + Pkg = Base.require(Base.PkgId(PKG_UUID, "Pkg")) + Base.invokelatest(Pkg.activate, project_dir; io = devnull) + Base.invokelatest(Pkg.resolve; io = devnull) + return nothing +end + +# precompile phase: with the Manifest already present, precompile the whole +# environment at once (parallel batch) into the fresh depot. +function precompile_all(project_file) + Base.set_active_project(project_file) + return @elapsed Base.Precompilation.precompilepkgs(; warn_loaded = false) +end + +# measure phase: load the single named stdlib (in this fresh process), timing it +# from the warm cache. +function measure_one(name) + uuid = nothing + for (n, u) in loadable_stdlibs() + n == name && (uuid = u; break) + end + uuid === nothing && error("Unknown stdlib: $(name)") + return @elapsed Base.require(Base.PkgId(Base.UUID(uuid), name)) +end + +function main() + length(ARGS) >= 3 || usage_error() + mode, out_file, project_dir = ARGS[1], ARGS[2], ARGS[3] + project_file = joinpath(project_dir, "Project.toml") + + if mode == "generate" + generate(project_dir) + return + elseif mode == "precompile" + timings = ["__precompile__" => precompile_all(project_file)] + elseif mode == "measure" + length(ARGS) == 4 || usage_error() + name = ARGS[4] + Base.set_active_project(project_file) + timings = [name => measure_one(name)] + else + usage_error() + end + + open(out_file, "w") do io + for (name, t) in timings + println(io, name, '\t', t) + end + end +end + +function usage_error() + println(stderr, """ + Usage: + julia measure_stdlib_load_times.jl generate + julia measure_stdlib_load_times.jl precompile + julia measure_stdlib_load_times.jl measure """) + exit(2) +end + +main() From a48506e872e399eb34aa4dd5485adb3f462738aa Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 19 Jun 2026 23:00:38 -0400 Subject: [PATCH 2/2] wip --- utilities/stdlib_load_time_regression.sh | 5 +- .../compare_stdlib_load_times.jl | 122 ++++------- .../measure_stdlib_load_times.jl | 193 ++++++++---------- 3 files changed, 120 insertions(+), 200 deletions(-) diff --git a/utilities/stdlib_load_time_regression.sh b/utilities/stdlib_load_time_regression.sh index a75b14de..4d831b20 100644 --- a/utilities/stdlib_load_time_regression.sh +++ b/utilities/stdlib_load_time_regression.sh @@ -37,9 +37,11 @@ buildkite-agent artifact download --step "build_${TRIPLET}" "${UPLOAD_FILENAME}. echo "--- Extract candidate build artifact" tar xzf "${UPLOAD_FILENAME}.tar.gz" "${JULIA_INSTALL_DIR}/" +# Codesign so the binary runs and JLL artifact dylibs load on aarch64. We don't +# update the bundled pkgimage checksums here: this check recompiles every stdlib +# into a throwaway depot, so the bundled pkgimages are never loaded. echo "--- [mac] Codesign candidate" .buildkite/utilities/macos/codesign.sh "${JULIA_INSTALL_DIR}" -"${JULIA_INSTALL_DIR}/bin/julia" .buildkite/utilities/update_stdlib_pkgimage_checksums.jl JULIA_A="$(pwd)/${JULIA_BINARY}" @@ -75,7 +77,6 @@ if curl -fL --retry 3 -o baseline.tar.gz "${NIGHTLY_URL}"; then echo "--- [mac] Codesign baseline nightly" .buildkite/utilities/macos/codesign.sh "${BASELINE_DIR}" - "${BASELINE_DIR}/bin/julia" .buildkite/utilities/update_stdlib_pkgimage_checksums.jl JULIA_B="${BASELINE_DIR}/bin/julia" else diff --git a/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl b/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl index 6c76b173..73e65d64 100644 --- a/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl +++ b/utilities/stdlib_load_time_regression/compare_stdlib_load_times.jl @@ -7,9 +7,12 @@ # For each binary, across several rounds, we: # 1. Force a fresh re-precompilation of every loadable stdlib into an isolated, # writable depot (so we never reuse the bundled pkgimages), timing the whole -# parallel precompile batch. +# precompile. # 2. Measure the time to load each stdlib from that freshly-populated depot. # +# The work runs entirely against a throwaway depot via `Base.compilecache` / +# `Base.require`, so it needs no registry, manifest, or network. +# # To control for machine noise we run the rounds in an A/B/B/A/B/A order and take # the per-binary minimum of each metric (the least noise-perturbed sample) before # comparing. @@ -66,60 +69,15 @@ function setup_depot(julia::String) return depot end -# Run the measure script with `julia`, using the isolated depot as the *only* -# depot (a single path, no trailing `:`), so compiled-cache lookups never fall -# through to the bundled pkgimages. `project` is the shared dir whose generated -# `Project.toml`/`Manifest.toml` lists the stdlibs as deps. -function run_phase(julia::String, depot::String, project::String, mode::String) +# Run the measure helper with `julia` against the isolated `depot` as the *only* +# depot (a single path, no trailing `:`), so cache lookups never fall through to +# the bundled pkgimages. Returns the path of the output file it wrote. +function run_measure(julia::String, depot::String, mode::String, args...) out = tempname() env = copy(ENV) env["JULIA_DEPOT_PATH"] = depot - cmd = `$(julia) --startup-file=no --color=yes $(MEASURE_SCRIPT) $(mode) $(out) $(project)` - run(setenv(cmd, env)) - return read_timings(out) -end - -# Generate the `Project.toml`/`Manifest.toml` for `julia` using its *standard* -# depot, so resolution can consult the registry. The resolved environment is -# reused (against the fresh depot) by the precompile and measure phases. -function generate_project(julia::String, project::String) - cmd = `$(julia) --startup-file=no --color=yes $(MEASURE_SCRIPT) generate /dev/null $(project)` - run(cmd) - return nothing -end - -# Names of the stdlibs listed under `[deps]` in the generated `Project.toml`. -function stdlib_names(project::String) - names = String[] - in_deps = false - for line in eachline(joinpath(project, "Project.toml")) - s = strip(line) - if s == "[deps]" - in_deps = true - elseif startswith(s, "[") - in_deps = false - elseif in_deps - m = match(r"^(\S+)\s*=", s) - m === nothing || push!(names, String(m.captures[1])) - end - end - return names -end - -# Measure the load time of every stdlib, *each in its own fresh process* (loading -# one stdlib pulls in its deps, so a shared session would report later stdlibs as -# already-resident). Returns a `name => seconds` Dict for this sample. -function measure_sample(julia::String, depot::String, project::String, names) - env = copy(ENV) - env["JULIA_DEPOT_PATH"] = depot - timings = Dict{String,Float64}() - for name in names - out = tempname() - cmd = `$(julia) --startup-file=no $(MEASURE_SCRIPT) measure $(out) $(project) $(name)` - run(setenv(cmd, env)) - merge!(timings, read_timings(out)) - end - return timings + run(setenv(`$(julia) --startup-file=no $(MEASURE_SCRIPT) $(mode) $(out) $(args...)`, env)) + return out end # Per-key minimum across repeated samples (each a Dict). The min is the least @@ -130,17 +88,28 @@ function min_load(samples) return Dict(k => minimum(s[k] for s in samples) for k in common) end -# One measurement round for `julia`: create a fresh, isolated depot, precompile the -# whole stdlib environment into it (timing the parallel batch), then measure each -# stdlib's load time against that warm depot. The depot is removed afterward so that -# repeated rounds don't accumulate compiled caches on disk. Returns -# `(precompile_seconds, name => load_seconds Dict)`. -function measure_round(julia::String, project::String, names) +# Load time of every stdlib in `names`, each in its own fresh process (loading one +# stdlib pulls in its deps, so a shared session would report later ones as already +# resident). Returns a `name => seconds` Dict for this sample. +function measure_sample(julia::String, depot::String, names) + timings = Dict{String,Float64}() + for name in names + merge!(timings, read_timings(run_measure(julia, depot, "measure", name))) + end + return timings +end + +# One measurement round for `julia`: create a fresh, isolated depot, precompile +# every stdlib into it (timing it), then measure each stdlib's warm-cache load +# time. The depot is removed afterward so repeated rounds don't pile up compiled +# caches. Returns `(precompile_seconds, name => load_seconds Dict)`. +function measure_round(julia::String) depot = setup_depot(julia) try - precompile_s = run_phase(julia, depot, project, "precompile")["__precompile__"] - load = measure_sample(julia, depot, project, names) - return precompile_s, load + lines = readlines(run_measure(julia, depot, "precompile")) + precompile_s = parse(Float64, lines[1]) + names = lines[2:end] + return precompile_s, measure_sample(julia, depot, names) finally rm(depot; recursive = true, force = true) end @@ -154,33 +123,14 @@ function main() pre_rel_threshold = parse(Float64, get(ENV, "PRECOMPILE_REL_THRESHOLD", "1.5")) pre_abs_threshold = parse(Float64, get(ENV, "PRECOMPILE_ABS_THRESHOLD_S", "10")) - project_a = mktempdir(; prefix = "stdlib-loadtime-proj-a-") - project_b = mktempdir(; prefix = "stdlib-loadtime-proj-b-") - - # Resolve each environment with the standard depot first, so the fresh-depot - # precompile phase has a manifest and never needs the registry. - println("--- Generate stdlib environment (candidate A)") - generate_project(julia_a, project_a) - println("--- Generate stdlib environment (baseline B)") - generate_project(julia_b, project_b) - - names_a = stdlib_names(project_a) - names_b = stdlib_names(project_b) - # Run the precompile+load rounds in A/B/B/A/B/A order so machine drift/noise is # spread across both binaries, then take the per-binary minimum of each metric. - println("--- Round 1: precompile + load (A)") - pa1, a1 = measure_round(julia_a, project_a, names_a) - println("--- Round 1: precompile + load (B)") - pb1, b1 = measure_round(julia_b, project_b, names_b) - println("--- Round 2: precompile + load (B)") - pb2, b2 = measure_round(julia_b, project_b, names_b) - println("--- Round 2: precompile + load (A)") - pa2, a2 = measure_round(julia_a, project_a, names_a) - println("--- Round 3: precompile + load (B)") - pb3, b3 = measure_round(julia_b, project_b, names_b) - println("--- Round 3: precompile + load (A)") - pa3, a3 = measure_round(julia_a, project_a, names_a) + println("--- Round 1: precompile + load (A)"); pa1, a1 = measure_round(julia_a) + println("--- Round 1: precompile + load (B)"); pb1, b1 = measure_round(julia_b) + println("--- Round 2: precompile + load (B)"); pb2, b2 = measure_round(julia_b) + println("--- Round 2: precompile + load (A)"); pa2, a2 = measure_round(julia_a) + println("--- Round 3: precompile + load (B)"); pb3, b3 = measure_round(julia_b) + println("--- Round 3: precompile + load (A)"); pa3, a3 = measure_round(julia_a) pre_a = minimum((pa1, pa2, pa3)) pre_b = minimum((pb1, pb2, pb3)) diff --git a/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl b/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl index 0c633ddc..cfbafa53 100644 --- a/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl +++ b/utilities/stdlib_load_time_regression/measure_stdlib_load_times.jl @@ -1,143 +1,112 @@ # Helper invoked (in separate processes) by the orchestrator -# `compare_stdlib_load_times.jl` for one Julia binary at a time. It has three -# modes, run in this order: +# `compare_stdlib_load_times.jl`, once per Julia binary, against a fresh isolated +# depot. It has two modes: # -# generate Run with the binary's *standard* depot. Write a `Project.toml` -# that lists every loadable stdlib as a dependency and `Pkg.resolve` -# it to a `Manifest.toml`. Resolution needs the standard depot (it -# may consult the registry), which is why this is its own phase. +# precompile Precompile every loadable stdlib (dependencies first, so each is +# compiled exactly once) into the depot with `Base.compilecache`, and +# record the total elapsed time. No Pkg/registry/manifest is involved, +# so this never touches the network or the agent's depot. Writes the +# elapsed seconds on the first line of , then the names of +# the stdlibs that were compiled, one per line. # -# precompile Run with a *fresh, isolated* depot (a single, writable path with -# no compiled cache). With the manifest already present, precompile -# the *entire* environment in one parallel batch via -# `Base.Precompilation.precompilepkgs`, building every stdlib's -# pkgimage into the fresh depot. No resolve happens here, so no -# registry is needed. The aggregate precompile time is recorded. -# -# measure Run with the same fresh depot, *one stdlib per process*. Load the -# single named stdlib with `Base.require` and record its time. A -# separate process per stdlib is required because loading one -# stdlib pulls in its dependencies, so measuring many in a single -# session would report later ones as already-resident (ā‰ˆ0s). -# -# Generating the Project/Manifest with the standard depot but precompiling and -# loading against the fresh depot is what forces real (re)precompilation instead -# of reusing the binary's bundled pkgimages. -# -# The measured time is written to the output file as a tab-separated -# `nameseconds` line (`generate` writes nothing). +# measure Load the single named stdlib with `Base.require` (from the warm +# cache the precompile mode just built) and record its time. A +# separate process per stdlib is required because loading one stdlib +# pulls in its dependencies, so a shared session would report later +# stdlibs as already-resident (~0s). Writes a `nameseconds` line. # # Usage: -# julia measure_stdlib_load_times.jl generate -# julia measure_stdlib_load_times.jl precompile -# julia measure_stdlib_load_times.jl measure - -const PKG_UUID = Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") +# julia measure_stdlib_load_times.jl precompile +# julia measure_stdlib_load_times.jl measure - -# Return `name => uuid` for a stdlib that can be added as a project dependency, -# or `nothing` if it has no `Project.toml`/`uuid` (and so can't be precompiled as -# an ordinary package). We scan the TOML by hand to avoid loading the `TOML` -# stdlib (which would itself perturb the timings). -function stdlib_name_uuid(dir, name) - proj = joinpath(dir, name, "Project.toml") - isfile(proj) || return nothing +# Parse a stdlib's `Project.toml` for its uuid and the names listed under `[deps]`. +# We scan the TOML by hand to avoid loading the `TOML` stdlib (which would itself +# perturb the timings). +function read_project(proj) uuid = nothing + deps = String[] + in_deps = false for line in eachline(proj) - m = match(r"^\s*uuid\s*=\s*\"([^\"]+)\"", line) - m === nothing || (uuid = m.captures[1]) + s = strip(line) + if s == "[deps]" + in_deps = true + elseif startswith(s, "[") + in_deps = false + elseif in_deps + m = match(r"^(\S+)\s*=", s) + m === nothing || push!(deps, String(m.captures[1])) + else + m = match(r"^uuid\s*=\s*\"([^\"]+)\"", s) + m === nothing || (uuid = String(m.captures[1])) + end end - uuid === nothing && return nothing - return name => uuid + return uuid, deps end -# All stdlibs (sorted) that expose a `uuid`, as `name => uuid` pairs. +# Every loadable stdlib (one that exposes a uuid) as `name => (uuid, dep_names)`. function loadable_stdlibs() - dir = Sys.STDLIB - deps = Pair{String,String}[] - for name in sort(readdir(dir)) - isdir(joinpath(dir, name)) || continue - nu = stdlib_name_uuid(dir, name) - nu === nothing || push!(deps, nu) + stdlibs = Dict{String,Tuple{String,Vector{String}}}() + for name in readdir(Sys.STDLIB) + proj = joinpath(Sys.STDLIB, name, "Project.toml") + isfile(proj) || continue + uuid, deps = read_project(proj) + uuid === nothing || (stdlibs[name] = (uuid, deps)) end - return deps + return stdlibs end -# Write a `Project.toml` whose `[deps]` lists every loadable stdlib, so that -# activating it turns the stdlibs into ordinary, precompilable dependencies. -function write_project(project_dir, deps) - mkpath(project_dir) - open(joinpath(project_dir, "Project.toml"), "w") do io - println(io, "[deps]") - for (name, uuid) in deps - println(io, name, " = \"", uuid, "\"") - end +# Precompile every stdlib into the (fresh) depot, dependencies first so that each +# is compiled exactly once. Returns `(elapsed_seconds, sorted_names)`. +function precompile_all() + stdlibs = loadable_stdlibs() + done = Set{String}() + function precompile_one(name) + (name in done || !haskey(stdlibs, name)) && return + push!(done, name) + uuid, deps = stdlibs[name] + foreach(precompile_one, deps) + Base.compilecache(Base.PkgId(Base.UUID(uuid), name), devnull, devnull) end - return joinpath(project_dir, "Project.toml") + names = sort!(collect(keys(stdlibs))) + elapsed = @elapsed foreach(precompile_one, names) + return elapsed, names end -# generate phase: write the Project.toml and resolve it to a Manifest.toml using -# the (standard) depot this process was started with. -function generate(project_dir) - deps = loadable_stdlibs() - write_project(project_dir, deps) - Pkg = Base.require(Base.PkgId(PKG_UUID, "Pkg")) - Base.invokelatest(Pkg.activate, project_dir; io = devnull) - Base.invokelatest(Pkg.resolve; io = devnull) - return nothing -end - -# precompile phase: with the Manifest already present, precompile the whole -# environment at once (parallel batch) into the fresh depot. -function precompile_all(project_file) - Base.set_active_project(project_file) - return @elapsed Base.Precompilation.precompilepkgs(; warn_loaded = false) -end - -# measure phase: load the single named stdlib (in this fresh process), timing it -# from the warm cache. +# Load the single named stdlib, timing it from the warm cache. function measure_one(name) - uuid = nothing - for (n, u) in loadable_stdlibs() - n == name && (uuid = u; break) - end - uuid === nothing && error("Unknown stdlib: $(name)") + stdlibs = loadable_stdlibs() + haskey(stdlibs, name) || error("Unknown stdlib: $(name)") + uuid, _ = stdlibs[name] return @elapsed Base.require(Base.PkgId(Base.UUID(uuid), name)) end +function usage_error() + println(stderr, """ + Usage: + julia measure_stdlib_load_times.jl precompile + julia measure_stdlib_load_times.jl measure """) + exit(2) +end + function main() - length(ARGS) >= 3 || usage_error() - mode, out_file, project_dir = ARGS[1], ARGS[2], ARGS[3] - project_file = joinpath(project_dir, "Project.toml") + length(ARGS) >= 2 || usage_error() + mode, out_file = ARGS[1], ARGS[2] - if mode == "generate" - generate(project_dir) - return - elseif mode == "precompile" - timings = ["__precompile__" => precompile_all(project_file)] + if mode == "precompile" + elapsed, names = precompile_all() + open(out_file, "w") do io + println(io, elapsed) + foreach(n -> println(io, n), names) + end elseif mode == "measure" - length(ARGS) == 4 || usage_error() - name = ARGS[4] - Base.set_active_project(project_file) - timings = [name => measure_one(name)] + length(ARGS) == 3 || usage_error() + t = measure_one(ARGS[3]) + open(out_file, "w") do io + println(io, ARGS[3], '\t', t) + end else usage_error() end - - open(out_file, "w") do io - for (name, t) in timings - println(io, name, '\t', t) - end - end -end - -function usage_error() - println(stderr, """ - Usage: - julia measure_stdlib_load_times.jl generate - julia measure_stdlib_load_times.jl precompile - julia measure_stdlib_load_times.jl measure """) - exit(2) end main()