diff --git a/scripts/develop_sources.jl b/scripts/develop_sources.jl index 76f895c..e1f46f4 100644 --- a/scripts/develop_sources.jl +++ b/scripts/develop_sources.jl @@ -1,13 +1,13 @@ #!/usr/bin/env julia # -# Develop the in-repo [sources] path deps of a sub-project. +# Develop the [sources] deps of a sub-project. # # On Julia < 1.11 the [sources] table is ignored when an environment is # resolved/built/tested, so a monorepo sublibrary (e.g. lib/) that relies # on [sources] to pin its in-repo siblings would otherwise resolve them as # registered packages. This script restores the 1.11+ behavior on 1.10 (the -# SciML LTS) by Pkg.develop-ing each in-repo `path =` source by path. On Julia -# >= 1.11 it is a no-op (the table is honored natively). +# SciML LTS) by Pkg.develop-ing each `path =` or `url =` source. On Julia >= +# 1.11 it is a no-op (the table is honored natively). # # The walk is transitive: a developed source dep can itself declare further # *runtime* [sources] that must also be developed for the environment to load. @@ -25,9 +25,9 @@ # include("scripts/develop_sources.jl") # develop_sources(project_dir) # -# `develop_sources` activates `project_dir`, computes the source paths to -# develop via `collect_source_paths`, and Pkg.develop-s them. The pure -# path-collection logic is split out so it can be unit-tested without mutating +# `develop_sources` activates `project_dir`, computes the source specs to +# develop via `collect_source_specs`, and Pkg.develop-s them. The pure +# source-collection logic is split out so it can be unit-tested without mutating # any environment. using Pkg @@ -47,9 +47,40 @@ sources are left to `Pkg`. Visiting each resolved path once handles cycles and diamonds in the graph. This function is version-independent and mutates nothing. """ function collect_source_paths(proj::AbstractString) + return first(_collect_source_paths_and_specs(proj)) +end + +""" + collect_source_specs(proj) -> Vector{Pkg.PackageSpec} + +Walk the `[sources]` graph rooted at `proj` and return the ordered +`Pkg.PackageSpec`s to develop on Julia versions that do not natively honor +`[sources]`. Unlike `collect_source_paths`, this includes both local `path =` +sources and git `url =` sources, preserving `rev` and `subdir` when present. +""" +function collect_source_specs(proj::AbstractString) + return last(_collect_source_paths_and_specs(proj)) +end + +function _source_url_key(dep::AbstractString, spec::AbstractDict) + url = String(spec["url"]) + rev = String(get(spec, "rev", "")) + subdir = String(get(spec, "subdir", "")) + return "url:$dep:$url:$rev:$subdir" +end + +function _source_url_spec(dep::AbstractString, spec::AbstractDict) + kwargs = Pair{Symbol, Any}[:name => dep, :url => String(spec["url"])] + haskey(spec, "rev") && push!(kwargs, :rev => String(spec["rev"])) + haskey(spec, "subdir") && push!(kwargs, :subdir => String(spec["subdir"])) + return Pkg.PackageSpec(; kwargs...) +end + +function _collect_source_paths_and_specs(proj::AbstractString) projroot = normpath(abspath(proj)) developed = Set{String}([projroot]) # never develop the active project paths = String[] + specs = Pkg.PackageSpec[] queue = String[projroot] while !isempty(queue) dir = popfirst!(queue) @@ -65,29 +96,38 @@ function collect_source_paths(proj::AbstractString) # are not runtime deps -- those are the dep's test-only sources and # must not leak into the active environment. isroot || dep in runtimedeps || continue - spec isa AbstractDict && haskey(spec, "path") || continue - p = normpath(abspath(joinpath(dir, spec["path"]))) - if isdir(p) && !(p in developed) - push!(developed, p) - push!(paths, p) - push!(queue, p) # resolve this dep's own runtime [sources] too + spec isa AbstractDict || continue + if haskey(spec, "path") + p = normpath(abspath(joinpath(dir, spec["path"]))) + if isdir(p) && !(p in developed) + push!(developed, p) + push!(paths, p) + push!(specs, Pkg.PackageSpec(path = p)) + push!(queue, p) # resolve this dep's own runtime [sources] too + end + elseif haskey(spec, "url") + key = _source_url_key(dep, spec) + if !(key in developed) + push!(developed, key) + push!(specs, _source_url_spec(dep, spec)) + end end end end - return paths + return paths, specs end """ develop_sources(proj) -Activate `proj` and, on Julia < 1.11, `Pkg.develop` its in-repo `[sources]` -path deps (see `collect_source_paths`). No-op on Julia >= 1.11. +Activate `proj` and, on Julia < 1.11, `Pkg.develop` its `[sources]` deps +(see `collect_source_specs`). No-op on Julia >= 1.11. """ function develop_sources(proj::AbstractString) Pkg.activate(proj) VERSION < v"1.11.0-DEV.0" || return nothing - paths = collect_source_paths(proj) - isempty(paths) || Pkg.develop([Pkg.PackageSpec(path = p) for p in paths]) + specs = collect_source_specs(proj) + isempty(specs) || Pkg.develop(specs) return nothing end diff --git a/test/runtests.jl b/test/runtests.jl index 586cfe6..aaf5a4b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,10 @@ using Test const SCRIPT = joinpath(@__DIR__, "..", "scripts", "compute_affected_sublibraries.jl") include(SCRIPT) +# Load the [sources] develop helper's pure collection functions. +const DEVELOP_SCRIPT = joinpath(@__DIR__, "..", "scripts", "develop_sources.jl") +include(DEVELOP_SCRIPT) + # Build a fixture monorepo `lib/` tree in a temp dir. # A (base, no internal deps) # B deps A @@ -21,7 +25,7 @@ function make_fixture(; c_groups::Union{Nothing, String} = nothing) write(joinpath(d, "Project.toml"), "name = \"$name\"\nuuid = \"00000000-0000-0000-0000-0000000000$(lpad(hash(name) % 100, 2, '0'))\"\n$depblock") write(joinpath(d, "src", "$name.jl"), "module $name\nend\n") write(joinpath(d, "test", "runtests.jl"), "using Test\n") - groups === nothing || write(joinpath(d, "test", "test_groups.toml"), groups) + return groups === nothing || write(joinpath(d, "test", "test_groups.toml"), groups) end pkg("A", String[]) pkg("B", ["A"]) @@ -30,6 +34,64 @@ function make_fixture(; c_groups::Union{Nothing, String} = nothing) return root end +@testset "develop_sources: path and URL source collection" begin + root = mktempdir() + localdep = joinpath(root, "lib", "LocalDep") + nested = joinpath(root, "lib", "NestedRuntime") + mkpath(localdep) + mkpath(nested) + write( + joinpath(root, "Project.toml"), """ + name = "Root" + uuid = "00000000-0000-0000-0000-000000000001" + + [deps] + LocalDep = "00000000-0000-0000-0000-000000000002" + UrlDep = "00000000-0000-0000-0000-000000000003" + + [extras] + TestOnlyUrl = "00000000-0000-0000-0000-000000000004" + + [sources] + LocalDep = {path = "lib/LocalDep"} + UrlDep = {url = "https://example.com/runtime.git", rev = "main"} + TestOnlyUrl = {url = "https://example.com/test-only.git"} + """ + ) + write( + joinpath(localdep, "Project.toml"), """ + name = "LocalDep" + uuid = "00000000-0000-0000-0000-000000000002" + + [deps] + NestedRuntime = "00000000-0000-0000-0000-000000000005" + + [extras] + NestedTestOnly = "00000000-0000-0000-0000-000000000006" + + [sources] + NestedRuntime = {path = "../NestedRuntime"} + NestedTestOnly = {url = "https://example.com/nested-test-only.git"} + """ + ) + write( + joinpath(nested, "Project.toml"), """ + name = "NestedRuntime" + uuid = "00000000-0000-0000-0000-000000000005" + """ + ) + + paths = collect_source_paths(root) + @test paths == [normpath(localdep), normpath(nested)] + + specs = collect_source_specs(root) + @test Set(filter(!isnothing, getfield.(specs, :path))) == Set(paths) + url_specs = filter(s -> !isnothing(s.url), specs) + @test any(s -> s.name == "UrlDep" && s.url == "https://example.com/runtime.git" && s.rev == "main", url_specs) + @test any(s -> s.name == "TestOnlyUrl" && s.url == "https://example.com/test-only.git", url_specs) + @test !any(s -> s.name == "NestedTestOnly", url_specs) +end + @testset "compute_affected_sublibraries" begin root = make_fixture() lib = joinpath(root, "lib") @@ -85,7 +147,7 @@ end # A is directly changed: default Core on lts,1,pre + QA on 1 (QA defaults to v1 only) a = filter(e -> e.project == "lib/A", m) @test Set((e.group, e.version) for e in a) == - Set([("Core", "lts"), ("Core", "1"), ("Core", "pre"), ("QA", "1")]) + Set([("Core", "lts"), ("Core", "1"), ("Core", "pre"), ("QA", "1")]) # B and C are downstream: version "1" only for p in ("lib/B", "lib/C") ds = filter(e -> e.project == p, m) @@ -175,7 +237,7 @@ end d = mktempdir() # No test/test_groups.toml -> single Core group on the standard set. @test Set((e.group, e.version) for e in build_root_matrix(d)) == - Set([("Core", "lts"), ("Core", "1"), ("Core", "pre")]) + Set([("Core", "lts"), ("Core", "1"), ("Core", "pre")]) @test all(e -> e.continue_on_error == false, build_root_matrix(d)) # The default matrix runner is self-hosted-capable `ubuntu-latest` (the # SciML demeter*/arctic* pool answers that label). Only legs that set @@ -242,7 +304,7 @@ end core = filter(e -> e.group == "Core", m) @test length(core) == 6 @test Set((e.version, e.runner) for e in core) == - Set((v, o) for v in ["lts", "1"] for o in ["ubuntu-latest", "windows-latest", "macos-latest"]) + Set((v, o) for v in ["lts", "1"] for o in ["ubuntu-latest", "windows-latest", "macos-latest"]) # QA: no os -> single default ubuntu runner. qa = filter(e -> e.group == "QA", m) @test length(qa) == 1 && only(qa).runner == "ubuntu-latest" @@ -344,7 +406,7 @@ end # GPU / explicit self-hosted runner override is preserved, NOT forced to # ubuntu-24.04. @test resolve(apt = "", container = "", default = ["self-hosted", "Linux", "X64", "gpu"]) == - ["self-hosted", "Linux", "X64", "gpu"] + ["self-hosted", "Linux", "X64", "gpu"] end end