Skip to content

Develop URL sources in monorepo CI helper #112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChrisRackauckas merged 1 commit into from
Jul 12, 2026
Merged

Develop URL sources in monorepo CI helper #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 57 additions & 17 deletions scripts/develop_sources.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
#!/usr/bin/env julia
#
# Develop the in-repo [sources] path deps of a sub-project.
# Develop the [sources] deps of a sub-project.
#
# On Julia < 1.11 the [sources] table is ignored when an environment is
# resolved/built/tested, so a monorepo sublibrary (e.g. lib/<name>) that relies
# on [sources] to pin its in-repo siblings would otherwise resolve them as
# registered packages. This script restores the 1.11+ behavior on 1.10 (the
# SciML LTS) by Pkg.develop-ing each in-repo `path =` source by path. On Julia
# >= 1.11 it is a no-op (the table is honored natively).
# SciML LTS) by Pkg.develop-ing each `path =` or `url =` source. On Julia >=
# 1.11 it is a no-op (the table is honored natively).
#
# The walk is transitive: a developed source dep can itself declare further
# *runtime* [sources] that must also be developed for the environment to load.
Expand All @@ -25,9 +25,9 @@
# include("scripts/develop_sources.jl")
# develop_sources(project_dir)
#
# `develop_sources` activates `project_dir`, computes the source paths to
# develop via `collect_source_paths`, and Pkg.develop-s them. The pure
# path-collection logic is split out so it can be unit-tested without mutating
# `develop_sources` activates `project_dir`, computes the source specs to
# develop via `collect_source_specs`, and Pkg.develop-s them. The pure
# source-collection logic is split out so it can be unit-tested without mutating
# any environment.

using Pkg
Expand All @@ -47,9 +47,40 @@ sources are left to `Pkg`. Visiting each resolved path once handles cycles and
diamonds in the graph. This function is version-independent and mutates nothing.
"""
function collect_source_paths(proj::AbstractString)
return first(_collect_source_paths_and_specs(proj))
end

"""
collect_source_specs(proj) -> Vector{Pkg.PackageSpec}

Walk the `[sources]` graph rooted at `proj` and return the ordered
`Pkg.PackageSpec`s to develop on Julia versions that do not natively honor
`[sources]`. Unlike `collect_source_paths`, this includes both local `path =`
sources and git `url =` sources, preserving `rev` and `subdir` when present.
"""
function collect_source_specs(proj::AbstractString)
return last(_collect_source_paths_and_specs(proj))
end

function _source_url_key(dep::AbstractString, spec::AbstractDict)
url = String(spec["url"])
rev = String(get(spec, "rev", ""))
subdir = String(get(spec, "subdir", ""))
return "url:$dep:$url:$rev:$subdir"
end

function _source_url_spec(dep::AbstractString, spec::AbstractDict)
kwargs = Pair{Symbol, Any}[:name => dep, :url => String(spec["url"])]
haskey(spec, "rev") && push!(kwargs, :rev => String(spec["rev"]))
haskey(spec, "subdir") && push!(kwargs, :subdir => String(spec["subdir"]))
return Pkg.PackageSpec(; kwargs...)
end

function _collect_source_paths_and_specs(proj::AbstractString)
projroot = normpath(abspath(proj))
developed = Set{String}([projroot]) # never develop the active project
paths = String[]
specs = Pkg.PackageSpec[]
queue = String[projroot]
while !isempty(queue)
dir = popfirst!(queue)
Expand All @@ -65,29 +96,38 @@ function collect_source_paths(proj::AbstractString)
# are not runtime deps -- those are the dep's test-only sources and
# must not leak into the active environment.
isroot || dep in runtimedeps || continue
spec isa AbstractDict && haskey(spec, "path") || continue
p = normpath(abspath(joinpath(dir, spec["path"])))
if isdir(p) && !(p in developed)
push!(developed, p)
push!(paths, p)
push!(queue, p) # resolve this dep's own runtime [sources] too
spec isa AbstractDict || continue
if haskey(spec, "path")
p = normpath(abspath(joinpath(dir, spec["path"])))
if isdir(p) && !(p in developed)
push!(developed, p)
push!(paths, p)
push!(specs, Pkg.PackageSpec(path = p))
push!(queue, p) # resolve this dep's own runtime [sources] too
end
elseif haskey(spec, "url")
key = _source_url_key(dep, spec)
if !(key in developed)
push!(developed, key)
push!(specs, _source_url_spec(dep, spec))
end
end
end
end
return paths
return paths, specs
end

"""
develop_sources(proj)

Activate `proj` and, on Julia < 1.11, `Pkg.develop` its in-repo `[sources]`
path deps (see `collect_source_paths`). No-op on Julia >= 1.11.
Activate `proj` and, on Julia < 1.11, `Pkg.develop` its `[sources]` deps
(see `collect_source_specs`). No-op on Julia >= 1.11.
"""
function develop_sources(proj::AbstractString)
Pkg.activate(proj)
VERSION < v"1.11.0-DEV.0" || return nothing
paths = collect_source_paths(proj)
isempty(paths) || Pkg.develop([Pkg.PackageSpec(path = p) for p in paths])
specs = collect_source_specs(proj)
isempty(specs) || Pkg.develop(specs)
return nothing
end

Expand Down
72 changes: 67 additions & 5 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ using Test
const SCRIPT = joinpath(@__DIR__, "..", "scripts", "compute_affected_sublibraries.jl")
include(SCRIPT)

# Load the [sources] develop helper's pure collection functions.
const DEVELOP_SCRIPT = joinpath(@__DIR__, "..", "scripts", "develop_sources.jl")
include(DEVELOP_SCRIPT)

# Build a fixture monorepo `lib/` tree in a temp dir.
# A (base, no internal deps)
# B deps A
Expand All @@ -21,7 +25,7 @@ function make_fixture(; c_groups::Union{Nothing, String} = nothing)
write(joinpath(d, "Project.toml"), "name = \"$name\"\nuuid = \"00000000-0000-0000-0000-0000000000$(lpad(hash(name) % 100, 2, '0'))\"\n$depblock")
write(joinpath(d, "src", "$name.jl"), "module $name\nend\n")
write(joinpath(d, "test", "runtests.jl"), "using Test\n")
groups === nothing || write(joinpath(d, "test", "test_groups.toml"), groups)
return groups === nothing || write(joinpath(d, "test", "test_groups.toml"), groups)
end
pkg("A", String[])
pkg("B", ["A"])
Expand All @@ -30,6 +34,64 @@ function make_fixture(; c_groups::Union{Nothing, String} = nothing)
return root
end

@testset "develop_sources: path and URL source collection" begin
root = mktempdir()
localdep = joinpath(root, "lib", "LocalDep")
nested = joinpath(root, "lib", "NestedRuntime")
mkpath(localdep)
mkpath(nested)
write(
joinpath(root, "Project.toml"), """
name = "Root"
uuid = "00000000-0000-0000-0000-000000000001"

[deps]
LocalDep = "00000000-0000-0000-0000-000000000002"
UrlDep = "00000000-0000-0000-0000-000000000003"

[extras]
TestOnlyUrl = "00000000-0000-0000-0000-000000000004"

[sources]
LocalDep = {path = "lib/LocalDep"}
UrlDep = {url = "https://example.com/runtime.git", rev = "main"}
TestOnlyUrl = {url = "https://example.com/test-only.git"}
"""
)
write(
joinpath(localdep, "Project.toml"), """
name = "LocalDep"
uuid = "00000000-0000-0000-0000-000000000002"

[deps]
NestedRuntime = "00000000-0000-0000-0000-000000000005"

[extras]
NestedTestOnly = "00000000-0000-0000-0000-000000000006"

[sources]
NestedRuntime = {path = "../NestedRuntime"}
NestedTestOnly = {url = "https://example.com/nested-test-only.git"}
"""
)
write(
joinpath(nested, "Project.toml"), """
name = "NestedRuntime"
uuid = "00000000-0000-0000-0000-000000000005"
"""
)

paths = collect_source_paths(root)
@test paths == [normpath(localdep), normpath(nested)]

specs = collect_source_specs(root)
@test Set(filter(!isnothing, getfield.(specs, :path))) == Set(paths)
url_specs = filter(s -> !isnothing(s.url), specs)
@test any(s -> s.name == "UrlDep" && s.url == "https://example.com/runtime.git" && s.rev == "main", url_specs)
@test any(s -> s.name == "TestOnlyUrl" && s.url == "https://example.com/test-only.git", url_specs)
@test !any(s -> s.name == "NestedTestOnly", url_specs)
end

@testset "compute_affected_sublibraries" begin
root = make_fixture()
lib = joinpath(root, "lib")
Expand Down Expand Up @@ -85,7 +147,7 @@ end
# A is directly changed: default Core on lts,1,pre + QA on 1 (QA defaults to v1 only)
a = filter(e -> e.project == "lib/A", m)
@test Set((e.group, e.version) for e in a) ==
Set([("Core", "lts"), ("Core", "1"), ("Core", "pre"), ("QA", "1")])
Set([("Core", "lts"), ("Core", "1"), ("Core", "pre"), ("QA", "1")])
# B and C are downstream: version "1" only
for p in ("lib/B", "lib/C")
ds = filter(e -> e.project == p, m)
Expand Down Expand Up @@ -175,7 +237,7 @@ end
d = mktempdir()
# No test/test_groups.toml -> single Core group on the standard set.
@test Set((e.group, e.version) for e in build_root_matrix(d)) ==
Set([("Core", "lts"), ("Core", "1"), ("Core", "pre")])
Set([("Core", "lts"), ("Core", "1"), ("Core", "pre")])
@test all(e -> e.continue_on_error == false, build_root_matrix(d))
# The default matrix runner is self-hosted-capable `ubuntu-latest` (the
# SciML demeter*/arctic* pool answers that label). Only legs that set
Expand Down Expand Up @@ -242,7 +304,7 @@ end
core = filter(e -> e.group == "Core", m)
@test length(core) == 6
@test Set((e.version, e.runner) for e in core) ==
Set((v, o) for v in ["lts", "1"] for o in ["ubuntu-latest", "windows-latest", "macos-latest"])
Set((v, o) for v in ["lts", "1"] for o in ["ubuntu-latest", "windows-latest", "macos-latest"])
# QA: no os -> single default ubuntu runner.
qa = filter(e -> e.group == "QA", m)
@test length(qa) == 1 && only(qa).runner == "ubuntu-latest"
Expand Down Expand Up @@ -344,7 +406,7 @@ end
# GPU / explicit self-hosted runner override is preserved, NOT forced to
# ubuntu-24.04.
@test resolve(apt = "", container = "", default = ["self-hosted", "Linux", "X64", "gpu"]) ==
["self-hosted", "Linux", "X64", "gpu"]
["self-hosted", "Linux", "X64", "gpu"]
end
end

Expand Down
Loading