diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 9608319..b35d7a5 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -20,39 +20,5 @@ concurrency: jobs: tests: - name: "Tests" - strategy: - fail-fast: false - matrix: - version: - - "1" - - "lts" - - "pre" - os: - - "ubuntu-latest" - - "macos-latest" - - "windows-latest" - uses: "SciML/.github/.github/workflows/tests.yml@v1" - with: - julia-version: "${{ matrix.version }}" - os: "${{ matrix.os }}" + uses: "SciML/.github/.github/workflows/grouped-tests.yml@v1" secrets: "inherit" - - alloccheck: - name: "AllocCheck" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: julia-actions/setup-julia@v3 - with: - version: "1" - - uses: julia-actions/cache@v3 - - name: Run allocation tests - run: | - julia --project -e ' - using Pkg - Pkg.instantiate() - Pkg.test() - ' - env: - GROUP: nopre diff --git a/Project.toml b/Project.toml index a94ae4f..46e9cd7 100644 --- a/Project.toml +++ b/Project.toml @@ -8,6 +8,7 @@ AllocCheck = "0.2" Aqua = "0.8" JET = "0.9, 0.10, 0.11" LinearAlgebra = "1.10" +Pkg = "1.10" SafeTestsets = "0.1" Statistics = "1.10" Test = "1.10" @@ -18,9 +19,10 @@ AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["AllocCheck", "Aqua", "JET", "LinearAlgebra", "SafeTestsets", "Statistics", "Test"] +test = ["AllocCheck", "Aqua", "JET", "LinearAlgebra", "Pkg", "SafeTestsets", "Statistics", "Test"] diff --git a/test/core_tests.jl b/test/core_tests.jl new file mode 100644 index 0000000..c0ef5bf --- /dev/null +++ b/test/core_tests.jl @@ -0,0 +1,115 @@ +using SurrogatesBase + +using Test +using LinearAlgebra +import Statistics + +struct DummySurrogate{X, Y} <: AbstractDeterministicSurrogate + xs::Vector{X} + ys::Vector{Y} +end +# return y value of the closest ξ in xs to x +(s::DummySurrogate)(x) = s.ys[argmin(norm(x - ξ) for ξ in s.xs)] +function SurrogatesBase.update!(s::DummySurrogate, new_xs, new_ys) + append!(s.xs, new_xs) + return append!(s.ys, new_ys) +end + +mutable struct HyperparameterDummySurrogate{X, Y} <: AbstractDeterministicSurrogate + xs::Vector{X} + ys::Vector{Y} + θ::NamedTuple +end +# return y value of the closest ξ in xs to x, in p-norm where p is a hyperparameter +(s::HyperparameterDummySurrogate)(x) = s.ys[argmin(norm(x - ξ, s.θ.p) for ξ in s.xs)] +function SurrogatesBase.update!(s::HyperparameterDummySurrogate, new_xs, new_ys) + append!(s.xs, new_xs) + return append!(s.ys, new_ys) +end + +SurrogatesBase.hyperparameters(s::HyperparameterDummySurrogate) = s.θ + +function SurrogatesBase.update_hyperparameters!(s::HyperparameterDummySurrogate, prior) + # "hyperparmeter optimization" + return s.θ = (; p = (s.θ.p + prior.p) / 2) +end + +@testset "update!" begin + # use DummySurrogate + d = DummySurrogate(Vector{Vector{Float64}}(), Vector{Int}()) + update!(d, [[10.3, 0.1], [1.9, 2.1]], [5, 6]) + update!(d, [[-0.3, 9.9], [-0.1, -10.0]], [1, 3]) + @test length(d.xs) == 4 + @test d([0.0, -9.9]) == 3 +end + +@testset "default implementations" begin + # use DummySurrogate + d = DummySurrogate(Vector{Vector{Float64}}(), Vector{Float64}()) + update!(d, [[1.9, 2.1]], [5.0]) + update!(d, [[10.3, 0.1]], [9.0]) + + @test d([2.0, 2.0]) == 5.0 + @test_throws MethodError hyperparameters(d) + @test_throws MethodError update_hyperparameters!(d, 5) +end + +@testset "hyperparameter interface" begin + # use HyperparameterDummySurrogate + hd = HyperparameterDummySurrogate( + Vector{Vector{Float64}}(), + Vector{Float64}(), + (; p = 2) + ) + update!(hd, [[1.9, 2.1], [10.3, 0.1]], [5.0, 9.0]) + + @test hyperparameters(hd).p == 2 + update_hyperparameters!(hd, (; p = 4)) + @test hyperparameters(hd).p == 3 +end + +mutable struct DummyStochasticSurrogate{X, Y} <: AbstractStochasticSurrogate + xs::Vector{X} + ys::Vector{Y} + ys_mean::Y +end +function SurrogatesBase.update!(s::DummyStochasticSurrogate, new_xs, new_ys) + append!(s.xs, new_xs) + append!(s.ys, new_ys) + # update mean + return s.ys_mean = (s.ys_mean * (length(s.xs) - length(new_xs)) + sum(new_ys)) / length(s.xs) +end + +SurrogatesBase.parameters(s::DummyStochasticSurrogate) = s.ys_mean + +struct FiniteDummyStochasticSurrogate{X} + means::Vector{X} +end +Statistics.mean(s::FiniteDummyStochasticSurrogate) = s.means + +# xs are arbitrary points where we wish to get a joint posterior +function FiniteDummyStochasticSurrogate(s, xs) + return FiniteDummyStochasticSurrogate(s.ys_mean .* ones(length(xs))) +end + +function SurrogatesBase.finite_posterior(s::DummyStochasticSurrogate, xs) + return FiniteDummyStochasticSurrogate(s, xs) +end + +@testset "finite_posterior, parameters" begin + # use HyperparameterDummySurrogate + ss = DummyStochasticSurrogate( + Vector{Vector{Float64}}(), + Vector{Float64}(), 0.0 + ) + + update!(ss, [[1.9, 2.1], [10.3, 0.1]], [5.0, 9.0]) + # test parameters + @test parameters(ss) ≈ 7.0 + update!(ss, [[2.0, 4.0]], [3.0]) + @test parameters(ss) ≈ 17 / 3 + + m = Statistics.mean(finite_posterior(ss, [[3.5, 2.0], [4.0, 5.0], [1.0, 67.0]])) + @test length(m) == 3 + @test m[1] ≈ parameters(ss) +end diff --git a/test/jet.jl b/test/jet.jl deleted file mode 100644 index 1a9342d..0000000 --- a/test/jet.jl +++ /dev/null @@ -1,97 +0,0 @@ -using SurrogatesBase -using JET -using LinearAlgebra -import Statistics - -# On Julia 1.12+, LinearAlgebra.norm_recursive_check has a type inference issue -# that causes JET false positives. We ignore LinearAlgebra and Base modules -# to filter these stdlib issues while still checking our own code. -const JET_CONFIG = ( - ignored_modules = ( - JET.AnyFrameModule(LinearAlgebra), - JET.AnyFrameModule(Base), - ), -) - -@testset "JET static analysis" begin - @testset "Package analysis" begin - result = JET.report_package(SurrogatesBase; JET_CONFIG...) - @test length(JET.get_reports(result)) == 0 - end - - @testset "DummySurrogate type stability" begin - # Test implementation from runtests.jl - struct JETDummySurrogate{X, Y} <: AbstractDeterministicSurrogate - xs::Vector{X} - ys::Vector{Y} - end - (s::JETDummySurrogate)(x) = s.ys[argmin([norm(x - ξ) for ξ in s.xs])] - function SurrogatesBase.update!(s::JETDummySurrogate, new_xs, new_ys) - append!(s.xs, new_xs) - append!(s.ys, new_ys) - end - - d = JETDummySurrogate(Vector{Vector{Float64}}(), Vector{Int}()) - SurrogatesBase.update!(d, [[10.3, 0.1], [1.9, 2.1]], [5, 6]) - - # Test call method - result = JET.report_call(d, Tuple{Vector{Float64}}; JET_CONFIG...) - @test length(JET.get_reports(result)) == 0 - - # Test update! method - result = JET.report_call( - SurrogatesBase.update!, - Tuple{ - JETDummySurrogate{Vector{Float64}, Int}, Vector{Vector{Float64}}, - Vector{Int}, - }; JET_CONFIG... - ) - @test length(JET.get_reports(result)) == 0 - end - - @testset "Stochastic surrogate type stability" begin - mutable struct JETDummyStochasticSurrogate{X, Y} <: AbstractStochasticSurrogate - xs::Vector{X} - ys::Vector{Y} - ys_mean::Y - end - function SurrogatesBase.update!(s::JETDummyStochasticSurrogate, new_xs, new_ys) - append!(s.xs, new_xs) - append!(s.ys, new_ys) - s.ys_mean = (s.ys_mean * (length(s.xs) - length(new_xs)) + sum(new_ys)) / - length(s.xs) - end - SurrogatesBase.parameters(s::JETDummyStochasticSurrogate) = s.ys_mean - - struct JETFiniteDummyStochasticSurrogate{X} - means::Vector{X} - end - Statistics.mean(s::JETFiniteDummyStochasticSurrogate) = s.means - function JETFiniteDummyStochasticSurrogate(s, xs) - return JETFiniteDummyStochasticSurrogate(s.ys_mean .* ones(length(xs))) - end - function SurrogatesBase.finite_posterior(s::JETDummyStochasticSurrogate, xs) - JETFiniteDummyStochasticSurrogate(s, xs) - end - - # Test update! method - result = JET.report_call( - SurrogatesBase.update!, - Tuple{ - JETDummyStochasticSurrogate{Vector{Float64}, Float64}, - Vector{Vector{Float64}}, Vector{Float64}, - }; JET_CONFIG... - ) - @test length(JET.get_reports(result)) == 0 - - # Test finite_posterior - result = JET.report_call( - SurrogatesBase.finite_posterior, - Tuple{ - JETDummyStochasticSurrogate{Vector{Float64}, Float64}, - Vector{Vector{Float64}}, - }; JET_CONFIG... - ) - @test length(JET.get_reports(result)) == 0 - end -end diff --git a/test/qa.jl b/test/qa.jl index 877cf78..81692d9 100644 --- a/test/qa.jl +++ b/test/qa.jl @@ -1,4 +1,10 @@ -using SurrogatesBase, Aqua +using SurrogatesBase +using Aqua +using JET +using LinearAlgebra +using Test +import Statistics + @testset "Aqua" begin Aqua.find_persistent_tasks_deps(SurrogatesBase) Aqua.test_ambiguities(SurrogatesBase, recursive = false) @@ -9,3 +15,96 @@ using SurrogatesBase, Aqua Aqua.test_unbound_args(SurrogatesBase) Aqua.test_undefined_exports(SurrogatesBase) end + +# On Julia 1.12+, LinearAlgebra.norm_recursive_check has a type inference issue +# that causes JET false positives. We ignore LinearAlgebra and Base modules +# to filter these stdlib issues while still checking our own code. +const JET_CONFIG = ( + ignored_modules = ( + JET.AnyFrameModule(LinearAlgebra), + JET.AnyFrameModule(Base), + ), +) + +@testset "JET static analysis" begin + @testset "Package analysis" begin + result = JET.report_package(SurrogatesBase; JET_CONFIG...) + @test length(JET.get_reports(result)) == 0 + end + + @testset "DummySurrogate type stability" begin + # Test implementation from runtests.jl + struct JETDummySurrogate{X, Y} <: AbstractDeterministicSurrogate + xs::Vector{X} + ys::Vector{Y} + end + (s::JETDummySurrogate)(x) = s.ys[argmin([norm(x - ξ) for ξ in s.xs])] + function SurrogatesBase.update!(s::JETDummySurrogate, new_xs, new_ys) + append!(s.xs, new_xs) + append!(s.ys, new_ys) + end + + d = JETDummySurrogate(Vector{Vector{Float64}}(), Vector{Int}()) + SurrogatesBase.update!(d, [[10.3, 0.1], [1.9, 2.1]], [5, 6]) + + # Test call method + result = JET.report_call(d, Tuple{Vector{Float64}}; JET_CONFIG...) + @test length(JET.get_reports(result)) == 0 + + # Test update! method + result = JET.report_call( + SurrogatesBase.update!, + Tuple{ + JETDummySurrogate{Vector{Float64}, Int}, Vector{Vector{Float64}}, + Vector{Int}, + }; JET_CONFIG... + ) + @test length(JET.get_reports(result)) == 0 + end + + @testset "Stochastic surrogate type stability" begin + mutable struct JETDummyStochasticSurrogate{X, Y} <: AbstractStochasticSurrogate + xs::Vector{X} + ys::Vector{Y} + ys_mean::Y + end + function SurrogatesBase.update!(s::JETDummyStochasticSurrogate, new_xs, new_ys) + append!(s.xs, new_xs) + append!(s.ys, new_ys) + s.ys_mean = (s.ys_mean * (length(s.xs) - length(new_xs)) + sum(new_ys)) / + length(s.xs) + end + SurrogatesBase.parameters(s::JETDummyStochasticSurrogate) = s.ys_mean + + struct JETFiniteDummyStochasticSurrogate{X} + means::Vector{X} + end + Statistics.mean(s::JETFiniteDummyStochasticSurrogate) = s.means + function JETFiniteDummyStochasticSurrogate(s, xs) + return JETFiniteDummyStochasticSurrogate(s.ys_mean .* ones(length(xs))) + end + function SurrogatesBase.finite_posterior(s::JETDummyStochasticSurrogate, xs) + JETFiniteDummyStochasticSurrogate(s, xs) + end + + # Test update! method + result = JET.report_call( + SurrogatesBase.update!, + Tuple{ + JETDummyStochasticSurrogate{Vector{Float64}, Float64}, + Vector{Vector{Float64}}, Vector{Float64}, + }; JET_CONFIG... + ) + @test length(JET.get_reports(result)) == 0 + + # Test finite_posterior + result = JET.report_call( + SurrogatesBase.finite_posterior, + Tuple{ + JETDummyStochasticSurrogate{Vector{Float64}, Float64}, + Vector{Vector{Float64}}, + }; JET_CONFIG... + ) + @test length(JET.get_reports(result)) == 0 + end +end diff --git a/test/qa/Project.toml b/test/qa/Project.toml new file mode 100644 index 0000000..7b8da5f --- /dev/null +++ b/test/qa/Project.toml @@ -0,0 +1,18 @@ +[deps] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +SurrogatesBase = "89f642e6-4179-4274-8202-c11f4bd9a72c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[sources] +SurrogatesBase = {path = "../.."} + +[compat] +Aqua = "0.8" +JET = "0.9, 0.10, 0.11" +LinearAlgebra = "1.10" +Statistics = "1.10" +Test = "1.10" +julia = "1.10" diff --git a/test/runtests.jl b/test/runtests.jl index 64f499a..5e77d88 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,124 +1,19 @@ -using SurrogatesBase - -using Test +using Pkg using SafeTestsets -@safetestset "Quality Assurance" include("qa.jl") -@safetestset "JET Static Analysis" include("jet.jl") - -using LinearAlgebra -import Statistics - -struct DummySurrogate{X, Y} <: AbstractDeterministicSurrogate - xs::Vector{X} - ys::Vector{Y} -end -# return y value of the closest ξ in xs to x -(s::DummySurrogate)(x) = s.ys[argmin(norm(x - ξ) for ξ in s.xs)] -function SurrogatesBase.update!(s::DummySurrogate, new_xs, new_ys) - append!(s.xs, new_xs) - return append!(s.ys, new_ys) -end - -mutable struct HyperparameterDummySurrogate{X, Y} <: AbstractDeterministicSurrogate - xs::Vector{X} - ys::Vector{Y} - θ::NamedTuple -end -# return y value of the closest ξ in xs to x, in p-norm where p is a hyperparameter -(s::HyperparameterDummySurrogate)(x) = s.ys[argmin(norm(x - ξ, s.θ.p) for ξ in s.xs)] -function SurrogatesBase.update!(s::HyperparameterDummySurrogate, new_xs, new_ys) - append!(s.xs, new_xs) - return append!(s.ys, new_ys) -end - -SurrogatesBase.hyperparameters(s::HyperparameterDummySurrogate) = s.θ - -function SurrogatesBase.update_hyperparameters!(s::HyperparameterDummySurrogate, prior) - # "hyperparmeter optimization" - return s.θ = (; p = (s.θ.p + prior.p) / 2) -end - -@testset "update!" begin - # use DummySurrogate - d = DummySurrogate(Vector{Vector{Float64}}(), Vector{Int}()) - update!(d, [[10.3, 0.1], [1.9, 2.1]], [5, 6]) - update!(d, [[-0.3, 9.9], [-0.1, -10.0]], [1, 3]) - @test length(d.xs) == 4 - @test d([0.0, -9.9]) == 3 -end - -@testset "default implementations" begin - # use DummySurrogate - d = DummySurrogate(Vector{Vector{Float64}}(), Vector{Float64}()) - update!(d, [[1.9, 2.1]], [5.0]) - update!(d, [[10.3, 0.1]], [9.0]) - - @test d([2.0, 2.0]) == 5.0 - @test_throws MethodError hyperparameters(d) - @test_throws MethodError update_hyperparameters!(d, 5) -end - -@testset "hyperparameter interface" begin - # use HyperparameterDummySurrogate - hd = HyperparameterDummySurrogate( - Vector{Vector{Float64}}(), - Vector{Float64}(), - (; p = 2) - ) - update!(hd, [[1.9, 2.1], [10.3, 0.1]], [5.0, 9.0]) - - @test hyperparameters(hd).p == 2 - update_hyperparameters!(hd, (; p = 4)) - @test hyperparameters(hd).p == 3 -end - -mutable struct DummyStochasticSurrogate{X, Y} <: AbstractStochasticSurrogate - xs::Vector{X} - ys::Vector{Y} - ys_mean::Y -end -function SurrogatesBase.update!(s::DummyStochasticSurrogate, new_xs, new_ys) - append!(s.xs, new_xs) - append!(s.ys, new_ys) - # update mean - return s.ys_mean = (s.ys_mean * (length(s.xs) - length(new_xs)) + sum(new_ys)) / length(s.xs) -end - -SurrogatesBase.parameters(s::DummyStochasticSurrogate) = s.ys_mean - -struct FiniteDummyStochasticSurrogate{X} - means::Vector{X} -end -Statistics.mean(s::FiniteDummyStochasticSurrogate) = s.means - -# xs are arbitrary points where we wish to get a joint posterior -function FiniteDummyStochasticSurrogate(s, xs) - return FiniteDummyStochasticSurrogate(s.ys_mean .* ones(length(xs))) -end - -function SurrogatesBase.finite_posterior(s::DummyStochasticSurrogate, xs) - return FiniteDummyStochasticSurrogate(s, xs) -end - -@testset "finite_posterior, parameters" begin - # use HyperparameterDummySurrogate - ss = DummyStochasticSurrogate( - Vector{Vector{Float64}}(), - Vector{Float64}(), 0.0 - ) +using Test - update!(ss, [[1.9, 2.1], [10.3, 0.1]], [5.0, 9.0]) - # test parameters - @test parameters(ss) ≈ 7.0 - update!(ss, [[2.0, 4.0]], [3.0]) - @test parameters(ss) ≈ 17 / 3 +const GROUP = get(ENV, "GROUP", "All") - m = Statistics.mean(finite_posterior(ss, [[3.5, 2.0], [4.0, 5.0], [1.0, 67.0]])) - @test length(m) == 3 - @test m[1] ≈ parameters(ss) -end +@testset "SurrogatesBase" begin + if GROUP == "QA" + Pkg.activate(joinpath(@__DIR__, "qa")) + Pkg.develop(path = joinpath(@__DIR__, "..")) + Pkg.instantiate() + @safetestset "Quality Assurance" include("qa.jl") + end -# Run allocation tests in nopre group to avoid precompilation interference -if get(ENV, "GROUP", "all") == "all" || get(ENV, "GROUP", "all") == "nopre" - @safetestset "Allocation Tests" include("alloc_tests.jl") + if GROUP == "All" || GROUP == "Core" + @safetestset "Core" include("core_tests.jl") + @safetestset "Allocation Tests" include("alloc_tests.jl") + end end diff --git a/test/test_groups.toml b/test/test_groups.toml new file mode 100644 index 0000000..c7ec044 --- /dev/null +++ b/test/test_groups.toml @@ -0,0 +1,6 @@ +[Core] +versions = ["lts", "1", "pre"] +os = ["ubuntu-latest", "macos-latest", "windows-latest"] + +[QA] +versions = ["lts", "1"]