Skip to content

Use SciMLTesting v1.1.0 (run_tests harness) #615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChrisRackauckas merged 1 commit into from
Jun 14, 2026
+91 −160
Merged

Use SciMLTesting v1.1.0 (run_tests harness) #615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Project.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ RecipesBase = "1.3.4"
RecursiveArrayToolsShorthandConstructors = "1"
ReverseDiff = "1.15"
SafeTestsets = "0.1"
SciMLTesting = "1"
SparseArrays = "1.10"
StaticArrays = "1.6"
StaticArraysCore = "1.4.2"
Expand All @@ -94,6 +95,7 @@ Polyester = "f517fe37-dbe3-4b94-8317-1923a5111588"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
RecursiveArrayToolsShorthandConstructors = "39fb7555-b4ad-4efd-8abe-30331df017d3"
SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
SciMLTesting = "09d9d899-5365-40a9-917a-5f67fddea283"
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
Expand All @@ -104,4 +106,4 @@ Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"

[targets]
test = ["FastBroadcast", "ForwardDiff", "KernelAbstractions", "Measurements", "Mooncake", "Pkg", "Polyester", "Random", "RecursiveArrayToolsShorthandConstructors", "ReverseDiff", "SafeTestsets", "SparseArrays", "StaticArrays", "Statistics", "StructArrays", "Tables", "Test", "Unitful", "Zygote"]
test = ["FastBroadcast", "ForwardDiff", "KernelAbstractions", "Measurements", "Mooncake", "Pkg", "Polyester", "Random", "RecursiveArrayToolsShorthandConstructors", "ReverseDiff", "SafeTestsets", "SciMLTesting", "SparseArrays", "StaticArrays", "Statistics", "StructArrays", "Tables", "Test", "Unitful", "Zygote"]
247 changes: 88 additions & 159 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,163 +1,92 @@
using Pkg
using RecursiveArrayTools
using Test
using SafeTestsets
using SciMLTesting
using Pkg

const GROUP = get(ENV, "GROUP", "All")

function activate_downstream_env()
Pkg.activate(joinpath(@__DIR__, "Downstream"))
Pkg.develop(PackageSpec(path = dirname(@__DIR__)))
return Pkg.instantiate()
end

function activate_gpu_env()
Pkg.activate(joinpath(@__DIR__, "GPU"))
Pkg.develop(PackageSpec(path = dirname(@__DIR__)))
Pkg.develop(
PackageSpec(
path = joinpath(dirname(@__DIR__), "lib", "RecursiveArrayToolsArrayPartitionAnyAll")
)
)
return Pkg.instantiate()
end

function activate_nopre_env()
Pkg.activate(joinpath(@__DIR__, "NoPre"))
Pkg.develop(PackageSpec(path = dirname(@__DIR__)))
return Pkg.instantiate()
end

function activate_qa_env()
Pkg.activate(joinpath(@__DIR__, "QA"))
# On Julia < 1.11, the [sources] section in the QA Project.toml is not
# honored, so Pkg.develop the umbrella root path explicitly.
if VERSION < v"1.11.0-DEV.0"
Pkg.develop(PackageSpec(path = dirname(@__DIR__)))
end
return Pkg.instantiate()
end

@time begin
# Detect sublibrary test groups.
# GROUP can be a bare sublibrary name (Core test group) or
# "{sublibrary}_{TEST_GROUP}" for any custom group (e.g., QA, GPU, etc.).
# Sublibraries declare their groups in test/test_groups.toml.
lib_dir = joinpath(dirname(@__DIR__), "lib")

# Check if GROUP matches a sublibrary, possibly with a _SUFFIX for the test group.
# Scan underscores right-to-left to find the longest matching sublibrary prefix.
function _detect_sublibrary_group(group, lib_dir)
isdir(joinpath(lib_dir, group)) && return (group, "Core")
for i in length(group):-1:1
if group[i] == '_' && isdir(joinpath(lib_dir, group[1:(i - 1)]))
return (group[1:(i - 1)], group[(i + 1):end])
end
end
return (group, "Core")
end
base_group, test_group = _detect_sublibrary_group(GROUP, lib_dir)

if isdir(joinpath(lib_dir, base_group))
Pkg.activate(joinpath(lib_dir, base_group))
# On Julia < 1.11, the [sources] section in Project.toml is not supported.
# Manually Pkg.develop local path dependencies so CI tests the PR branch code.
# The sublibraries depend on the umbrella root via [sources] (../..); develop
# those local paths, skipping the active project itself.
if VERSION < v"1.11.0-DEV.0"
developed = Set{String}()
push!(developed, normpath(joinpath(lib_dir, base_group)))
specs = Pkg.PackageSpec[]
queue = [joinpath(lib_dir, base_group)]
while !isempty(queue)
pkg_dir = popfirst!(queue)
toml_path = joinpath(pkg_dir, "Project.toml")
isfile(toml_path) || continue
toml = Pkg.TOML.parsefile(toml_path)
if haskey(toml, "sources")
for (dep_name, source_spec) in toml["sources"]
if source_spec isa Dict && haskey(source_spec, "path")
dep_path = normpath(joinpath(pkg_dir, source_spec["path"]))
if isdir(dep_path) && !(dep_path in developed)
push!(developed, dep_path)
push!(specs, Pkg.PackageSpec(path = dep_path))
push!(queue, dep_path)
end
end
end
end
end
isempty(specs) || Pkg.develop(specs)
end
withenv("RECURSIVEARRAYTOOLS_TEST_GROUP" => test_group) do
Pkg.test(base_group, julia_args = ["--check-bounds=auto"], force_latest_compatible_version = false, allow_reresolve = true)
end
else
# Root package's own test groups.
# Captured so that, in a local "All" run, the QA group's isolated env does
# not leak into the subsequent functional Core tests run in the main env.
main_test_project = Base.active_project()

if GROUP == "QA"
# QA (Aqua) runs in its own dep-adding environment (test/QA/Project.toml)
# so the QA tooling is not part of the main test dependency set.
activate_qa_env()
@time @safetestset "Quality Assurance" include("QA/qa.jl")
Pkg.activate(main_test_project)
end

if GROUP == "Core" || GROUP == "All"
# The root Core tests use the VA[...]/AP[...] shorthand syntax, which
# lives in the RecursiveArrayToolsShorthandConstructors sublibrary.
Pkg.develop(
PackageSpec(
path = joinpath(dirname(@__DIR__), "lib", "RecursiveArrayToolsShorthandConstructors")
)
run_tests(;
core = function ()
# The root Core tests use the VA[...]/AP[...] shorthand syntax, which lives
# in the RecursiveArrayToolsShorthandConstructors sublibrary; develop it into
# the active main env before running them.
Pkg.develop(
Pkg.PackageSpec(
path = joinpath(dirname(@__DIR__), "lib", "RecursiveArrayToolsShorthandConstructors")
)
@time @safetestset "Utils Tests" include("Core/utils_test.jl")
@time @safetestset "NamedArrayPartition Tests" include("Core/named_array_partition_tests.jl")
@time @safetestset "Partitions Tests" include("Core/partitions_test.jl")
@time @safetestset "Partitions and StaticArrays Tests" include("Core/partitions_and_static_arrays.jl")
@time @safetestset "VecOfArr Indexing Tests" include("Core/basic_indexing.jl")
@time @safetestset "VecOfArr Interface Tests" include("Core/interface_tests.jl")
@time @safetestset "Table traits" include("Core/tabletraits.jl")
@time @safetestset "StaticArrays Tests" include("Core/copy_static_array_test.jl")
@time @safetestset "Linear Algebra Tests" include("Core/linalg.jl")
@time @safetestset "Adjoint Tests" include("Core/adjoints.jl")
@time @safetestset "Mooncake Tests" include("Core/mooncake.jl")
@time @safetestset "Measurement Tests" include("Core/measurements.jl")
end

if GROUP == "SymbolicIndexingInterface" || GROUP == "All"
@time @safetestset "SymbolicIndexingInterface API test" include("SymbolicIndexingInterface/symbolic_indexing_interface_test.jl")
end

if GROUP == "Downstream"
activate_downstream_env()
@time @safetestset "ODE Solve Tests" include("Downstream/odesolve.jl")
@time @safetestset "Event Tests with ArrayPartition" include("Downstream/downstream_events.jl")
@time @safetestset "Measurements and Units" include("Downstream/measurements_and_units.jl")
@time @safetestset "TrackerExt" include("Downstream/TrackerExt.jl")
# TODO: re-enable after SciMLBase compat bump for RAT v4 (SciML/SciMLBase.jl#1297)
# @time @safetestset "Downstream Adjoint Tests" include("Downstream/adjoints.jl")
end

if GROUP == "SymbolicIndexingInterface" || GROUP == "Downstream"
if GROUP == "SymbolicIndexingInterface"
activate_downstream_env()
end
@time @safetestset "DiffEqArray Indexing Tests" include("Downstream/symbol_indexing.jl")
end

if GROUP == "GPU"
activate_gpu_env()
@time @safetestset "VectorOfArray GPU" include("GPU/vectorofarray_gpu.jl")
@time @safetestset "ArrayPartition GPU" include("GPU/arraypartition_gpu.jl")
end

if GROUP == "NoPre"
activate_nopre_env()
@time @safetestset "JET Tests" include("NoPre/jet_tests.jl")
end
end
end
)
@time @safetestset "Utils Tests" include("Core/utils_test.jl")
@time @safetestset "NamedArrayPartition Tests" include("Core/named_array_partition_tests.jl")
@time @safetestset "Partitions Tests" include("Core/partitions_test.jl")
@time @safetestset "Partitions and StaticArrays Tests" include("Core/partitions_and_static_arrays.jl")
@time @safetestset "VecOfArr Indexing Tests" include("Core/basic_indexing.jl")
@time @safetestset "VecOfArr Interface Tests" include("Core/interface_tests.jl")
@time @safetestset "Table traits" include("Core/tabletraits.jl")
@time @safetestset "StaticArrays Tests" include("Core/copy_static_array_test.jl")
@time @safetestset "Linear Algebra Tests" include("Core/linalg.jl")
@time @safetestset "Adjoint Tests" include("Core/adjoints.jl")
@time @safetestset "Mooncake Tests" include("Core/mooncake.jl")
return @time @safetestset "Measurement Tests" include("Core/measurements.jl")
end,
groups = Dict(
# The SII API safetestset runs in the main test env. It is the in-main-env
# half shared between the "All" group (Core + this) and the
# "SymbolicIndexingInterface" umbrella.
"SII_Main" => function ()
return @time @safetestset "SymbolicIndexingInterface API test" include("SymbolicIndexingInterface/symbolic_indexing_interface_test.jl")
end,
# The DiffEqArray symbol-indexing test runs in the Downstream env. It is the
# second half of both the "SymbolicIndexingInterface" and "Downstream" paths.
"SII_Downstream" => (;
env = joinpath(@__DIR__, "Downstream"),
body = function ()
return @time @safetestset "DiffEqArray Indexing Tests" include("Downstream/symbol_indexing.jl")
end,
),
"Downstream" => (;
env = joinpath(@__DIR__, "Downstream"),
body = function ()
@time @safetestset "ODE Solve Tests" include("Downstream/odesolve.jl")
@time @safetestset "Event Tests with ArrayPartition" include("Downstream/downstream_events.jl")
@time @safetestset "Measurements and Units" include("Downstream/measurements_and_units.jl")
@time @safetestset "TrackerExt" include("Downstream/TrackerExt.jl")
# TODO: re-enable after SciMLBase compat bump for RAT v4 (SciML/SciMLBase.jl#1297)
# @time @safetestset "Downstream Adjoint Tests" include("Downstream/adjoints.jl")
return @time @safetestset "DiffEqArray Indexing Tests" include("Downstream/symbol_indexing.jl")
end,
),
"GPU" => (;
env = joinpath(@__DIR__, "GPU"),
parent = [
dirname(@__DIR__),
joinpath(dirname(@__DIR__), "lib", "RecursiveArrayToolsArrayPartitionAnyAll"),
],
body = function ()
@time @safetestset "VectorOfArray GPU" include("GPU/vectorofarray_gpu.jl")
return @time @safetestset "ArrayPartition GPU" include("GPU/arraypartition_gpu.jl")
end,
),
"NoPre" => (;
env = joinpath(@__DIR__, "NoPre"),
body = function ()
return @time @safetestset "JET Tests" include("NoPre/jet_tests.jl")
end,
),
),
qa = (;
env = joinpath(@__DIR__, "QA"),
body = joinpath(@__DIR__, "QA", "qa.jl"),
),
# "All" runs exactly Core plus the in-main-env SII API safetestset (matching the
# original `GROUP == "Core" || "All"` and `GROUP == "SymbolicIndexingInterface"
# || "All"` branches). It deliberately excludes QA, Downstream, GPU and NoPre.
all = ["Core", "SII_Main"],
umbrellas = Dict(
# GROUP=SymbolicIndexingInterface runs the SII API safetestset (main env) and
# then the DiffEqArray symbol-indexing test (Downstream env), in that order.
"SymbolicIndexingInterface" => ["SII_Main", "SII_Downstream"],
),
lib_dir = joinpath(dirname(@__DIR__), "lib"),
# Root reads GROUP to pick the sublibrary, but the sublibraries read
# RECURSIVEARRAYTOOLS_TEST_GROUP for their own sub-group; hand it off via that
# variable (matching the original withenv around Pkg.test(sublib)).
sublib_env = "RECURSIVEARRAYTOOLS_TEST_GROUP",
)
Loading