Canonicalize tests around @safetestset (isolation, matches OrdinaryDiffEq)

Project.toml

@@ -17,14 +17,16 @@ ADTypesEnzymeCoreExt = "EnzymeCore"
 ChainRulesCore = "1.0.2"
 ConstructionBase = "1.5"
 EnzymeCore = "0.5.3,0.6,0.7,0.8"
+SafeTestsets = "0.1, 1"
 julia = "1.10"
 
 [extras]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["ChainRulesCore", "EnzymeCore", "Pkg", "Setfield", "Test"]
+test = ["ChainRulesCore", "EnzymeCore", "Pkg", "SafeTestsets", "Setfield", "Test"]

test/dense.jl

@@ -1,3 +1,5 @@
+include("test_setup.jl")
+
 @testset "AutoChainRules" begin
     ad = AutoChainRules(; ruleconfig = ForwardOrReverseRuleConfig())
     @test ad isa AbstractADType

test/legacy.jl

@@ -1,3 +1,5 @@
+include("test_setup.jl")
+
 @testset "AutoModelingToolkit" begin
     ad_sparse1 = @test_deprecated AutoModelingToolkit(;
         obj_sparse = true, cons_sparse = false

test/misc.jl

@@ -1,3 +1,5 @@
+include("test_setup.jl")
+
 @testset "Broadcasting" begin
     for ad in every_ad()
         @test identity.(ad) == ad

test/runtests.jl

@@ -1,28 +1,11 @@
 using Pkg
-using ADTypes
-using ADTypes: AbstractADType,
-    mode,
-    ForwardMode,
-    ForwardOrReverseMode,
-    ReverseMode,
-    SymbolicMode
-using ADTypes: dense_ad,
-    NoSparsityDetector,
-    KnownJacobianSparsityDetector,
-    KnownHessianSparsityDetector,
-    sparsity_detector,
-    jacobian_sparsity,
-    hessian_sparsity,
-    NoColoringAlgorithm,
-    coloring_algorithm,
-    column_coloring,
-    row_coloring,
-    symmetric_coloring
-using ChainRulesCore: ChainRulesCore, RuleConfig,
-    HasForwardsMode, HasReverseMode,
-    NoForwardsMode, NoReverseMode
-using EnzymeCore: EnzymeCore
+using SafeTestsets
 using Test
+# `using ADTypes` in Main is load-bearing for the printing tests: Julia qualifies a
+# type's module in `show` based on visibility in `Base.active_module()` (Main), not the
+# module the test runs in. Without it, `string(AutoForwardDiff())` becomes
+# "ADTypes.AutoForwardDiff()" and the `startswith(..., "Auto")` assertions fail.
+using ADTypes
 
 const GROUP = get(ENV, "GROUP", "All")
 
@@ -36,91 +19,27 @@ function activate_qa_env()
     return Pkg.instantiate()
 end
 
-## Backend-specific
-
-struct CustomTag end
-
-struct ForwardRuleConfig <: RuleConfig{Union{HasForwardsMode, NoReverseMode}} end
-struct ReverseRuleConfig <: RuleConfig{Union{NoForwardsMode, HasReverseMode}} end
-struct ForwardOrReverseRuleConfig <: RuleConfig{Union{HasForwardsMode, HasReverseMode}} end
-
-struct FakeSparsityDetector <: ADTypes.AbstractSparsityDetector end
-struct FakeColoringAlgorithm <: ADTypes.AbstractColoringAlgorithm end
-
-function every_ad()
-    return [
-        AutoChainRules(; ruleconfig = :rc),
-        AutoDiffractor(),
-        AutoEnzyme(),
-        AutoFastDifferentiation(),
-        AutoFiniteDiff(),
-        AutoFiniteDifferences(; fdm = :fdm),
-        AutoForwardDiff(),
-        AutoGTPSA(),
-        AutoHyperHessians(),
-        AutoPolyesterForwardDiff(),
-        AutoReverseDiff(),
-        AutoSymbolics(),
-        AutoTapir(),
-        AutoTracker(),
-        AutoZygote(),
-    ]
-end
-
-function every_ad_with_options()
-    return [
-        AutoChainRules(; ruleconfig = :rc),
-        AutoDiffractor(),
-        AutoEnzyme(),
-        AutoEnzyme(mode = :forward),
-        AutoFastDifferentiation(),
-        AutoFiniteDiff(),
-        AutoFiniteDiff(
-            fdtype = :fd, fdjtype = :fdj, fdhtype = :fdh,
-            relstep = 1, absstep = 2, dir = false
-        ),
-        AutoFiniteDifferences(; fdm = :fdm),
-        AutoForwardDiff(),
-        AutoForwardDiff(chunksize = 3, tag = :tag),
-        AutoGTPSA(),
-        AutoGTPSA(descriptor = Val(:descriptor)),
-        AutoHyperHessians(),
-        AutoHyperHessians(chunksize = 8),
-        AutoMooncake(; config = :config),
-        AutoMooncakeForward(; config = :config),
-        AutoPolyesterForwardDiff(),
-        AutoPolyesterForwardDiff(chunksize = 3, tag = :tag),
-        AutoReverseDiff(),
-        AutoReverseDiff(compile = true),
-        AutoSymbolics(),
-        AutoTapir(),
-        AutoTapir(safe_mode = false),
-        AutoTracker(),
-        AutoZygote(),
-    ]
-end
-
 ## Tests
 
 if GROUP == "All" || GROUP == "Core"
     @testset verbose = true "ADTypes.jl" begin
-        @testset "Dense" begin
+        @safetestset "Dense" begin
             include("dense.jl")
         end
-        @testset "Sparse" begin
+        @safetestset "Sparse" begin
             include("sparse.jl")
         end
-        @testset "Symbols" begin
+        @safetestset "Symbols" begin
             include("symbols.jl")
         end
-        @testset "Legacy" begin
+        @safetestset "Legacy" begin
             include("legacy.jl")
         end
-        @testset "Miscellaneous" begin
+        @safetestset "Miscellaneous" begin
             include("misc.jl")
         end
         if VERSION >= v"1.11.0-DEV.469"
-            @testset "Public" begin
+            @safetestset "Public" begin
                 include("public.jl")
             end
         end

test/sparse.jl

@@ -1,3 +1,5 @@
+include("test_setup.jl")
+
 @testset "Subtyping" begin
     for ad in every_ad()
         sparse_ad = AutoSparse(ad)

test/test_setup.jl

@@ -0,0 +1,86 @@
+using ADTypes
+using ADTypes: AbstractADType,
+    mode,
+    ForwardMode,
+    ForwardOrReverseMode,
+    ReverseMode,
+    SymbolicMode
+using ADTypes: dense_ad,
+    NoSparsityDetector,
+    KnownJacobianSparsityDetector,
+    KnownHessianSparsityDetector,
+    sparsity_detector,
+    jacobian_sparsity,
+    hessian_sparsity,
+    NoColoringAlgorithm,
+    coloring_algorithm,
+    column_coloring,
+    row_coloring,
+    symmetric_coloring
+using ChainRulesCore: ChainRulesCore, RuleConfig,
+    HasForwardsMode, HasReverseMode,
+    NoForwardsMode, NoReverseMode
+using EnzymeCore: EnzymeCore
+using Test
+
+struct CustomTag end
+
+struct ForwardRuleConfig <: RuleConfig{Union{HasForwardsMode, NoReverseMode}} end
+struct ReverseRuleConfig <: RuleConfig{Union{NoForwardsMode, HasReverseMode}} end
+struct ForwardOrReverseRuleConfig <: RuleConfig{Union{HasForwardsMode, HasReverseMode}} end
+
+struct FakeSparsityDetector <: ADTypes.AbstractSparsityDetector end
+struct FakeColoringAlgorithm <: ADTypes.AbstractColoringAlgorithm end
+
+function every_ad()
+    return [
+        AutoChainRules(; ruleconfig = :rc),
+        AutoDiffractor(),
+        AutoEnzyme(),
+        AutoFastDifferentiation(),
+        AutoFiniteDiff(),
+        AutoFiniteDifferences(; fdm = :fdm),
+        AutoForwardDiff(),
+        AutoGTPSA(),
+        AutoHyperHessians(),
+        AutoPolyesterForwardDiff(),
+        AutoReverseDiff(),
+        AutoSymbolics(),
+        AutoTapir(),
+        AutoTracker(),
+        AutoZygote(),
+    ]
+end
+
+function every_ad_with_options()
+    return [
+        AutoChainRules(; ruleconfig = :rc),
+        AutoDiffractor(),
+        AutoEnzyme(),
+        AutoEnzyme(mode = :forward),
+        AutoFastDifferentiation(),
+        AutoFiniteDiff(),
+        AutoFiniteDiff(
+            fdtype = :fd, fdjtype = :fdj, fdhtype = :fdh,
+            relstep = 1, absstep = 2, dir = false
+        ),
+        AutoFiniteDifferences(; fdm = :fdm),
+        AutoForwardDiff(),
+        AutoForwardDiff(chunksize = 3, tag = :tag),
+        AutoGTPSA(),
+        AutoGTPSA(descriptor = Val(:descriptor)),
+        AutoHyperHessians(),
+        AutoHyperHessians(chunksize = 8),
+        AutoMooncake(; config = :config),
+        AutoMooncakeForward(; config = :config),
+        AutoPolyesterForwardDiff(),
+        AutoPolyesterForwardDiff(chunksize = 3, tag = :tag),
+        AutoReverseDiff(),
+        AutoReverseDiff(compile = true),
+        AutoSymbolics(),
+        AutoTapir(),
+        AutoTapir(safe_mode = false),
+        AutoTracker(),
+        AutoZygote(),
+    ]
+end