diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 378a9f12..02743ea9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,12 +19,11 @@ jobs: strategy: fail-fast: false matrix: + # Root-only test groups. The lib/* sublibraries (DataDrivenDMD, DataDrivenSR, + # DataDrivenSparse, DataDrivenLux) are covered by SublibraryCI.yml via the + # project model and must not be double-tested through the root matrix. group: - Core - - DataDrivenDMD - - DataDrivenSR - - DataDrivenSparse - - DataDrivenLux - nopre version: - '1' @@ -37,5 +36,5 @@ jobs: with: julia-version: "${{ matrix.version }}" group: "${{ matrix.group }}" - coverage-directories: "src,lib/DataDrivenDMD/src,lib/DataDrivenSparse/src,lib/DataDrivenSR/src,lib/DataDrivenLux/src" + coverage-directories: "src" secrets: "inherit" diff --git a/.github/workflows/Downgrade.yml b/.github/workflows/Downgrade.yml index 60a30367..514f6061 100644 --- a/.github/workflows/Downgrade.yml +++ b/.github/workflows/Downgrade.yml @@ -10,11 +10,13 @@ on: - master paths-ignore: - 'docs/**' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: test: name: "Downgrade" uses: "SciML/.github/.github/workflows/downgrade.yml@v1" with: - julia-version: "1.10" - skip: "Pkg,TOML" + group: "Core" secrets: "inherit" diff --git a/.github/workflows/DowngradeSublibraries.yml b/.github/workflows/DowngradeSublibraries.yml index 6fba9003..abe9f107 100644 --- a/.github/workflows/DowngradeSublibraries.yml +++ b/.github/workflows/DowngradeSublibraries.yml @@ -19,4 +19,6 @@ jobs: secrets: "inherit" with: julia-version: "lts" - skip: "Pkg,TOML,Statistics,LinearAlgebra,SparseArrays,InteractiveUtils,Random,Test,DataDrivenDiffEq" + group-env-name: "DATADRIVENDIFFEQ_TEST_GROUP" + group-env-value: "Core" + # Every lib/* sublibrary is downgrade-tested (projects auto-discovered, no exclusions). diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml new file mode 100644 index 00000000..8542fadf --- /dev/null +++ b/.github/workflows/Downstream.yml @@ -0,0 +1,62 @@ +name: IntegrationTest +on: + push: + branches: [master] + tags: [v*] + pull_request: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: ${{ matrix.package.repo }}/${{ matrix.package.group }}/${{ matrix.julia-version }} + runs-on: ${{ matrix.os }} + env: + GROUP: ${{ matrix.package.group }} + strategy: + fail-fast: false + matrix: + julia-version: ['1.10'] + os: [ubuntu-latest] + package: + - {user: SciML, repo: SymbolicNumericIntegration.jl, group: All} + - {user: ODINN-SciML, repo: ODINN.jl, group: Core1} + + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v3 + with: + version: ${{ matrix.julia-version }} + arch: x64 + - name: Clone Downstream + uses: actions/checkout@v6 + with: + repository: ${{ matrix.package.user }}/${{ matrix.package.repo }} + path: downstream + - name: Load this and run the downstream tests + shell: julia --color=yes --project=downstream {0} + run: | + using Pkg + try + # Develop the root umbrella DataDrivenDiffEq plus every lib/* sublibrary + # so the downstream tests run against this checkout rather than releases. + Pkg.develop(PackageSpec(path=".")); + Pkg.develop(map(path ->Pkg.PackageSpec.(;path="lib/$(path)"), readdir("./lib"))); + Pkg.test(coverage=true) # resolver may fail with test time deps + catch err + err isa Pkg.Resolve.ResolverError || rethrow() + # If we can't resolve that means this is incompatible by SemVer and this is fine + # It means we marked this as a breaking change, so we don't need to worry about + # mistakenly introducing a breaking change, as we have intentionally made one + @info "Not compatible with this release. No problem." exception=err + exit(0) # Exit immediately, as a success + end + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v6 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info + fail_ci_if_error: false + disable_safe_directory: true diff --git a/.github/workflows/SublibraryCI.yml b/.github/workflows/SublibraryCI.yml index 6cec66bb..3a6d609f 100644 --- a/.github/workflows/SublibraryCI.yml +++ b/.github/workflows/SublibraryCI.yml @@ -19,4 +19,7 @@ concurrency: jobs: sublibraries: uses: "SciML/.github/.github/workflows/sublibrary-project-tests.yml@v1" + with: + group-env-name: DATADRIVENDIFFEQ_TEST_GROUP + check-bounds: auto secrets: "inherit" diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index f49313b6..576699a6 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -1,15 +1,22 @@ -name: TagBot +name: "TagBot" on: issue_comment: - types: - - created + types: [created] workflow_dispatch: jobs: - TagBot: - if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' - runs-on: ubuntu-latest - steps: - - uses: JuliaRegistries/TagBot@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - ssh: ${{ secrets.DOCUMENTER_KEY }} + tagbot: + uses: "SciML/.github/.github/workflows/tagbot.yml@v1" + secrets: "inherit" + tagbot-subpackages: + strategy: + fail-fast: false + matrix: + subdir: + - "lib/DataDrivenDMD" + - "lib/DataDrivenLux" + - "lib/DataDrivenSparse" + - "lib/DataDrivenSR" + uses: "SciML/.github/.github/workflows/tagbot.yml@v1" + with: + subdir: "${{ matrix.subdir }}" + secrets: "inherit" diff --git a/LICENSE b/LICENSE.md similarity index 100% rename from LICENSE rename to LICENSE.md diff --git a/Project.toml b/Project.toml index c9178645..19fb0149 100644 --- a/Project.toml +++ b/Project.toml @@ -31,15 +31,19 @@ CommonSolve = "0.2" DataInterpolations = "4, 5, 6, 7, 8" DiffEqBase = "6.211, 7" DocStringExtensions = "0.7, 0.8, 0.9" +LinearAlgebra = "1" MLUtils = "0.3, 0.4" ModelingToolkit = "11.21.0" OrdinaryDiffEqTsit5 = "1, 2" Parameters = "0.12.1" +Pkg = "1" PrecompileTools = "1" ProgressMeter = "1.6" QuadGK = "2.4" +Random = "1" RecipesBase = "1" Reexport = "1.0" +SafeTestsets = "0.1" SciMLBase = "2.155, 3" SciMLStructures = "1" Setfield = "1" @@ -47,6 +51,7 @@ Statistics = "1" StatsBase = "0.32.0, 0.33, 0.34" SymbolicUtils = "4" Symbolics = "7" +Test = "1" julia = "1.10" [extras] diff --git a/lib/DataDrivenDMD/LICENSE b/lib/DataDrivenDMD/LICENSE.md similarity index 100% rename from lib/DataDrivenDMD/LICENSE rename to lib/DataDrivenDMD/LICENSE.md diff --git a/lib/DataDrivenDMD/Project.toml b/lib/DataDrivenDMD/Project.toml index 110d2223..53280ed5 100644 --- a/lib/DataDrivenDMD/Project.toml +++ b/lib/DataDrivenDMD/Project.toml @@ -7,10 +7,18 @@ version = "0.1.3" DataDrivenDiffEq = "2445eb08-9709-466a-b3fc-47e12bd697a2" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +[sources] +DataDrivenDiffEq = {path = "../.."} + [compat] -DataDrivenDiffEq = "1.14" +DataDrivenDiffEq = "1.15" +LinearAlgebra = "1.10" OrdinaryDiffEq = "6, 7" OrdinaryDiffEqFunctionMap = "1, 2" +SafeTestsets = "0.1" +StableRNGs = "1" +StatsBase = "0.34" +Test = "1.10" julia = "1.10" [extras] diff --git a/lib/DataDrivenDMD/README.md b/lib/DataDrivenDMD/README.md new file mode 100644 index 00000000..dd66bc1c --- /dev/null +++ b/lib/DataDrivenDMD/README.md @@ -0,0 +1,12 @@ +# DataDrivenDMD.jl + +[![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged) +[![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/DataDrivenDiffEq/stable/) + +[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor%27s%20Guide-blueviolet)](https://github.com/SciML/ColPrac) +[![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) + +DataDrivenDMD.jl is a component of the [DataDrivenDiffEq.jl](https://github.com/SciML/DataDrivenDiffEq.jl) monorepo. Dynamic Mode Decomposition (DMD) and Koopman-based methods for discovering linear operator approximations of dynamical systems from data. +While completely independent and usable on its own, users wanting the full data-driven modeling suite should use [DataDrivenDiffEq.jl](https://github.com/SciML/DataDrivenDiffEq.jl). + +For a full description of the methods and their options, see the [DataDrivenDiffEq documentation](https://docs.sciml.ai/DataDrivenDiffEq/stable/). diff --git a/lib/DataDrivenDMD/test/linear_autonomous.jl b/lib/DataDrivenDMD/test/Core/linear_autonomous.jl similarity index 100% rename from lib/DataDrivenDMD/test/linear_autonomous.jl rename to lib/DataDrivenDMD/test/Core/linear_autonomous.jl diff --git a/lib/DataDrivenDMD/test/linear_forced.jl b/lib/DataDrivenDMD/test/Core/linear_forced.jl similarity index 100% rename from lib/DataDrivenDMD/test/linear_forced.jl rename to lib/DataDrivenDMD/test/Core/linear_forced.jl diff --git a/lib/DataDrivenDMD/test/nonlinear_autonomous.jl b/lib/DataDrivenDMD/test/Core/nonlinear_autonomous.jl similarity index 100% rename from lib/DataDrivenDMD/test/nonlinear_autonomous.jl rename to lib/DataDrivenDMD/test/Core/nonlinear_autonomous.jl diff --git a/lib/DataDrivenDMD/test/nonlinear_forced.jl b/lib/DataDrivenDMD/test/Core/nonlinear_forced.jl similarity index 100% rename from lib/DataDrivenDMD/test/nonlinear_forced.jl rename to lib/DataDrivenDMD/test/Core/nonlinear_forced.jl diff --git a/lib/DataDrivenDMD/test/runtests.jl b/lib/DataDrivenDMD/test/runtests.jl index a9fe50b4..81d9ddb2 100644 --- a/lib/DataDrivenDMD/test/runtests.jl +++ b/lib/DataDrivenDMD/test/runtests.jl @@ -5,21 +5,21 @@ using Test @info "Finished loading packages" -const GROUP = get(ENV, "GROUP", "All") +const GROUP = get(ENV, "DATADRIVENDIFFEQ_TEST_GROUP", get(ENV, "GROUP", "All")) @time begin - if GROUP == "All" || GROUP == "DataDrivenDMD" + if GROUP == "All" || GROUP == "Core" || GROUP == "DataDrivenDMD" @safetestset "Linear autonomous" begin - include("./linear_autonomous.jl") + include("./Core/linear_autonomous.jl") end @safetestset "Linear forced" begin - include("./linear_forced.jl") + include("./Core/linear_forced.jl") end @safetestset "Nonlinear autonomous" begin - include("./nonlinear_autonomous.jl") + include("./Core/nonlinear_autonomous.jl") end @safetestset "Nonlinear forced" begin - include("./nonlinear_forced.jl") + include("./Core/nonlinear_forced.jl") end end end diff --git a/lib/DataDrivenDMD/test/test_groups.toml b/lib/DataDrivenDMD/test/test_groups.toml new file mode 100644 index 00000000..8f657c06 --- /dev/null +++ b/lib/DataDrivenDMD/test/test_groups.toml @@ -0,0 +1,4 @@ +# Each sublibrary only defines functional ("Core") tests; there are no QA +# (Aqua/JET/AllocCheck) suites here, so the default QA group is not declared. +[Core] +versions = ["lts", "1", "pre"] diff --git a/lib/DataDrivenLux/LICENSE b/lib/DataDrivenLux/LICENSE.md similarity index 100% rename from lib/DataDrivenLux/LICENSE rename to lib/DataDrivenLux/LICENSE.md diff --git a/lib/DataDrivenLux/Project.toml b/lib/DataDrivenLux/Project.toml index 03cb2c79..aa6675be 100644 --- a/lib/DataDrivenLux/Project.toml +++ b/lib/DataDrivenLux/Project.toml @@ -30,27 +30,37 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" TransformVariables = "84d833dd-6860-57f9-a1a7-6da5db126cff" WeightInitializers = "d49dbf32-c5c2-4618-8acc-27bb2598ef2d" +[sources] +DataDrivenDiffEq = {path = "../.."} + [compat] AbstractDifferentiation = "0.6" ChainRulesCore = "1.15" CommonSolve = "0.2.4" ComponentArrays = "0.15" ConcreteStructs = "0.2.3" -DataDrivenDiffEq = "1.14" -OrdinaryDiffEq = "6, 7" +DataDrivenDiffEq = "1.15" +Distributed = "1.10" Distributions = "0.25" DistributionsAD = "0.6" DocStringExtensions = "0.9.3" ForwardDiff = "0.10, 1" IntervalArithmetic = "0.22, 0.23, 1" InverseFunctions = "0.1" +LinearAlgebra = "1.10" +Logging = "1.10" Lux = "1" LuxCore = "1" Optim = "1.7, 2" Optimisers = "0.3, 0.4" +OrdinaryDiffEq = "6, 7" ProgressMeter = "1.7" +Random = "1.10" +SafeTestsets = "0.1" Setfield = "1" +StableRNGs = "1" StatsBase = "0.34.3" +Test = "1.10" TransformVariables = "0.8" WeightInitializers = "1" julia = "1.10" diff --git a/lib/DataDrivenLux/README.md b/lib/DataDrivenLux/README.md new file mode 100644 index 00000000..8f00d27e --- /dev/null +++ b/lib/DataDrivenLux/README.md @@ -0,0 +1,12 @@ +# DataDrivenLux.jl + +[![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged) +[![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/DataDrivenDiffEq/stable/) + +[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor%27s%20Guide-blueviolet)](https://github.com/SciML/ColPrac) +[![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) + +DataDrivenLux.jl is a component of the [DataDrivenDiffEq.jl](https://github.com/SciML/DataDrivenDiffEq.jl) monorepo. Deep-learning-based structure search, built on Lux.jl, for discovering governing equations via differentiable directed-acyclic-graph candidate models. +While completely independent and usable on its own, users wanting the full data-driven modeling suite should use [DataDrivenDiffEq.jl](https://github.com/SciML/DataDrivenDiffEq.jl). + +For a full description of the methods and their options, see the [DataDrivenDiffEq documentation](https://docs.sciml.ai/DataDrivenDiffEq/stable/). diff --git a/lib/DataDrivenLux/test/cache.jl b/lib/DataDrivenLux/test/Core/cache.jl similarity index 100% rename from lib/DataDrivenLux/test/cache.jl rename to lib/DataDrivenLux/test/Core/cache.jl diff --git a/lib/DataDrivenLux/test/candidate.jl b/lib/DataDrivenLux/test/Core/candidate.jl similarity index 100% rename from lib/DataDrivenLux/test/candidate.jl rename to lib/DataDrivenLux/test/Core/candidate.jl diff --git a/lib/DataDrivenLux/test/crossentropy_solve.jl b/lib/DataDrivenLux/test/Core/crossentropy_solve.jl similarity index 100% rename from lib/DataDrivenLux/test/crossentropy_solve.jl rename to lib/DataDrivenLux/test/Core/crossentropy_solve.jl diff --git a/lib/DataDrivenLux/test/graphs.jl b/lib/DataDrivenLux/test/Core/graphs.jl similarity index 100% rename from lib/DataDrivenLux/test/graphs.jl rename to lib/DataDrivenLux/test/Core/graphs.jl diff --git a/lib/DataDrivenLux/test/layers.jl b/lib/DataDrivenLux/test/Core/layers.jl similarity index 100% rename from lib/DataDrivenLux/test/layers.jl rename to lib/DataDrivenLux/test/Core/layers.jl diff --git a/lib/DataDrivenLux/test/nodes.jl b/lib/DataDrivenLux/test/Core/nodes.jl similarity index 100% rename from lib/DataDrivenLux/test/nodes.jl rename to lib/DataDrivenLux/test/Core/nodes.jl diff --git a/lib/DataDrivenLux/test/randomsearch_solve.jl b/lib/DataDrivenLux/test/Core/randomsearch_solve.jl similarity index 100% rename from lib/DataDrivenLux/test/randomsearch_solve.jl rename to lib/DataDrivenLux/test/Core/randomsearch_solve.jl diff --git a/lib/DataDrivenLux/test/reinforce_solve.jl b/lib/DataDrivenLux/test/Core/reinforce_solve.jl similarity index 100% rename from lib/DataDrivenLux/test/reinforce_solve.jl rename to lib/DataDrivenLux/test/Core/reinforce_solve.jl diff --git a/lib/DataDrivenLux/test/runtests.jl b/lib/DataDrivenLux/test/runtests.jl index 9db36e13..f3a27c55 100644 --- a/lib/DataDrivenLux/test/runtests.jl +++ b/lib/DataDrivenLux/test/runtests.jl @@ -5,25 +5,25 @@ using Test @info "Finished loading packages" -const GROUP = get(ENV, "GROUP", "All") +const GROUP = get(ENV, "DATADRIVENDIFFEQ_TEST_GROUP", get(ENV, "GROUP", "All")) @testset "DataDrivenLux" begin - if GROUP == "All" || GROUP == "DataDrivenLux" + if GROUP == "All" || GROUP == "Core" || GROUP == "DataDrivenLux" @testset "Lux" begin - @safetestset "Nodes" include("nodes.jl") - @safetestset "Layers" include("layers.jl") - @safetestset "Graphs" include("graphs.jl") + @safetestset "Nodes" include("Core/nodes.jl") + @safetestset "Layers" include("Core/layers.jl") + @safetestset "Graphs" include("Core/graphs.jl") end @testset "Caches" begin - @safetestset "Candidate" include("candidate.jl") - @safetestset "Cache" include("cache.jl") + @safetestset "Candidate" include("Core/candidate.jl") + @safetestset "Cache" include("Core/cache.jl") end @testset "Algorithms" begin - @safetestset "RandomSearch" include("randomsearch_solve.jl") - @safetestset "Reinforce" include("reinforce_solve.jl") - @safetestset "CrossEntropy" include("crossentropy_solve.jl") + @safetestset "RandomSearch" include("Core/randomsearch_solve.jl") + @safetestset "Reinforce" include("Core/reinforce_solve.jl") + @safetestset "CrossEntropy" include("Core/crossentropy_solve.jl") end end end diff --git a/lib/DataDrivenLux/test/test_groups.toml b/lib/DataDrivenLux/test/test_groups.toml new file mode 100644 index 00000000..8f657c06 --- /dev/null +++ b/lib/DataDrivenLux/test/test_groups.toml @@ -0,0 +1,4 @@ +# Each sublibrary only defines functional ("Core") tests; there are no QA +# (Aqua/JET/AllocCheck) suites here, so the default QA group is not declared. +[Core] +versions = ["lts", "1", "pre"] diff --git a/lib/DataDrivenSR/LICENSE b/lib/DataDrivenSR/LICENSE.md similarity index 100% rename from lib/DataDrivenSR/LICENSE rename to lib/DataDrivenSR/LICENSE.md diff --git a/lib/DataDrivenSR/Project.toml b/lib/DataDrivenSR/Project.toml index 5da78b17..dcff83e5 100644 --- a/lib/DataDrivenSR/Project.toml +++ b/lib/DataDrivenSR/Project.toml @@ -8,10 +8,16 @@ DataDrivenDiffEq = "2445eb08-9709-466a-b3fc-47e12bd697a2" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" SymbolicRegression = "8254be44-1295-4e6a-a16d-46603ac705cb" +[sources] +DataDrivenDiffEq = {path = "../.."} + [compat] +DataDrivenDiffEq = "1.15" Reexport = "1.2" -DataDrivenDiffEq = "1.14" +StableRNGs = "1" +StatsBase = "0.34" SymbolicRegression = "1" +Test = "1.10" julia = "1.10" [extras] diff --git a/lib/DataDrivenSR/README.md b/lib/DataDrivenSR/README.md new file mode 100644 index 00000000..3187d7cb --- /dev/null +++ b/lib/DataDrivenSR/README.md @@ -0,0 +1,12 @@ +# DataDrivenSR.jl + +[![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged) +[![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/DataDrivenDiffEq/stable/) + +[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor%27s%20Guide-blueviolet)](https://github.com/SciML/ColPrac) +[![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) + +DataDrivenSR.jl is a component of the [DataDrivenDiffEq.jl](https://github.com/SciML/DataDrivenDiffEq.jl) monorepo. Symbolic regression methods, built on SymbolicRegression.jl, for discovering closed-form governing equations from data. +While completely independent and usable on its own, users wanting the full data-driven modeling suite should use [DataDrivenDiffEq.jl](https://github.com/SciML/DataDrivenDiffEq.jl). + +For a full description of the methods and their options, see the [DataDrivenDiffEq documentation](https://docs.sciml.ai/DataDrivenDiffEq/stable/). diff --git a/lib/DataDrivenSR/test/Core/symbolic_regression.jl b/lib/DataDrivenSR/test/Core/symbolic_regression.jl new file mode 100644 index 00000000..be923bb8 --- /dev/null +++ b/lib/DataDrivenSR/test/Core/symbolic_regression.jl @@ -0,0 +1,62 @@ +using DataDrivenDiffEq +using DataDrivenSR +using Test +using StableRNGs + +rng = StableRNG(42) +X = rand(rng, 2, 50) + +@testset "Simple" begin + alg = DataDrivenSR.EQSearch( + eq_options = Options( + unary_operators = [sin, exp], + binary_operators = [*], maxdepth = 1, + seed = 42, + verbosity = -1, progress = false + ) + ) + f(x) = [sin(x[1]); exp(x[2])] + Y = hcat(map(f, eachcol(X))...) + prob = DirectDataDrivenProblem(X, Y) + res = solve(prob, alg) + @test r2(res) >= 0.95 + @test rss(res) <= 1.0e-5 +end + +@testset "Univariate" begin + alg = DataDrivenSR.EQSearch( + eq_options = Options( + unary_operators = [sin, exp], + binary_operators = [*], maxdepth = 1, + seed = 42, + verbosity = -1, progress = false + ) + ) + + Y = sin.(X[1:1, :]) + prob = DirectDataDrivenProblem(X, Y) + res = solve(prob, alg) + @test r2(res) >= 0.95 + @test rss(res) <= 1.0e-5 +end + +@testset "Lifted" begin + alg = DataDrivenSR.EQSearch( + eq_options = Options( + unary_operators = [sin, exp], + binary_operators = [+], maxdepth = 1, + seed = 42, + verbosity = -1, progress = false + ) + ) + + f(x) = [sin(x[1] .^ 2); exp(x[2] * x[1])] + Y = hcat(map(f, eachcol(X))...) + + @variables x y + basis = Basis([x; y; x^2; y^2; x * y], [x; y]) + prob = DirectDataDrivenProblem(X, Y) + res = solve(prob, basis, alg) + @test r2(res) >= 0.95 + @test rss(res) <= 1.0e-5 +end diff --git a/lib/DataDrivenSR/test/runtests.jl b/lib/DataDrivenSR/test/runtests.jl index be923bb8..83c48667 100644 --- a/lib/DataDrivenSR/test/runtests.jl +++ b/lib/DataDrivenSR/test/runtests.jl @@ -3,60 +3,10 @@ using DataDrivenSR using Test using StableRNGs -rng = StableRNG(42) -X = rand(rng, 2, 50) +const GROUP = get(ENV, "DATADRIVENDIFFEQ_TEST_GROUP", get(ENV, "GROUP", "All")) -@testset "Simple" begin - alg = DataDrivenSR.EQSearch( - eq_options = Options( - unary_operators = [sin, exp], - binary_operators = [*], maxdepth = 1, - seed = 42, - verbosity = -1, progress = false - ) - ) - f(x) = [sin(x[1]); exp(x[2])] - Y = hcat(map(f, eachcol(X))...) - prob = DirectDataDrivenProblem(X, Y) - res = solve(prob, alg) - @test r2(res) >= 0.95 - @test rss(res) <= 1.0e-5 -end - -@testset "Univariate" begin - alg = DataDrivenSR.EQSearch( - eq_options = Options( - unary_operators = [sin, exp], - binary_operators = [*], maxdepth = 1, - seed = 42, - verbosity = -1, progress = false - ) - ) - - Y = sin.(X[1:1, :]) - prob = DirectDataDrivenProblem(X, Y) - res = solve(prob, alg) - @test r2(res) >= 0.95 - @test rss(res) <= 1.0e-5 -end - -@testset "Lifted" begin - alg = DataDrivenSR.EQSearch( - eq_options = Options( - unary_operators = [sin, exp], - binary_operators = [+], maxdepth = 1, - seed = 42, - verbosity = -1, progress = false - ) - ) - - f(x) = [sin(x[1] .^ 2); exp(x[2] * x[1])] - Y = hcat(map(f, eachcol(X))...) - - @variables x y - basis = Basis([x; y; x^2; y^2; x * y], [x; y]) - prob = DirectDataDrivenProblem(X, Y) - res = solve(prob, basis, alg) - @test r2(res) >= 0.95 - @test rss(res) <= 1.0e-5 +if GROUP == "All" || GROUP == "Core" || GROUP == "DataDrivenSR" + @testset "Symbolic Regression" begin + include("./Core/symbolic_regression.jl") + end end diff --git a/lib/DataDrivenSR/test/test_groups.toml b/lib/DataDrivenSR/test/test_groups.toml new file mode 100644 index 00000000..8f657c06 --- /dev/null +++ b/lib/DataDrivenSR/test/test_groups.toml @@ -0,0 +1,4 @@ +# Each sublibrary only defines functional ("Core") tests; there are no QA +# (Aqua/JET/AllocCheck) suites here, so the default QA group is not declared. +[Core] +versions = ["lts", "1", "pre"] diff --git a/lib/DataDrivenSparse/LICENSE b/lib/DataDrivenSparse/LICENSE.md similarity index 100% rename from lib/DataDrivenSparse/LICENSE rename to lib/DataDrivenSparse/LICENSE.md diff --git a/lib/DataDrivenSparse/Project.toml b/lib/DataDrivenSparse/Project.toml index 00b82d72..0ae18070 100644 --- a/lib/DataDrivenSparse/Project.toml +++ b/lib/DataDrivenSparse/Project.toml @@ -9,12 +9,21 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" +[sources] +DataDrivenDiffEq = {path = "../.."} [compat] -julia = "1.10" -DataDrivenDiffEq = "1.14" +DataDrivenDiffEq = "1.15" +LinearAlgebra = "1.10" OrdinaryDiffEq = "6, 7" +Printf = "1.10" +Random = "1.10" Reexport = "1.2" +SafeTestsets = "0.1" +StableRNGs = "1" +StatsBase = "0.34" +Test = "1.10" +julia = "1.10" [extras] OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" diff --git a/lib/DataDrivenSparse/README.md b/lib/DataDrivenSparse/README.md new file mode 100644 index 00000000..2100d556 --- /dev/null +++ b/lib/DataDrivenSparse/README.md @@ -0,0 +1,12 @@ +# DataDrivenSparse.jl + +[![Join the chat at https://julialang.zulipchat.com #sciml-bridged](https://img.shields.io/static/v1?label=Zulip&message=chat&color=9558b2&labelColor=389826)](https://julialang.zulipchat.com/#narrow/stream/279055-sciml-bridged) +[![Global Docs](https://img.shields.io/badge/docs-SciML-blue.svg)](https://docs.sciml.ai/DataDrivenDiffEq/stable/) + +[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor%27s%20Guide-blueviolet)](https://github.com/SciML/ColPrac) +[![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) + +DataDrivenSparse.jl is a component of the [DataDrivenDiffEq.jl](https://github.com/SciML/DataDrivenDiffEq.jl) monorepo. Sparse regression methods (e.g. SINDy, STLSQ, ADMM) for discovering parsimonious governing equations from data. +While completely independent and usable on its own, users wanting the full data-driven modeling suite should use [DataDrivenDiffEq.jl](https://github.com/SciML/DataDrivenDiffEq.jl). + +For a full description of the methods and their options, see the [DataDrivenDiffEq documentation](https://docs.sciml.ai/DataDrivenDiffEq/stable/). diff --git a/lib/DataDrivenSparse/test/cartpole.jl b/lib/DataDrivenSparse/test/Core/cartpole.jl similarity index 100% rename from lib/DataDrivenSparse/test/cartpole.jl rename to lib/DataDrivenSparse/test/Core/cartpole.jl diff --git a/lib/DataDrivenSparse/test/michaelis_menten.jl b/lib/DataDrivenSparse/test/Core/michaelis_menten.jl similarity index 100% rename from lib/DataDrivenSparse/test/michaelis_menten.jl rename to lib/DataDrivenSparse/test/Core/michaelis_menten.jl diff --git a/lib/DataDrivenSparse/test/pendulum.jl b/lib/DataDrivenSparse/test/Core/pendulum.jl similarity index 100% rename from lib/DataDrivenSparse/test/pendulum.jl rename to lib/DataDrivenSparse/test/Core/pendulum.jl diff --git a/lib/DataDrivenSparse/test/sparse_linear_solve.jl b/lib/DataDrivenSparse/test/Core/sparse_linear_solve.jl similarity index 100% rename from lib/DataDrivenSparse/test/sparse_linear_solve.jl rename to lib/DataDrivenSparse/test/Core/sparse_linear_solve.jl diff --git a/lib/DataDrivenSparse/test/runtests.jl b/lib/DataDrivenSparse/test/runtests.jl index 9d797903..8971b69e 100644 --- a/lib/DataDrivenSparse/test/runtests.jl +++ b/lib/DataDrivenSparse/test/runtests.jl @@ -2,18 +2,22 @@ using DataDrivenDiffEq using DataDrivenSparse using SafeTestsets -@safetestset "Basic Sparse Regression" begin - include("./sparse_linear_solve.jl") -end +const GROUP = get(ENV, "DATADRIVENDIFFEQ_TEST_GROUP", get(ENV, "GROUP", "All")) -@safetestset "Pendulum" begin - include("./pendulum.jl") -end +if GROUP == "All" || GROUP == "Core" || GROUP == "DataDrivenSparse" + @safetestset "Basic Sparse Regression" begin + include("./Core/sparse_linear_solve.jl") + end -@safetestset "Michaelis Menten" begin - include("./michaelis_menten.jl") -end + @safetestset "Pendulum" begin + include("./Core/pendulum.jl") + end + + @safetestset "Michaelis Menten" begin + include("./Core/michaelis_menten.jl") + end -@safetestset "Cartpole" begin - include("./cartpole.jl") + @safetestset "Cartpole" begin + include("./Core/cartpole.jl") + end end diff --git a/lib/DataDrivenSparse/test/test_groups.toml b/lib/DataDrivenSparse/test/test_groups.toml new file mode 100644 index 00000000..8f657c06 --- /dev/null +++ b/lib/DataDrivenSparse/test/test_groups.toml @@ -0,0 +1,4 @@ +# Each sublibrary only defines functional ("Core") tests; there are no QA +# (Aqua/JET/AllocCheck) suites here, so the default QA group is not declared. +[Core] +versions = ["lts", "1", "pre"] diff --git a/test/basis/basis.jl b/test/Core/basis.jl similarity index 100% rename from test/basis/basis.jl rename to test/Core/basis.jl diff --git a/test/commonsolve/commonsolve.jl b/test/Core/commonsolve.jl similarity index 100% rename from test/commonsolve/commonsolve.jl rename to test/Core/commonsolve.jl diff --git a/test/basis/generators.jl b/test/Core/generators.jl similarity index 100% rename from test/basis/generators.jl rename to test/Core/generators.jl diff --git a/test/basis/implicit_basis.jl b/test/Core/implicit_basis.jl similarity index 100% rename from test/basis/implicit_basis.jl rename to test/Core/implicit_basis.jl diff --git a/test/problem/problem.jl b/test/Core/problem.jl similarity index 100% rename from test/problem/problem.jl rename to test/Core/problem.jl diff --git a/test/problem/samplers.jl b/test/Core/samplers.jl similarity index 100% rename from test/problem/samplers.jl rename to test/Core/samplers.jl diff --git a/test/solution/solution.jl b/test/Core/solution.jl similarity index 100% rename from test/solution/solution.jl rename to test/Core/solution.jl diff --git a/test/utils.jl b/test/Core/utils.jl similarity index 100% rename from test/utils.jl rename to test/Core/utils.jl diff --git a/test/runtests.jl b/test/runtests.jl index 22fbae2f..8f418b2d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,59 +4,96 @@ using SafeTestsets, Test, Pkg const GROUP = get(ENV, "GROUP", "All") -function dev_subpkg(subpkg) - subpkg_path = joinpath(dirname(@__DIR__), "lib", subpkg) - return Pkg.develop(PackageSpec(path = subpkg_path)) -end - -function activate_subpkg_env(subpkg) - subpkg_path = joinpath(dirname(@__DIR__), "lib", subpkg) - Pkg.activate(subpkg_path) - Pkg.develop(PackageSpec(path = subpkg_path)) - return Pkg.instantiate() -end - function activate_nopre_env() Pkg.activate(joinpath(@__DIR__, "nopre")) Pkg.develop(PackageSpec(path = dirname(@__DIR__))) return Pkg.instantiate() end +# GROUP can name a sublibrary (e.g. "DataDrivenDMD" -> Core test group) or +# "{sublibrary}_{TEST_GROUP}" for a custom group (e.g. "DataDrivenDMD_QA"). +# Sublibraries declare any non-default groups in test/test_groups.toml; the +# DATADRIVENDIFFEQ_TEST_GROUP env var threads the group into the sublib's +# own runtests.jl. CI dispatches sublibraries through SublibraryCI.yml, so +# this branch is primarily for running a sublibrary's suite locally. +function _detect_sublibrary_group(group, lib_dir) + isdir(joinpath(lib_dir, group)) && return (group, "Core") + for i in length(group):-1:1 + if group[i] == '_' && isdir(joinpath(lib_dir, group[1:(i - 1)])) + return (group[1:(i - 1)], group[(i + 1):end]) + end + end + return (group, "Core") +end + @time begin - if GROUP == "All" || GROUP == "Core" || GROUP == "Downstream" + lib_dir = joinpath(dirname(@__DIR__), "lib") + base_group, test_group = _detect_sublibrary_group(GROUP, lib_dir) + + if isdir(joinpath(lib_dir, base_group)) + Pkg.activate(joinpath(lib_dir, base_group)) + # On Julia < 1.11 the [sources] section is not honored; develop the + # in-repo path dependencies (transitively) so the sublibrary tests run + # against this checkout of DataDrivenDiffEq rather than a released one. + if VERSION < v"1.11.0-DEV.0" + developed = Set{String}() + push!(developed, normpath(joinpath(lib_dir, base_group))) + specs = Pkg.PackageSpec[] + queue = [joinpath(lib_dir, base_group)] + while !isempty(queue) + pkg_dir = popfirst!(queue) + toml_path = joinpath(pkg_dir, "Project.toml") + isfile(toml_path) || continue + toml = Pkg.TOML.parsefile(toml_path) + if haskey(toml, "sources") + for (dep_name, source_spec) in toml["sources"] + if source_spec isa Dict && haskey(source_spec, "path") + dep_path = normpath(joinpath(pkg_dir, source_spec["path"])) + if isdir(dep_path) && !(dep_path in developed) + push!(developed, dep_path) + @info "Queuing local source dependency" dep_name dep_path + push!(specs, Pkg.PackageSpec(path = dep_path)) + push!(queue, dep_path) + end + end + end + end + end + isempty(specs) || Pkg.develop(specs) + end + withenv("DATADRIVENDIFFEQ_TEST_GROUP" => test_group) do + Pkg.test(base_group, coverage = true) + end + elseif GROUP == "nopre" + # nopre tests are excluded from Julia pre-release versions in CI + # to avoid failures from upstream changes (e.g., JET type inference) + activate_nopre_env() + @safetestset "JET Static Analysis" begin + include("nopre/jet_tests.jl") + end + elseif GROUP == "All" || GROUP == "Core" || GROUP == "Downstream" @testset "All" begin @safetestset "Basis" begin - include("./basis/basis.jl") + include("./Core/basis.jl") end @safetestset "Implicit Basis" begin - include("./basis/implicit_basis.jl") + include("./Core/implicit_basis.jl") end @safetestset "Basis generators" begin - include("./basis/generators.jl") + include("./Core/generators.jl") end @safetestset "DataDrivenProblem" begin - include("./problem/problem.jl") + include("./Core/problem.jl") end @safetestset "DataDrivenSolution" begin - include("./solution/solution.jl") + include("./Core/solution.jl") end @safetestset "Utilities" begin - include("./utils.jl") + include("./Core/utils.jl") end @safetestset "CommonSolve" begin - include("./commonsolve/commonsolve.jl") + include("./Core/commonsolve.jl") end end - elseif GROUP == "nopre" - # nopre tests are excluded from Julia pre-release versions in CI - # to avoid failures from upstream changes (e.g., JET type inference) - activate_nopre_env() - @safetestset "JET Static Analysis" begin - include("nopre/jet_tests.jl") - end - else - dev_subpkg(GROUP) - subpkg_path = joinpath(dirname(@__DIR__), "lib", GROUP) - Pkg.test(PackageSpec(name = GROUP, path = subpkg_path); coverage = true) end end diff --git a/test/test_groups.toml b/test/test_groups.toml new file mode 100644 index 00000000..dfe8e145 --- /dev/null +++ b/test/test_groups.toml @@ -0,0 +1,12 @@ +# Test group declarations for the DataDrivenDiffEq root package. +# The base-environment Core group runs in the package's main test env and is +# the only group in the `All` run. The `nopre` group is a dep-adding group: it +# carries JET in test/nopre/Project.toml, is excluded from `All`, and (as its +# name says) does not run on prerelease Julia. Its CI.yml matrix group input is +# lowercase, so the group name here is kept lowercase to stay consistent. + +[Core] +versions = ["lts", "1", "pre"] + +[nopre] +versions = ["lts", "1"]