From a31712a77a40f6fa0c73cfb326465ba97efda1ff Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Wed, 18 Mar 2026 13:45:45 +0100 Subject: [PATCH 01/12] test: prepare the work to introduce pytest by adding logging + fixing some paths handling in idfx_test.py --- pytools/idfx_test.py | 211 ++++++++++++++++-- test/Dust/DustEnergy/testme.py | 2 +- test/Dust/DustyShock/testme.py | 2 +- test/Dust/DustyWave/testme.py | 2 +- test/HD/FargoPlanet/testme.py | 2 +- test/HD/MachReflection/testme.py | 2 +- test/HD/SedovBlastWave/testme.py | 2 +- test/HD/ShearingBox/testme.py | 2 +- test/HD/ViscousDisk/testme.py | 2 +- test/HD/ViscousFlowPastCylinder/testme.py | 2 +- test/HD/sod-iso/testme.py | 2 +- test/HD/sod/testme.py | 2 +- test/HD/thermalDiffusion/testme.py | 2 +- test/MHD/AmbipolarCshock/testme.py | 2 +- test/MHD/AmbipolarCshock3D/testme.py | 2 +- test/MHD/AxisFluxTube/testme.py | 2 +- test/MHD/Coarsening/testme.py | 2 +- test/MHD/FargoMHDSpherical/testme.py | 2 +- test/MHD/HallWhistler/testme.py | 2 +- test/MHD/LinearWaveTest/testme.py | 2 +- test/MHD/MTI/testme.py | 2 +- test/MHD/OrszagTang/testme.py | 2 +- test/MHD/OrszagTang3D/testme.py | 2 +- test/MHD/ResistiveAlfvenWave/testme.py | 2 +- test/MHD/ShearingBox/testme.py | 2 +- test/MHD/clessTDiffusion/testme.py | 2 +- test/MHD/sod-iso/testme.py | 2 +- test/MHD/sod/testme.py | 2 +- test/MHD/sphBragTDiffusion/testme.py | 2 +- test/MHD/sphBragViscosity/testme.py | 2 +- test/Planet/Planet3Body/testme.py | 2 +- test/Planet/PlanetMigration2D/testme.py | 2 +- test/Planet/PlanetPlanetRK42D/testme.py | 2 +- test/Planet/PlanetSpiral2D/testme.py | 2 +- test/Planet/PlanetTorque3D/testme.py | 2 +- test/Planet/PlanetsIsActiveRK52D/testme.py | 2 +- test/SelfGravity/DustyCollapse/testme.py | 2 +- test/SelfGravity/JeansInstability/testme.py | 2 +- test/SelfGravity/RandomSphere/testme.py | 2 +- .../RandomSphereCartesian/testme.py | 2 +- test/SelfGravity/UniformCollapse/testme.py | 2 +- test/utils/columnDensity/testme.py | 2 +- test/utils/dumpImage/testme.py | 2 +- test/utils/lookupTable/testme.py | 2 +- 44 files changed, 237 insertions(+), 60 deletions(-) diff --git a/pytools/idfx_test.py b/pytools/idfx_test.py index fed74a22e..a87b2de44 100644 --- a/pytools/idfx_test.py +++ b/pytools/idfx_test.py @@ -4,6 +4,8 @@ import subprocess import sys import re +import json +import pytest import numpy as np import matplotlib.pyplot as plt @@ -22,7 +24,7 @@ class bcolors: UNDERLINE = '\033[4m' class idfxTest: - def __init__ (self): + def __init__ (self, current_test_file, name=""): parser = argparse.ArgumentParser() idefix_dir_env = os.getenv("IDEFIX_DIR") @@ -102,20 +104,83 @@ def __init__ (self): parser.add_argument("-ccache", help="Use ccache to reduce the build time over multiple run of the test suite.", action="store_true") + + parser.add_argument("-restart", + help="Enable creating a restart from a checkpoint.", + action='store_true') + + parser.add_argument("-v", "--verbose", + help="Enable verbose mode, by not capturing the output.", + action="store_true") + + parser.add_argument("--help-pytest", + help="Display the options you can transmit directly to pytest in addition to the specific to idefix tests.", + action="store_true") + + parser.add_argument("-fake", + help="Make a fake run by just logging the actions to validate that we generate same command over refactoring.", + action="store_true") + + parser.add_argument("-subdir", + default="./test", + help="Select the test in the given subdir not to run all.", + type=str) args, unknown=parser.parse_known_args() # transform all arguments from args into attributes of this instance self.__dict__.update(vars(args)) + # store the full path of problem directory + self.currentTestFile = current_test_file + self.currentTestName = name + self.problemDir=os.path.dirname(current_test_file) self.referenceDirectory = os.path.join(idefix_dir_env,"reference") # current directory relative to $IDEFIX_DIR/test (used to retrieve the path ot reference files) - self.testDir=os.path.relpath(os.curdir,os.path.join(idefix_dir_env,"test")) + self.testDir=os.path.relpath(self.problemDir,os.path.join(idefix_dir_env,"test")) # build directory, currently inside the test named build-test - self.buildDir=os.path.abspath("./build-test") - # store the full path of problem directory - self.problemDir=os.path.abspath("./") - - def configure(self,definitionFile=""): + self.buildDir=os.path.join(self.problemDir,"build-test") + # remind what build we dit last + self.lastCmakeCmd="" + # subdir + self.filterSubdir=args.subdir + # save + self.cmdArgs = vars(args) + self.cmdArgs.update({ + "restart_no_overwrite": [], + }) + self.log=[] + # when making a restart we should not overrite those files (will be checked) + self.restart_no_overwrite=[] + + # forward args for pytest + if args.verbose: + unknown.append("--capture=no") + if args.help_pytest: + unknown.append("--help") + # remaining args + self.remainingArgs=unknown + + def addLog(self, entry): + if self.fake: + self.log.append(entry) + with open(os.path.join(self.problemDir,"testsuite.log.json"), "w+") as fp: + json.dump(self.log, fp, indent='\t') + + def applyConfig(self, config: dict={}): + # check args + for key, value in config.items(): + if key not in ['ini', 'testfile', 'testname', 'dumpname']: + assert key in self.cmdArgs, f"The given configuration overriding try to set an invalid paramater : {key}={value}" + + # override options + newArgs = {} + newArgs.update(self.cmdArgs) + newArgs.update(config) + + # replace in dict + self.__dict__.update(newArgs) + + def _genCmakeCommand(self,definitionFile=""): comm=["cmake"] # add source directory comm.append(self.idefixDir) @@ -184,29 +249,90 @@ def configure(self,definitionFile=""): if self.ccache: comm.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache") + # ok + return comm + + + def configure(self,definitionFile="", reuse_last_same_build=True, override: dict={}): + # log + self.addLog({"call": "configure", "args":{ + 'definitionFile': definitionFile, + 'reuse_last_same_build': reuse_last_same_build, + 'override': override + }}) + + # gen command + comm = self._genCmakeCommand(definitionFile) + + # log + print("***************** CALLING CMAKE *******************") + print(f"mkdir -p {self.buildDir}") + print(f"cd {self.buildDir}") + print(' '.join(comm)) + print("***************************************************") + + # not action needed if same command or force no rebuild + if reuse_last_same_build == True and self.lastCmakeCmd == ' '.join(comm): + print("SKIP ALREADY DONE") + return + + # run try: - # clean before configuring again + # do cleanup self.clean() + # log and in fake mode we do not execute + self.addLog({"command": comm}) + # call cmake - cmake=subprocess.run(comm, cwd=os.path.abspath(self.buildDir)) - cmake.check_returncode() + if not self.fake: + cmake=subprocess.run(comm, cwd=os.path.abspath(self.buildDir)) + cmake.check_returncode() except subprocess.CalledProcessError as e: print(bcolors.FAIL+"***************************************************") print("Cmake failed") print("***************************************************"+bcolors.ENDC) raise e + finally: + # remind for next time + self.lastCmakeCmd = ' '.join(comm) def clean(self): + # log and in fake mode we do not execute + self.addLog({"call": "clean", "args":{}}) + if self.fake: + return + # remove the build directory before re-creating it if os.path.exists(self.buildDir): shutil.rmtree(self.buildDir) + + # remove previous build configuration cache in the test itself + # otherwise cmake take in in the build subdir + testDirCMakeCache = os.path.join(self.testDir, "CMakeCache.txt") + if os.path.exists(testDirCMakeCache): + os.unlink(testDirCMakeCache) + + # recreate os.makedirs(self.buildDir, exist_ok=False) def compile(self,jobs=8): + self.addLog({"call": "compile", "args":{ + 'jobs': jobs, + }}) + try: - make=subprocess.run(["make","-j"+str(jobs)], cwd=os.path.abspath(self.buildDir)) - make.check_returncode() + comm = ["make","-j"+str(jobs)] + self.addLog({"command": comm}) + + print("***************************************************") + print(f"cd {os.getcwd()}") + print(' '.join(comm)) + print("***************************************************") + + if not self.fake: + make=subprocess.run(comm, cwd=os.path.abspath(self.buildDir)) + make.check_returncode() except subprocess.CalledProcessError as e: print(bcolors.FAIL+"***************************************************") print("Compilation failed") @@ -214,6 +340,13 @@ def compile(self,jobs=8): raise e def run(self, inputFile="", np=2, nowrite=False, restart=-1): + # log + self.addLog({"call": "run", "args":{ + 'np': np, + 'nowrite': nowrite, + 'restart': restart, + }}) + comm=[os.path.join(self.buildDir,"idefix")] if inputFile: comm.append("-i") @@ -242,9 +375,16 @@ def run(self, inputFile="", np=2, nowrite=False, restart=-1): comm.append("-restart") comm.append(str(restart)) + print("***************************************************") + print(f"cd {os.getcwd()}") + print(' '.join(comm)) + print("***************************************************") + try: - make=subprocess.run(comm) - make.check_returncode() + self.addLog({"command": comm}) + if not self.fake: + make=subprocess.run(comm, cwd=self.problemDir) + make.check_returncode() except subprocess.CalledProcessError as e: print(bcolors.FAIL+"***************************************************") print("Execution failed") @@ -254,6 +394,9 @@ def run(self, inputFile="", np=2, nowrite=False, restart=-1): self._readLog() def _readLog(self): + if self.fake: + return + if not os.path.exists('./idefix.0.log'): # When no idefix file is produced, we leave return @@ -298,6 +441,12 @@ def _readLog(self): self.perf=float(line.group(1)) def checkOnly(self, filename, tolerance=0): + # log + self.addLog({"call": "checkOnly", "args":{ + 'filename': filename, + 'tolerance': tolerance, + }}) + # Assumes the code has been run manually using some configuration, so we simply # do the test suite witout configure/compile/run self._readLog() @@ -312,8 +461,14 @@ def checkOnly(self, filename, tolerance=0): self.nonRegressionTest(filename, tolerance) def standardTest(self): - if os.path.exists(os.path.join('python', 'testidefix.py')): - os.chdir("python") + # log and in fake mode do not execute. + self.addLog({"call": "standardTest", "args":{}}) + if self.fake: + return + + if os.path.exists(os.path.join(self.problemDir, 'python', 'testidefix.py')): + oldPwd = os.getcwd() + os.chdir(os.path.join(self.problemDir, "python")) comm = [sys.executable, "testidefix.py"] if self.noplot: comm.append("-noplot") @@ -328,12 +483,19 @@ def standardTest(self): print("***************************************************"+bcolors.ENDC) raise e print(bcolors.OKCYAN+"Standard test succeeded"+bcolors.ENDC) - os.chdir("..") + os.chdir(oldPwd) else: print(bcolors.WARNING+"No standard testidefix.py for this test"+bcolors.ENDC) sys.stdout.flush() def nonRegressionTest(self, filename,tolerance=0): + # log and in fake mode do not execute. + self.addLog({"call": "nonRegressionTest", "args":{ + "filename": filename, + "tolerance": tolerance + }}) + if self.fake: + return fileref=os.path.join(self.referenceDirectory, self.testDir, self._getReferenceFilename()) if not(os.path.exists(fileref)): @@ -357,6 +519,14 @@ def nonRegressionTest(self, filename,tolerance=0): sys.stdout.flush() def compareDump(self, file1, file2,tolerance=0): + self.addLog({"call": "compareDump", "args":{ + "file1": file1, + "file2": file2, + "tolerance": tolerance, + }}) + if self.fake: + return + Vref=readDump(file1) Vtest=readDump(file2) error=self._computeError(Vref,Vtest) @@ -371,6 +541,13 @@ def compareDump(self, file1, file2,tolerance=0): def makeReference(self,filename): + # log and in fake mode do not execute. + self.addLog({"call": "compareDump", "args":{ + "filename": filename, + }}) + if self.fake: + return + self._readLog() targetDir = os.path.join(self.referenceDirectory,self.testDir) if not os.path.exists(targetDir): diff --git a/test/Dust/DustEnergy/testme.py b/test/Dust/DustEnergy/testme.py index 7187d299b..12d476b66 100755 --- a/test/Dust/DustEnergy/testme.py +++ b/test/Dust/DustEnergy/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/Dust/DustyShock/testme.py b/test/Dust/DustyShock/testme.py index 2b03d1636..622f34772 100755 --- a/test/Dust/DustyShock/testme.py +++ b/test/Dust/DustyShock/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=1e-14) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/Dust/DustyWave/testme.py b/test/Dust/DustyWave/testme.py index 2b03d1636..622f34772 100755 --- a/test/Dust/DustyWave/testme.py +++ b/test/Dust/DustyWave/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=1e-14) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/HD/FargoPlanet/testme.py b/test/HD/FargoPlanet/testme.py index 5890d010f..b25a74bb0 100755 --- a/test/HD/FargoPlanet/testme.py +++ b/test/HD/FargoPlanet/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/HD/MachReflection/testme.py b/test/HD/MachReflection/testme.py index ce87f3923..7e6121815 100755 --- a/test/HD/MachReflection/testme.py +++ b/test/HD/MachReflection/testme.py @@ -23,7 +23,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp") -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/HD/SedovBlastWave/testme.py b/test/HD/SedovBlastWave/testme.py index 95fdd030a..c3ef7025f 100755 --- a/test/HD/SedovBlastWave/testme.py +++ b/test/HD/SedovBlastWave/testme.py @@ -12,7 +12,7 @@ name="dump.0001.dmp" -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','2'] diff --git a/test/HD/ShearingBox/testme.py b/test/HD/ShearingBox/testme.py index 3739910a0..18c792656 100755 --- a/test/HD/ShearingBox/testme.py +++ b/test/HD/ShearingBox/testme.py @@ -23,7 +23,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','1','2'] diff --git a/test/HD/ViscousDisk/testme.py b/test/HD/ViscousDisk/testme.py index 1e8ad742b..34628fd80 100755 --- a/test/HD/ViscousDisk/testme.py +++ b/test/HD/ViscousDisk/testme.py @@ -24,7 +24,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/HD/ViscousFlowPastCylinder/testme.py b/test/HD/ViscousFlowPastCylinder/testme.py index c831e8093..832a80e56 100755 --- a/test/HD/ViscousFlowPastCylinder/testme.py +++ b/test/HD/ViscousFlowPastCylinder/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/HD/sod-iso/testme.py b/test/HD/sod-iso/testme.py index e8f0f9ae2..109109baf 100755 --- a/test/HD/sod-iso/testme.py +++ b/test/HD/sod-iso/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/HD/sod/testme.py b/test/HD/sod/testme.py index e8f0f9ae2..109109baf 100755 --- a/test/HD/sod/testme.py +++ b/test/HD/sod/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/HD/thermalDiffusion/testme.py b/test/HD/thermalDiffusion/testme.py index 3806293d0..1d1a181b0 100755 --- a/test/HD/thermalDiffusion/testme.py +++ b/test/HD/thermalDiffusion/testme.py @@ -23,7 +23,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp") -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/AmbipolarCshock/testme.py b/test/MHD/AmbipolarCshock/testme.py index 675958a7b..0e2f0815e 100755 --- a/test/MHD/AmbipolarCshock/testme.py +++ b/test/MHD/AmbipolarCshock/testme.py @@ -23,7 +23,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/AmbipolarCshock3D/testme.py b/test/MHD/AmbipolarCshock3D/testme.py index 964224854..94a8b0f4c 100755 --- a/test/MHD/AmbipolarCshock3D/testme.py +++ b/test/MHD/AmbipolarCshock3D/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','1','1'] diff --git a/test/MHD/AxisFluxTube/testme.py b/test/MHD/AxisFluxTube/testme.py index 44a53dc28..5abb46568 100755 --- a/test/MHD/AxisFluxTube/testme.py +++ b/test/MHD/AxisFluxTube/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/Coarsening/testme.py b/test/MHD/Coarsening/testme.py index 52a083bd2..50d13c926 100755 --- a/test/MHD/Coarsening/testme.py +++ b/test/MHD/Coarsening/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/FargoMHDSpherical/testme.py b/test/MHD/FargoMHDSpherical/testme.py index 7ee3100b2..1a3b28adb 100755 --- a/test/MHD/FargoMHDSpherical/testme.py +++ b/test/MHD/FargoMHDSpherical/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','2'] diff --git a/test/MHD/HallWhistler/testme.py b/test/MHD/HallWhistler/testme.py index f8d633840..f8ba0a258 100755 --- a/test/MHD/HallWhistler/testme.py +++ b/test/MHD/HallWhistler/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/LinearWaveTest/testme.py b/test/MHD/LinearWaveTest/testme.py index 390a5759b..eeaaab0fe 100755 --- a/test/MHD/LinearWaveTest/testme.py +++ b/test/MHD/LinearWaveTest/testme.py @@ -24,7 +24,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','2'] diff --git a/test/MHD/MTI/testme.py b/test/MHD/MTI/testme.py index 7975c8075..0e2c9e317 100755 --- a/test/MHD/MTI/testme.py +++ b/test/MHD/MTI/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/OrszagTang/testme.py b/test/MHD/OrszagTang/testme.py index 761505caa..41f33aba6 100755 --- a/test/MHD/OrszagTang/testme.py +++ b/test/MHD/OrszagTang/testme.py @@ -33,7 +33,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/MHD/OrszagTang3D/testme.py b/test/MHD/OrszagTang3D/testme.py index b9626f9cc..7a66150c5 100755 --- a/test/MHD/OrszagTang3D/testme.py +++ b/test/MHD/OrszagTang3D/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) # if no decomposition is specified, use that one if not test.dec: diff --git a/test/MHD/ResistiveAlfvenWave/testme.py b/test/MHD/ResistiveAlfvenWave/testme.py index f9b8ae56f..ebc29ab57 100755 --- a/test/MHD/ResistiveAlfvenWave/testme.py +++ b/test/MHD/ResistiveAlfvenWave/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/ShearingBox/testme.py b/test/MHD/ShearingBox/testme.py index 364adde9b..ccee1a69c 100755 --- a/test/MHD/ShearingBox/testme.py +++ b/test/MHD/ShearingBox/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','1','2'] diff --git a/test/MHD/clessTDiffusion/testme.py b/test/MHD/clessTDiffusion/testme.py index 6ed5b1648..41193e141 100755 --- a/test/MHD/clessTDiffusion/testme.py +++ b/test/MHD/clessTDiffusion/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name, tolerance=2e-15) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/sod-iso/testme.py b/test/MHD/sod-iso/testme.py index 798aace57..c85cc4ce7 100755 --- a/test/MHD/sod-iso/testme.py +++ b/test/MHD/sod-iso/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/sod/testme.py b/test/MHD/sod/testme.py index 798aace57..c85cc4ce7 100755 --- a/test/MHD/sod/testme.py +++ b/test/MHD/sod/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/sphBragTDiffusion/testme.py b/test/MHD/sphBragTDiffusion/testme.py index 6ed5b1648..41193e141 100755 --- a/test/MHD/sphBragTDiffusion/testme.py +++ b/test/MHD/sphBragTDiffusion/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name, tolerance=2e-15) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/sphBragViscosity/testme.py b/test/MHD/sphBragViscosity/testme.py index 254417f44..b6c3467ea 100755 --- a/test/MHD/sphBragViscosity/testme.py +++ b/test/MHD/sphBragViscosity/testme.py @@ -29,7 +29,7 @@ def testMe(test): test.nonRegressionTest(filename=name, tolerance=1e-15) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/Planet/Planet3Body/testme.py b/test/Planet/Planet3Body/testme.py index c11d08963..6edac53ee 100755 --- a/test/Planet/Planet3Body/testme.py +++ b/test/Planet/Planet3Body/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/Planet/PlanetMigration2D/testme.py b/test/Planet/PlanetMigration2D/testme.py index 0921d4e62..6c435cbd3 100755 --- a/test/Planet/PlanetMigration2D/testme.py +++ b/test/Planet/PlanetMigration2D/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/Planet/PlanetPlanetRK42D/testme.py b/test/Planet/PlanetPlanetRK42D/testme.py index 26cb8ee21..9521aabac 100755 --- a/test/Planet/PlanetPlanetRK42D/testme.py +++ b/test/Planet/PlanetPlanetRK42D/testme.py @@ -38,7 +38,7 @@ def testMe(test): assert dump_mtime == os.path.getmtime("dump.0001.dmp"), "Dump was overwritten on restart" assert vtk_mtime == os.path.getmtime("data.0005.vtk"), "VTK file was overwritten on restart" -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/Planet/PlanetSpiral2D/testme.py b/test/Planet/PlanetSpiral2D/testme.py index 0921d4e62..6c435cbd3 100755 --- a/test/Planet/PlanetSpiral2D/testme.py +++ b/test/Planet/PlanetSpiral2D/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/Planet/PlanetTorque3D/testme.py b/test/Planet/PlanetTorque3D/testme.py index b4f04a507..4393636ea 100755 --- a/test/Planet/PlanetTorque3D/testme.py +++ b/test/Planet/PlanetTorque3D/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','2'] diff --git a/test/Planet/PlanetsIsActiveRK52D/testme.py b/test/Planet/PlanetsIsActiveRK52D/testme.py index ce0d405c0..aca39ae53 100755 --- a/test/Planet/PlanetsIsActiveRK52D/testme.py +++ b/test/Planet/PlanetsIsActiveRK52D/testme.py @@ -26,7 +26,7 @@ def testMe(test): # test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/SelfGravity/DustyCollapse/testme.py b/test/SelfGravity/DustyCollapse/testme.py index 689038265..2df3893ab 100755 --- a/test/SelfGravity/DustyCollapse/testme.py +++ b/test/SelfGravity/DustyCollapse/testme.py @@ -25,7 +25,7 @@ def testMe(test): test.standardTest() -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/SelfGravity/JeansInstability/testme.py b/test/SelfGravity/JeansInstability/testme.py index ff2f39a83..833feb262 100755 --- a/test/SelfGravity/JeansInstability/testme.py +++ b/test/SelfGravity/JeansInstability/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.standardTest() -test=tst.idfxTest() +test=tst.idfxTest(__file__) # if no decomposition is specified, use that one if not test.dec: diff --git a/test/SelfGravity/RandomSphere/testme.py b/test/SelfGravity/RandomSphere/testme.py index 38fc30d92..65123498b 100755 --- a/test/SelfGravity/RandomSphere/testme.py +++ b/test/SelfGravity/RandomSphere/testme.py @@ -26,7 +26,7 @@ def testMe(test): #test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','1'] diff --git a/test/SelfGravity/RandomSphereCartesian/testme.py b/test/SelfGravity/RandomSphereCartesian/testme.py index 64b4a9ba5..598aa3f21 100755 --- a/test/SelfGravity/RandomSphereCartesian/testme.py +++ b/test/SelfGravity/RandomSphereCartesian/testme.py @@ -26,7 +26,7 @@ def testMe(test): #test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: testMe(test) else: diff --git a/test/SelfGravity/UniformCollapse/testme.py b/test/SelfGravity/UniformCollapse/testme.py index 21debc0e0..69918896b 100755 --- a/test/SelfGravity/UniformCollapse/testme.py +++ b/test/SelfGravity/UniformCollapse/testme.py @@ -25,7 +25,7 @@ def testMe(test): test.standardTest() -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/utils/columnDensity/testme.py b/test/utils/columnDensity/testme.py index 6f017d7d8..cd94eb4ef 100755 --- a/test/utils/columnDensity/testme.py +++ b/test/utils/columnDensity/testme.py @@ -9,7 +9,7 @@ sys.path.append(os.getenv("IDEFIX_DIR")) import pytools.idfx_test as tst -test=tst.idfxTest() +test=tst.idfxTest(__file__) test.configure() test.compile() diff --git a/test/utils/dumpImage/testme.py b/test/utils/dumpImage/testme.py index 6f017d7d8..cd94eb4ef 100755 --- a/test/utils/dumpImage/testme.py +++ b/test/utils/dumpImage/testme.py @@ -9,7 +9,7 @@ sys.path.append(os.getenv("IDEFIX_DIR")) import pytools.idfx_test as tst -test=tst.idfxTest() +test=tst.idfxTest(__file__) test.configure() test.compile() diff --git a/test/utils/lookupTable/testme.py b/test/utils/lookupTable/testme.py index d40884652..53f6607f2 100755 --- a/test/utils/lookupTable/testme.py +++ b/test/utils/lookupTable/testme.py @@ -30,7 +30,7 @@ def MakeNumpyFile(): -test=tst.idfxTest() +test=tst.idfxTest(__file__) MakeNumpyFile() test.configure() From 1c1fd5c7d55cbdfc3adc31c091a0636538a1419e Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Wed, 18 Mar 2026 13:48:53 +0100 Subject: [PATCH 02/12] test: introduce the new runner based on pytest (./test.py) --- .gitignore | 1 + pytest.ini | 6 + pytools/idfx_test_gen.py | 249 +++++++++++++++++++++++++++ pytools/idfx_test_run.py | 251 ++++++++++++++++++++++++++++ pytools/pytest.ini | 2 + pytools/tests/__init__.py | 0 pytools/tests/test/pb1/testme.json | 11 ++ pytools/tests/test/pb2/testme.json | 18 ++ pytools/tests/test_idfx_test_gen.py | 208 +++++++++++++++++++++++ pytools/tests/test_idfx_test_run.py | 88 ++++++++++ test.py | 26 +++ test/python_requirements.txt | 4 + 12 files changed, 864 insertions(+) create mode 100644 pytest.ini create mode 100644 pytools/idfx_test_gen.py create mode 100644 pytools/idfx_test_run.py create mode 100644 pytools/pytest.ini create mode 100644 pytools/tests/__init__.py create mode 100644 pytools/tests/test/pb1/testme.json create mode 100644 pytools/tests/test/pb2/testme.json create mode 100644 pytools/tests/test_idfx_test_gen.py create mode 100644 pytools/tests/test_idfx_test_run.py create mode 100755 test.py diff --git a/.gitignore b/.gitignore index 88b83a749..e3bf696a2 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ test/**/*.pyc test/**/*.dat test/**/cmake_packages* test/**/ +idefix-tests.junit.xml # machine specific cache and hidden files .* diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..0b8cb7ff8 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +markers= + default: Test to run by default. +python_files=./test.py +junit_logging="system-out" +junit_log_passing_tests=false diff --git a/pytools/idfx_test_gen.py b/pytools/idfx_test_gen.py new file mode 100644 index 000000000..5a7f620b1 --- /dev/null +++ b/pytools/idfx_test_gen.py @@ -0,0 +1,249 @@ +##################################################################################### +# Idefix MHD astrophysical code +# Copyright(C) Sébastien Valat +# and other code contributors +# Licensed under CeCILL 2.1 License, see COPYING for more information +##################################################################################### + +import copy +import pytest + +DO_NOT_LOOP_ON = ['restart_no_overwrite', "dec", "multirun"] + +class IdefixDirTestGenerator: + ''' + Class used to generate the various configuration to run by parsing the + files `testme.py` found in the hierarchy of the /test directory of Idefix. + ''' + + def __init__(self, currentTestFile: str, name: str = ""): + ''' + Constructor or the class. + + Args: + currentTestFile (str): + Path the the current python file. Normally you + simply pass __file__ to this parameter. + name (str): + Define a name for the test, can be empty. + ''' + + self.currentTestFile = currentTestFile + self.currentTestName = name + + # generate the list of configs to run + def genTestConfigs(self, names:str, params, whenClauses = {}) -> list: + ''' + Generate the the list of configurations as pytest parameters. + It will unpack the configuration set by looping on all combinations defined + by the given sets. + + Args: + names (str): + Comma separated list of variables do consider to build the + name of the file. + params (dict|list): + A configuration set as a dictionnary or a list of + configuration set. + whenCaluses (dict|list): + Provide a set of clauses to apply after unpacking + the configuration so we can patch some values depending on some others. + + Returns: + A list of pytest.param() ready to be fiven to parametrized pytest functions. + ''' + # get name ordering list + nameList = names.split(',') + + # gen list of complete configs + all_configs = [] + if isinstance(params, dict): + all_configs += self._genOneConfigSeries(names, params) + elif isinstance(params, list): + for p in params: + all_configs += self._genOneConfigSeries(names, p) + else: + raise Exception("Should never be called !") + + # convert as parametrize with nice name + result = [] + for config in all_configs: + # append the file + config['testfile'] = self.currentTestFile + config['testname'] = self.currentTestName + # gen name + nameParts = [self.currentTestName] + for name in nameList: + if isinstance(config[name], bool): + if config[name]: + nameParts.append(name) + elif isinstance(config[name], str): + nameParts.append(str(config[name])) + else: + nameParts.append(f"{name}-{config[name]}") + confName = "-".join(nameParts) + + # apply when clause + config = self._applyWhen(config, whenClauses) + + result.append(pytest.param(config, id=confName)) + + # ok + return result + + def extractNamingParameters(self, params) -> list: + ''' + Loop on the parameters and check automatically what are the list of variable parameters. + + Args: + params (list|dict): + The list of configuration sets to scan or a single set. + + Returns: + The list of names of the variable parameters. + ''' + + # if not a list make a list + if not isinstance(params, list): + params = [params] + + # see params + seen = {} + variables = [] + + # loop + for param_set in params: + for key, value in param_set.items(): + if key in variables: + pass + elif key in DO_NOT_LOOP_ON: + pass + elif isinstance(value, list): + variables.append(key) + elif key in seen and seen[key] != value: + variables.append(key) + elif key not in seen: + seen[key] = value + + # by default sort by alphabetic order about var names + # TODO make something better by assigning a priority to vars + variables.sort() + + # ok + return ','.join(variables) + + def _genNextLevelCombinations(self, input: list, paramName: str, paramValues: list) -> list: + ''' + Take an input case list and unpack the given parameter to build the new combinations. + + Args: + input (list): + The incoming list of combinations already unpacked before this call. + paramName (str): + Name of the parameter to unpack. + paramValues (list): + The list of values to unpack and to build combinations for. + + Returns: + The updated list of run sets. + ''' + result = [] + for entry in input: + for value in paramValues: + v = copy.deepcopy(entry) + v[paramName] = value + result.append(v) + return result + + def _genOneConfigSeries(self, names: str, config: dict) -> list: + ''' + Generate the the list of configurations as pytest parameters. + It will unpack the configuration set by looping on all combinations defined + by the given sets. + + Args: + names (str): + Comma separated list of variables do consider to build the + name of the file. + config (dict): + A configuration set as a dictionnary. + + Returns: + A list of pytest.param() ready to be fiven to parametrized pytest functions. + ''' + # get name ordering list + nameList = names.split(',') + + # if there is ini in the list we put it at the end + loopOrder = nameList.copy() + if 'ini' in loopOrder: + loopOrder.remove('ini') + loopOrder.append('ini') + + # init core with everything not a list + core = {} + for key, value in config.items(): + if isinstance(value, list) and key not in DO_NOT_LOOP_ON: + assert key in nameList, f"All variable parameteres should be ordered in the names list, '{key}' is not." + else: + core[key] = copy.deepcopy(value) + + # at start we have only default core + result = [core] + + # loop + for key in loopOrder: + value = config[key] + assert isinstance(value, list), f"This parameter is marked as a list but is not a list : {key}={value} !" + result = self._genNextLevelCombinations(result, key, value) + + # ok + return result + + def _matchWhenClause(self, config: dict, when_clause: dict) -> bool: + ''' + Check the matching of a given when clause on the given configuration. + + Args: + config (dict): The configuration. + when_clause (dict): The when clause to check. + + Returns: + True if the clause, match, False otherwise. + ''' + for key, value in when_clause.items(): + if config[key] != value: + return False + return True + + def _applyWhen(self, config: dict, when: dict) -> dict: + ''' + Check if the when clause applies and apply if if any. + + Args: + config (dict): The configuration. + when_clause (dict): The when clause to check and apply. + + Returns: + The fixed config. + ''' + # nothing to do + if when == {}: + return config + + # clone + result = copy.deepcopy(config) + + # loop on when + if isinstance(when, list): + for clause in when: + if self._matchWhenClause(config, clause['conditions']): + result.update(clause['apply']) + elif isinstance(when, dict): + if self._matchWhenClause(config, when['conditions']): + result.update(when['apply']) + else: + raise Exception(f"Invalid type for 'when' : {when}") + + # ok + return result diff --git a/pytools/idfx_test_run.py b/pytools/idfx_test_run.py new file mode 100644 index 000000000..c30e4b928 --- /dev/null +++ b/pytools/idfx_test_run.py @@ -0,0 +1,251 @@ +##################################################################################### +# Idefix MHD astrophysical code +# Copyright(C) Sébastien Valat +# and other code contributors +# Licensed under CeCILL 2.1 License, see COPYING for more information +##################################################################################### + +import os +import sys +import json +import glob +import copy +import pytest +# idefix test class +import pytools.idfx_test as tst +from pytools.idfx_test_gen import IdefixDirTestGenerator +from contextlib import contextmanager + +@contextmanager +def moveInDir(path): + ''' + Change current working dir for the given directoy for what is inside the + with statement. + ''' + oldpwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(oldpwd) + +class IdexPytestRunner: + ''' + Implement the Idefix pytest runner to scan and run all the tests described by the files + testme.json into the /test directory of Idefix sources. + ''' + + def __init__(self, parentScriptFile: str): + self.currentTestFile="" + self.currentTestRunner: tst.idfxTest=None + self.parentScritFile=parentScriptFile + self.filterSubdir = os.environ.get("IDEFIX_TEST_FILTER_SUBDIR", "./test/") + + def _validateNaming(self, namings: str, autoExtracted: str, file: str): + names = namings.split(',') + for n in autoExtracted.split(','): + if not n in names: + raise Exception(f"Naming parameter list not match the auto-detectection, some are missinge. You gave : '{names}', detected '{autoExtracted}' into {file}") + + def _makeVariableArgAsList(self, namings: str, variants) -> list: + # make as a list + if isinstance(variants, list) == False: + variants = [variants] + + # split namings + namings = namings.split(",") + + # loop on each + for variant in variants: + for name in namings: + if name in variant and isinstance(variant[name], list) == False: + variant[name] = [variant[name]] + + def genTests(self) -> list: + sourceDir = os.path.dirname(self.parentScritFile) + + # loop over all tests + result = [] + with moveInDir(sourceDir): + # if missing / + if self.filterSubdir != "" and self.filterSubdir[-1] != "/": + self.filterSubdir += "/" + + # walk in test dir to find the tests & sort by name + testfiles = glob.glob(self.filterSubdir + "**/testme.json", recursive=True) + testfiles.sort() + + # loop over each + for testfile in testfiles: + try: + # calc some paths + testfileRelPath = os.path.relpath(testfile, os.path.join(sourceDir, 'test')) + testfilePath = os.path.abspath(os.path.join('test',testfileRelPath)) + testfileDir = os.path.dirname(testfileRelPath) + + # load json & build the inner test combinations + with open(testfilePath, 'r') as fp: + test = json.load(fp) + idefixTestGenerator=IdefixDirTestGenerator(testfilePath, testfileDir) + if 'namings' in test: + namings = test['namings'] + autoExtracted = idefixTestGenerator.extractNamingParameters(test['variants']) + self._validateNaming(namings, autoExtracted, testfilePath) + else: + namings = idefixTestGenerator.extractNamingParameters(test['variants']) + + # required to simplify the algos later, if var is listed as variable, we need to loop over it. + self._makeVariableArgAsList(namings, test['variants']) + + # gen + result += idefixTestGenerator.genTestConfigs(namings, test['variants'], test.get('when', {})) + except Exception as e: + raise Exception(f"Fail to generate tests from {testfileRelPath} : {e}") + + # ok + return result + + def run(self, config: dict) -> None: + # clone before modify to not modity for caller + config = copy.deepcopy(config) + + # print config + print("***************************************************") + print(json.dumps(config, indent='\t')) + print("***************************************************") + + # extract some infos for local usage + testfile = config["testfile"] + dumpname = config['dumpname'] + testname = config["testname"] + tolerance = config.get("tolerance", 0) + definitionFile = config.get("definitionFile", "") + standardTest = config.get("standardTest", True) + nonRegressionTest = config.get("nonRegressionTest", True) + nonRegressionTestIni = config.get("nonRegressionTestIni", None) + problemDir = os.path.dirname(testfile) + + # cleanup some keyword not handled at the + # level of idx_test so we don't perturbate it + del config['dumpname'] + if 'definitionFile' in config: + del config['definitionFile'] + if 'tolerance' in config: + del config['tolerance'] + if 'standardTest' in config: + del config['standardTest'] + if 'nonRegressionTest' in config: + del config['nonRegressionTest'] + if 'nonRegressionTestIni' in config: + del config['nonRegressionTestIni'] + + # if switch from test, rebuild the runner (a runner make for one dir) + if self.currentTestFile != testfile: + self.currentTestRunner = tst.idfxTest(testfile, name=testname) + self.currentTestFile = testfile + + # run + with moveInDir(problemDir): + self._runNonRegression(dumpname, config['ini'], config, tolerance=tolerance, definitionFile=definitionFile, standardTest=standardTest, nonReg=nonRegressionTest, nonRegIni=nonRegressionTestIni) + + def _runNonRegression(self, dumpname, ini, config_override, tolerance=0, definitionFile="", nonReg=True, nonRegIni=None, standardTest=True, first_run_ini=None,first_run_dumpname=None,configure_and_compile=True): + if 'multirun' in config_override: + self._runNonRegMultirun(dumpname, ini, config_override, tolerance=tolerance, nonReg=nonReg, nonRegIni=nonRegIni, standardTest=standardTest, configure_and_compile=configure_and_compile, definitionFile=definitionFile, first_run_ini=first_run_ini, first_run_dumpname=first_run_dumpname) + else: + # single basic run + self._runNonRegSingleRun(dumpname, ini, config_override, tolerance=tolerance, nonReg=nonReg, standardTest=standardTest, configure_and_compile=configure_and_compile, definitionFile=definitionFile, first_run_ini=first_run_ini, first_run_dumpname=first_run_dumpname) + + def _runNonRegMultirun(self, dumpname, ini, config_override, tolerance=0, definitionFile="", nonReg=True, nonRegIni=None, standardTest=True, first_run_ini=None,first_run_dumpname=None,configure_and_compile=True): + # check + assert 'multirun' in config_override + + # loop over runs + for run in config_override['multirun']: + # copy config + run_config = copy.deepcopy(config_override) + + # patch a bit + del run_config['multirun'] + run_config.update(run) + nonReg=run_config.get('nonRegressionTest', nonReg) + dumpname=run_config.get('dumpname', dumpname) + if 'nonRegressionTest' in run_config: + del run_config['nonRegressionTest'] + standardTest=run_config.get('standardTest', standardTest) + if 'standardTest' in run_config: + del run_config['standardTest'] + + # make single run + self._runNonRegSingleRun(dumpname, run_config['ini'], run_config, definitionFile=definitionFile, tolerance=tolerance, nonReg=nonReg, nonRegIni=nonRegIni, standardTest=standardTest) + + def _runNonRegSingleRun(self, dumpname, ini, config_override, tolerance=0, definitionFile="", nonReg=True, nonRegIni=None, standardTest=True, first_run_ini=None,first_run_dumpname=None,configure_and_compile=True): + # build the runner + idefixTest = self.currentTestRunner + + # handle special override which should not go into + # idefixTest because it is not supported by the idfx_test layer. + config_override = copy.deepcopy(config_override) + nonRegIni = config_override.get("nonRegressionTestIni", nonRegIni) + if 'nonRegressionTestIni' in config_override: + del config_override['nonRegressionTestIni'] + + # apply config + idefixTest.applyConfig(config_override) + + # recompile if needed + if configure_and_compile: + idefixTest.configure(override=config_override, definitionFile=definitionFile) + idefixTest.compile() + + if first_run_ini: + idefixTest.run(inputFile=first_run_ini) + if nonReg: + idefixTest.nonRegressionTest(filename=first_run_dumpname, tolerance=tolerance) + + # Test the restart option + file_mtime={} + if not idefixTest.fake: + for file in idefixTest.restart_no_overwrite: + file_mtime[file] = os.path.getmtime(file) + + # restart + if idefixTest.restart: + restart = 1 + else: + restart = -1 + + # run + idefixTest.run(inputFile=ini, restart=restart) + + # regen ref if needed + if idefixTest.init: + idefixTest.makeReference(filename=dumpname) + + # check outputs + if standardTest: + idefixTest.standardTest() + if nonReg: + if nonRegIni: + idefixTest.inifile = nonRegIni + idefixTest.nonRegressionTest(filename=dumpname, tolerance=tolerance) + + # check that we didn't overrite the file during the restart + if not idefixTest.fake: + for file in idefixTest.restart_no_overwrite: + assert file_mtime[file] == os.path.getmtime(file), f"Dump file {file} was overwritten on restart" + + def main(self, all: bool = False): + if all: + sys.argv.append("-all") + idefixTest = tst.idfxTest(self.parentScritFile, name="main") + os.environ["IDEFIX_TEST_FILTER_SUBDIR"] = idefixTest.filterSubdir + + if idefixTest.all: + pytest.main(['-v', '--no-header', '--junit-xml=idefix-tests.junit.xml', '--tb=short'] + idefixTest.remainingArgs + [self.parentScritFile]) + else: + assert False, "Not yet supported !" + #elif self.check: + # idefixTest.checkOnly(filename=dumpname, tolerance=tolerance) + #else: + # for ini in ini_list: + # self.runNonRegression(dumpname, ini, {}, tolerance=tolerance) diff --git a/pytools/pytest.ini b/pytools/pytest.ini new file mode 100644 index 000000000..25e73fa4e --- /dev/null +++ b/pytools/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files=test_*.py diff --git a/pytools/tests/__init__.py b/pytools/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pytools/tests/test/pb1/testme.json b/pytools/tests/test/pb1/testme.json new file mode 100644 index 000000000..b82d1bcba --- /dev/null +++ b/pytools/tests/test/pb1/testme.json @@ -0,0 +1,11 @@ +{ + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] +} diff --git a/pytools/tests/test/pb2/testme.json b/pytools/tests/test/pb2/testme.json new file mode 100644 index 000000000..f7b8b1b5b --- /dev/null +++ b/pytools/tests/test/pb2/testme.json @@ -0,0 +1,18 @@ +{ + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + }, + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": false, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] +} diff --git a/pytools/tests/test_idfx_test_gen.py b/pytools/tests/test_idfx_test_gen.py new file mode 100644 index 000000000..fc4f6261e --- /dev/null +++ b/pytools/tests/test_idfx_test_gen.py @@ -0,0 +1,208 @@ +##################################################################################### +# Idefix MHD astrophysical code +# Copyright(C) Sébastien Valat +# and other code contributors +# Licensed under CeCILL 2.1 License, see COPYING for more information +##################################################################################### + +from ..idfx_test_gen import IdefixDirTestGenerator +import pytest + +def test_extractNamingParameters(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + lst = gen.extractNamingParameters([ + { + "dec": [1,2,3], + "a": 10, + "b": 11, + "c": [True, False] + }, + { + "dec": [1,2,4], + "a": 10, + "b": 12, + "d": [True, False] + }, + ]) + + # valid + assert lst == 'b,c,d' + +def test_genNextLevelCombinations(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # init with single element + core = {} + init = [core] + + # gen first set of combinations + lst1 = gen._genNextLevelCombinations(init, "mpi", [True, False]) + assert lst1 == [ + {"mpi": True}, + {"mpi": False}, + ] + + # gen first second set of combinations + lst3 = gen._genNextLevelCombinations(lst1, "name", ["a", "b"]) + assert lst3 == [ + {"mpi": True, "name": "a"}, + {"mpi": True, "name": "b"}, + {"mpi": False, "name": "a"}, + {"mpi": False, "name": "b"}, + ] + +def test_genOneConfigSeries(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # gen + result = gen._genOneConfigSeries("mpi,name", { + "mpi": [True, False], + "name": ["a", "b"] + }) + + # check + assert result == [ + { + 'mpi': True, + 'name': 'a', + },{ + 'mpi': True, + 'name': 'b', + },{ + 'mpi': False, + 'name': 'a', + },{ + 'mpi': False, + 'name': 'b', + }, + ] + +def test_matchWhenClause_single(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # calls + assert gen._matchWhenClause({"a":10, "b": 11}, {"a":10}) == True + assert gen._matchWhenClause({"a":10, "b": 11}, {"a":11}) == False + assert gen._matchWhenClause({"a":10, "b": 11}, {"b":11}) == True + assert gen._matchWhenClause({"a":10, "b": 11}, {"b":10}) == False + +def test_matchWhenClause_and(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # calls + assert gen._matchWhenClause({"a":10, "b": 11}, {"a":10, "b":11}) == True + assert gen._matchWhenClause({"a":10, "b": 11}, {"a":11, "b":11}) == False + +def test_applyWhen_none(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # call when clause + res = gen._applyWhen({ + "a": 10, + "b": 11 + }, when={}) + + # check result + assert res == {"a": 10, "b": 11} + +def test_applyWhen_single_apply_yes(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # call when clause + res = gen._applyWhen({ + "a": 10, + "b": 11 + }, when={ + "conditions": { + "a": 10, + }, + "apply": { + "b": 12 + } + }) + + # check result + assert res == {"a": 10, "b": 12} + +def test_applyWhen_single_apply_no(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # call when clause + res = gen._applyWhen({ + "a": 10, + "b": 11 + }, when={ + "conditions": { + "a": 9, + }, + "apply": { + "b": 12 + } + }) + + # check result + assert res == {"a": 10, "b": 11} + +def test_applyWhen_list_apply_yes(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # call when clause + res = gen._applyWhen({ + "a": 10, + "b": 11 + }, when=[ + { + "conditions": { + "a": 10, + }, + "apply": { + "b": 12 + } + }, + { + "conditions": { + "a": 13, + }, + "apply": { + "b": 13 + } + } + ]) + + # check result + assert res == {"a": 10, "b": 12} + +def test_gen_full(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # gen + result = gen.genTestConfigs("mpi,name", { + "mpi": [True, False], + "name": ["a", "b"] + }, { + "conditions": { + "mpi": True + }, + "apply": { + "extra": 10, + } + }) + + # check + assert result == [ + pytest.param({'mpi': True, 'name': 'a', 'testfile': __file__, 'testname': 'unit-test', 'extra': 10}, id='unit-test-mpi-a'), + pytest.param({'mpi': True, 'name': 'b', 'testfile': __file__, 'testname': 'unit-test', 'extra': 10}, id='unit-test-mpi-b'), + pytest.param({'mpi': False, 'name': 'a', 'testfile': __file__, 'testname': 'unit-test'}, id='unit-test-a'), + pytest.param({'mpi': False, 'name': 'b', 'testfile': __file__, 'testname': 'unit-test'}, id='unit-test-b'), + ] diff --git a/pytools/tests/test_idfx_test_run.py b/pytools/tests/test_idfx_test_run.py new file mode 100644 index 000000000..2aa788416 --- /dev/null +++ b/pytools/tests/test_idfx_test_run.py @@ -0,0 +1,88 @@ +##################################################################################### +# Idefix MHD astrophysical code +# Copyright(C) Sébastien Valat +# and other code contributors +# Licensed under CeCILL 2.1 License, see COPYING for more information +##################################################################################### + +from ..idfx_test_run import IdexPytestRunner +import pytest +import os + +def test_genTests(): + # build runner + runner = IdexPytestRunner(__file__) + + # dir + dir = os.path.dirname(__file__) + + # generate + result = runner.genTests() + assert result == [ + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': True, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix.ini', + 'testfile': dir + '/test/pb1/testme.json', + 'testname': 'pb1' + }, marks=(), id='pb1-idefix.ini' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': True, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix-implicit.ini', + 'testfile': dir + '/test/pb1/testme.json', + 'testname': 'pb1' + }, marks=(), id='pb1-idefix-implicit.ini' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': True, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix.ini', + 'testfile': dir + '/test/pb2/testme.json', + 'testname': 'pb2' + }, marks=(), id='pb2-idefix.ini-noplot' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': True, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix-implicit.ini', + 'testfile': dir + '/test/pb2/testme.json', + 'testname': 'pb2' + }, marks=(), id='pb2-idefix-implicit.ini-noplot' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': False, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix.ini', + 'testfile': dir + '/test/pb2/testme.json', + 'testname': 'pb2' + }, marks=(), id='pb2-idefix.ini' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': False, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix-implicit.ini', + 'testfile': dir + '/test/pb2/testme.json', + 'testname': 'pb2' + }, marks=(), id='pb2-idefix-implicit.ini' + ), + ] \ No newline at end of file diff --git a/test.py b/test.py new file mode 100755 index 000000000..6ded87fa0 --- /dev/null +++ b/test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +import os +import sys +import pytest + +# set IDEFIX_DIR +source_dir = os.path.dirname(__file__) +os.environ["IDEFIX_DIR"] = source_dir + +# idefix test class +sys.path.append(source_dir) +from pytools.idfx_test_run import IdexPytestRunner + +# should be global so it remember state (last build) and avoids +# to build for each run if we just changed the ini file and run options. +gblIdefixPytestRunner = IdexPytestRunner(__file__) + +# define the pytest test +@pytest.mark.parametrize("config", gblIdefixPytestRunner.genTests()) +def test_idefix_build_run_check(config): + gblIdefixPytestRunner.run(config) + +# if called directly as a script +if __name__ == "__main__": + gblIdefixPytestRunner.main(all=True) diff --git a/test/python_requirements.txt b/test/python_requirements.txt index 77ce6e9c2..9cf2383a2 100644 --- a/test/python_requirements.txt +++ b/test/python_requirements.txt @@ -8,3 +8,7 @@ scipy>=1.2.3 # note that no version of inifix supports Python older than 3.6 inifix>=0.11.2 + +# To run the test suite, we can use pytest, mostly any version +# 6.0 is the one available on system available on debian-11 +pytest >= 6.0 From 95734831c937e2bce211406c35aba8ba739a9be5 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Wed, 18 Mar 2026 13:50:45 +0100 Subject: [PATCH 03/12] test: port all the idefix tests to the testme.json new semantic for test.py --- test/Dust/DustEnergy/testme.json | 12 +++ test/Dust/DustyShock/testme.json | 11 +++ test/Dust/DustyWave/testme.json | 12 +++ test/HD/FargoPlanet/testme.json | 25 +++++++ test/HD/MachReflection/testme.json | 15 ++++ test/HD/SedovBlastWave/testme.json | 28 +++++++ test/HD/ShearingBox/testme.json | 15 ++++ test/HD/ViscousDisk/testme.json | 15 ++++ test/HD/ViscousFlowPastCylinder/testme.json | 23 ++++++ test/HD/sod-iso/testme.json | 33 +++++++++ test/HD/sod/testme.json | 33 +++++++++ test/HD/thermalDiffusion/testme.json | 15 ++++ test/MHD/AmbipolarCshock/testme.json | 14 ++++ test/MHD/AmbipolarCshock3D/testme.json | 35 +++++++++ test/MHD/AxisFluxTube/testme.json | 16 ++++ test/MHD/Coarsening/testme.json | 12 +++ test/MHD/FargoMHDSpherical/testme.json | 17 +++++ test/MHD/HallWhistler/testme.json | 15 ++++ test/MHD/LinearWaveTest/testme.json | 24 ++++++ test/MHD/MTI/testme.json | 15 ++++ test/MHD/OrszagTang/testme.json | 74 +++++++++++++++++++ test/MHD/OrszagTang3D/testme.json | 42 +++++++++++ test/MHD/ResistiveAlfvenWave/testme.json | 22 ++++++ test/MHD/ShearingBox/testme.json | 15 ++++ test/MHD/clessTDiffusion/testme.json | 15 ++++ test/MHD/sod-iso/testme.json | 43 +++++++++++ test/MHD/sod/testme.json | 33 +++++++++ test/MHD/sphBragTDiffusion/testme.json | 15 ++++ test/MHD/sphBragViscosity/testme.json | 16 ++++ test/Planet/Planet3Body/testme.json | 15 ++++ test/Planet/PlanetMigration2D/testme.json | 16 ++++ test/Planet/PlanetPlanetRK42D/testme.json | 25 +++++++ test/Planet/PlanetSpiral2D/testme.json | 16 ++++ test/Planet/PlanetTorque3D/testme.json | 16 ++++ test/Planet/PlanetsIsActiveRK52D/testme.json | 17 +++++ test/SelfGravity/DustyCollapse/testme.json | 16 ++++ test/SelfGravity/JeansInstability/testme.json | 17 +++++ test/SelfGravity/RandomSphere/testme.json | 14 ++++ .../RandomSphereCartesian/testme.json | 12 +++ test/SelfGravity/UniformCollapse/testme.json | 16 ++++ test/utils/columnDensity/testme.json | 13 ++++ test/utils/dumpImage/testme.json | 13 ++++ test/utils/lookupTable/testme.json | 12 +++ 43 files changed, 878 insertions(+) create mode 100644 test/Dust/DustEnergy/testme.json create mode 100644 test/Dust/DustyShock/testme.json create mode 100644 test/Dust/DustyWave/testme.json create mode 100644 test/HD/FargoPlanet/testme.json create mode 100644 test/HD/MachReflection/testme.json create mode 100644 test/HD/SedovBlastWave/testme.json create mode 100644 test/HD/ShearingBox/testme.json create mode 100644 test/HD/ViscousDisk/testme.json create mode 100644 test/HD/ViscousFlowPastCylinder/testme.json create mode 100644 test/HD/sod-iso/testme.json create mode 100644 test/HD/sod/testme.json create mode 100644 test/HD/thermalDiffusion/testme.json create mode 100644 test/MHD/AmbipolarCshock/testme.json create mode 100644 test/MHD/AmbipolarCshock3D/testme.json create mode 100644 test/MHD/AxisFluxTube/testme.json create mode 100644 test/MHD/Coarsening/testme.json create mode 100644 test/MHD/FargoMHDSpherical/testme.json create mode 100644 test/MHD/HallWhistler/testme.json create mode 100644 test/MHD/LinearWaveTest/testme.json create mode 100644 test/MHD/MTI/testme.json create mode 100644 test/MHD/OrszagTang/testme.json create mode 100644 test/MHD/OrszagTang3D/testme.json create mode 100644 test/MHD/ResistiveAlfvenWave/testme.json create mode 100644 test/MHD/ShearingBox/testme.json create mode 100644 test/MHD/clessTDiffusion/testme.json create mode 100644 test/MHD/sod-iso/testme.json create mode 100644 test/MHD/sod/testme.json create mode 100644 test/MHD/sphBragTDiffusion/testme.json create mode 100644 test/MHD/sphBragViscosity/testme.json create mode 100644 test/Planet/Planet3Body/testme.json create mode 100644 test/Planet/PlanetMigration2D/testme.json create mode 100644 test/Planet/PlanetPlanetRK42D/testme.json create mode 100644 test/Planet/PlanetSpiral2D/testme.json create mode 100644 test/Planet/PlanetTorque3D/testme.json create mode 100644 test/Planet/PlanetsIsActiveRK52D/testme.json create mode 100644 test/SelfGravity/DustyCollapse/testme.json create mode 100644 test/SelfGravity/JeansInstability/testme.json create mode 100644 test/SelfGravity/RandomSphere/testme.json create mode 100644 test/SelfGravity/RandomSphereCartesian/testme.json create mode 100644 test/SelfGravity/UniformCollapse/testme.json create mode 100644 test/utils/columnDensity/testme.json create mode 100644 test/utils/dumpImage/testme.json create mode 100644 test/utils/lookupTable/testme.json diff --git a/test/Dust/DustEnergy/testme.json b/test/Dust/DustEnergy/testme.json new file mode 100644 index 000000000..c980b42cc --- /dev/null +++ b/test/Dust/DustEnergy/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 0 + } + ] +} diff --git a/test/Dust/DustyShock/testme.json b/test/Dust/DustyShock/testme.json new file mode 100644 index 000000000..b82d1bcba --- /dev/null +++ b/test/Dust/DustyShock/testme.json @@ -0,0 +1,11 @@ +{ + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] +} diff --git a/test/Dust/DustyWave/testme.json b/test/Dust/DustyWave/testme.json new file mode 100644 index 000000000..6f5c0b9ac --- /dev/null +++ b/test/Dust/DustyWave/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] +} diff --git a/test/HD/FargoPlanet/testme.json b/test/HD/FargoPlanet/testme.json new file mode 100644 index 000000000..1e38e4ddb --- /dev/null +++ b/test/HD/FargoPlanet/testme.json @@ -0,0 +1,25 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini", "idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": [2, 2], + "tolerance": 1e-13 + } + ], + "when": [ + { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply": { + "tolerance": 1e-12 + } + } + ] +} diff --git a/test/HD/MachReflection/testme.json b/test/HD/MachReflection/testme.json new file mode 100644 index 000000000..67256aafb --- /dev/null +++ b/test/HD/MachReflection/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": [2, 2], + "tolerance": 0 + } + ] +} diff --git a/test/HD/SedovBlastWave/testme.json b/test/HD/SedovBlastWave/testme.json new file mode 100644 index 000000000..24f3d1c42 --- /dev/null +++ b/test/HD/SedovBlastWave/testme.json @@ -0,0 +1,28 @@ +{ + "namings": "definitionFile,ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "definitionFile": "definitions.hpp", + "ini": ["idefix.ini"], + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": true, + "dec": [2, 2, 2 ], + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "definitionFile": "definitions-spherical.hpp", + "ini": ["idefix-spherical.ini"], + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": true, + "dec": [2, 2, 2 ], + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/HD/ShearingBox/testme.json b/test/HD/ShearingBox/testme.json new file mode 100644 index 000000000..3e473d0ba --- /dev/null +++ b/test/HD/ShearingBox/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-fargo.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": false, + "dec": ["2","1","2"], + "tolerance": 1e-15 + } + ] +} diff --git a/test/HD/ViscousDisk/testme.json b/test/HD/ViscousDisk/testme.json new file mode 100644 index 000000000..df8c19d86 --- /dev/null +++ b/test/HD/ViscousDisk/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": false, + "dec": ["2","1","2"], + "tolerance": 3e-15 + } + ] +} diff --git a/test/HD/ViscousFlowPastCylinder/testme.json b/test/HD/ViscousFlowPastCylinder/testme.json new file mode 100644 index 000000000..7b3763436 --- /dev/null +++ b/test/HD/ViscousFlowPastCylinder/testme.json @@ -0,0 +1,23 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 3e-14 + } + ], + "when": { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply": { + "tolerance": 1e-8 + } + } +} diff --git a/test/HD/sod-iso/testme.json b/test/HD/sod-iso/testme.json new file mode 100644 index 000000000..5016415d6 --- /dev/null +++ b/test/HD/sod-iso/testme.json @@ -0,0 +1,33 @@ +{ + "namings": "ini,single,reconstruction", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "vectPot": false, + "noplot": true, + "reconstruction": [2, 3], + "single": [false], + "mpi": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk3.ini","idefix-hllc-rk3.ini"], + "vectPot": false, + "noplot": true, + "reconstruction": [4], + "single": [false], + "mpi": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "vectPot": false, + "noplot": true, + "reconstruction": [2], + "single": [true], + "mpi": false, + "tolerance": 0 + } + ] +} diff --git a/test/HD/sod/testme.json b/test/HD/sod/testme.json new file mode 100644 index 000000000..9128c08d4 --- /dev/null +++ b/test/HD/sod/testme.json @@ -0,0 +1,33 @@ +{ + "namings": "ini,single,reconstruction", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "noplot": true, + "vectPot": false, + "single": [false], + "reconstruction": [2,3], + "mpi": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk3.ini","idefix-hllc-rk3.ini"], + "noplot": true, + "vectPot": false, + "single": [false], + "reconstruction": [4], + "mpi": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "noplot": true, + "vectPot": false, + "reconstruction": [2], + "mpi": false, + "single": [true], + "tolerance": 0 + } + ] +} diff --git a/test/HD/thermalDiffusion/testme.json b/test/HD/thermalDiffusion/testme.json new file mode 100644 index 000000000..2e89ad747 --- /dev/null +++ b/test/HD/thermalDiffusion/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": false, + "dec": ["2","1","2"], + "tolerance": 0 + } + ] +} diff --git a/test/MHD/AmbipolarCshock/testme.json b/test/MHD/AmbipolarCshock/testme.json new file mode 100644 index 000000000..2eb2ac51f --- /dev/null +++ b/test/MHD/AmbipolarCshock/testme.json @@ -0,0 +1,14 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": false, + "tolerance": 0 + } + ] +} diff --git a/test/MHD/AmbipolarCshock3D/testme.json b/test/MHD/AmbipolarCshock3D/testme.json new file mode 100644 index 000000000..fdcb11a76 --- /dev/null +++ b/test/MHD/AmbipolarCshock3D/testme.json @@ -0,0 +1,35 @@ +{ + "namings": "ini,mpi,vectPot", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [false], + "vectPot": [false, true], + "dec": ["2","1","1"], + "tolerance": 3e-14 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [true], + "vectPot": [false], + "dec": ["2","1","1"], + "tolerance": 3e-14 + } + ], + "when": { + "conditions": { + "ini": "idefix-rkl.ini", + "mpi": true + }, + "apply": { + "tolerance": 2e-10 + } + } +} diff --git a/test/MHD/AxisFluxTube/testme.json b/test/MHD/AxisFluxTube/testme.json new file mode 100644 index 000000000..781d4f7b3 --- /dev/null +++ b/test/MHD/AxisFluxTube/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-coarsening.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "vectPot": false, + "standardTest": false, + "tolerance": 1e-14 + } + ] +} diff --git a/test/MHD/Coarsening/testme.json b/test/MHD/Coarsening/testme.json new file mode 100644 index 000000000..5b38ffebb --- /dev/null +++ b/test/MHD/Coarsening/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini","idefix-x2.ini","idefix-x3.ini"], + "noplot": true, + "mpi": [false,true], + "tolerance": 0 + } + ] +} diff --git a/test/MHD/FargoMHDSpherical/testme.json b/test/MHD/FargoMHDSpherical/testme.json new file mode 100644 index 000000000..72d93f493 --- /dev/null +++ b/test/MHD/FargoMHDSpherical/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,mpi,vectPot", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "vectPot": [false, true], + "mpi": [false, true], + "dec": ["2","2","2"], + "standardTest": false, + "tolerance": 1e-14 + } + ] +} diff --git a/test/MHD/HallWhistler/testme.json b/test/MHD/HallWhistler/testme.json new file mode 100644 index 000000000..cc71c4bf7 --- /dev/null +++ b/test/MHD/HallWhistler/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "tolerance": 1e-15 + } + ] +} diff --git a/test/MHD/LinearWaveTest/testme.json b/test/MHD/LinearWaveTest/testme.json new file mode 100644 index 000000000..9a65e72dc --- /dev/null +++ b/test/MHD/LinearWaveTest/testme.json @@ -0,0 +1,24 @@ +{ + "namings": "ini,reconstruction,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix-fast.ini","idefix-slow.ini","idefix-alfven.ini","idefix-entropy.ini"], + "noplot": true, + "single": false, + "reconstruction": [2], + "mpi": [false,true], + "dec": ["2","2","2"], + "tolerance": 1e-14 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-fast.ini","idefix-slow.ini","idefix-alfven.ini","idefix-entropy.ini"], + "noplot": true, + "single": false, + "reconstruction": [3, 4], + "mpi": [false], + "dec": ["2","2","2"], + "tolerance": 1e-14 + } + ] +} diff --git a/test/MHD/MTI/testme.json b/test/MHD/MTI/testme.json new file mode 100644 index 000000000..3bd669cca --- /dev/null +++ b/test/MHD/MTI/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini","idefix-sl.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "tolerance": 0 + } + ] +} diff --git a/test/MHD/OrszagTang/testme.json b/test/MHD/OrszagTang/testme.json new file mode 100644 index 000000000..3128b3239 --- /dev/null +++ b/test/MHD/OrszagTang/testme.json @@ -0,0 +1,74 @@ +{ + "namings": "ini,reconstruction,mpi,single,vectPot", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": [ + "idefix.ini", + "idefix-hll.ini", + "idefix-hlld-arithmetic.ini", + "idefix-hlld-hll.ini", + "idefix-hlld-hlld.ini", + "idefix-hlld-uct0.ini", + "idefix-hlld.ini", + "idefix-tvdlf.ini" + ], + "noplot": true, + "vectPot": [false], + "single": false, + "reconstruction": [2,3,4], + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 1e-12, + "standardTest": false + },{ + "dumpname": "dump.0001.dmp", + "ini": [ + "idefix.ini", + "idefix-hll.ini", + "idefix-hlld-arithmetic.ini", + "idefix-hlld-hll.ini", + "idefix-hlld-hlld.ini", + "idefix-hlld-uct0.ini", + "idefix-hlld.ini", + "idefix-tvdlf.ini" + ], + "noplot": true, + "vectPot": [false], + "single": true, + "reconstruction": [2], + "mpi": [false], + "dec": ["2","2"], + "tolerance": 1e-12, + "standardTest": false + },{ + "dumpname": "dump.0001.dmp", + "ini": [ + "idefix.ini", + "idefix-hll.ini", + "idefix-hlld-arithmetic.ini", + "idefix-hlld-hll.ini", + "idefix-hlld-hlld.ini", + "idefix-hlld-uct0.ini", + "idefix-hlld.ini", + "idefix-tvdlf.ini" + ], + "noplot": true, + "vectPot": [true], + "single": false, + "reconstruction": [2], + "mpi": [false], + "dec": ["2","2"], + "tolerance": 1e-12, + "standardTest": false + } + ], + "when": { + "conditions": { + "single": true + }, + "apply" : { + "tolerance": 1e-5 + } + } +} diff --git a/test/MHD/OrszagTang3D/testme.json b/test/MHD/OrszagTang3D/testme.json new file mode 100644 index 000000000..867fe18b3 --- /dev/null +++ b/test/MHD/OrszagTang3D/testme.json @@ -0,0 +1,42 @@ +{ + "namings": "ini,single,vectPot,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": [false, true], + "reconstruction": 2, + "single": [false], + "mpi": [false, true], + "dec": ["2","2","2"], + "standardTest": false, + "tolerance": 1e-13, + "multirun": [ + {}, + {"ini": "idefix-checkrestart.ini", "dumpname": "dump.0002.dmp", "nonRegressionTestIni": "idefix.ini"} + ] + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": [false], + "reconstruction": 2, + "mpi": [false, true], + "single": [true], + "dec": ["2","2","2"], + "standardTest": false, + "tolerance": 1e-13, + "multirun": [ + {}, + {"ini": "idefix-checkrestart.ini", "dumpname": "dump.0002.dmp", "nonRegressionTestIni": "idefix.ini"} + ] + } + ], + "when": { + "conditions": { + "single": true + }, + "apply" : { + "tolerance": 1e-6 + } + } +} diff --git a/test/MHD/ResistiveAlfvenWave/testme.json b/test/MHD/ResistiveAlfvenWave/testme.json new file mode 100644 index 000000000..56ae122ec --- /dev/null +++ b/test/MHD/ResistiveAlfvenWave/testme.json @@ -0,0 +1,22 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "tolerance": 1e-14 + } + ], + "when": { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply" : { + "tolerance": 1e-10 + } + } +} diff --git a/test/MHD/ShearingBox/testme.json b/test/MHD/ShearingBox/testme.json new file mode 100644 index 000000000..71ae0ab0d --- /dev/null +++ b/test/MHD/ShearingBox/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-fargo.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","1","2"], + "tolerance": 1e-14 + } + ] +} diff --git a/test/MHD/clessTDiffusion/testme.json b/test/MHD/clessTDiffusion/testme.json new file mode 100644 index 000000000..d615696f5 --- /dev/null +++ b/test/MHD/clessTDiffusion/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": false, + "vectPot": false, + "tolerance": 2e-15 + } + ] +} diff --git a/test/MHD/sod-iso/testme.json b/test/MHD/sod-iso/testme.json new file mode 100644 index 000000000..9f5534fbc --- /dev/null +++ b/test/MHD/sod-iso/testme.json @@ -0,0 +1,43 @@ +{ + "namings": "ini,reconstruction,single", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"], + "vectPot": false, + "single": false, + "reconstruction": [2,3], + "mpi": false, + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk3.ini","idefix-hlld-rk3.ini"], + "vectPot": false, + "single": false, + "reconstruction": [4], + "mpi": false, + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"], + "vectPot": false, + "reconstruction": [2], + "mpi": false, + "single": true, + "standardTest": false, + "tolerance": 0 + } + ], + "when": [ + { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply": { + "tolerance": 1e-10 + } + } + ] +} diff --git a/test/MHD/sod/testme.json b/test/MHD/sod/testme.json new file mode 100644 index 000000000..1501e3049 --- /dev/null +++ b/test/MHD/sod/testme.json @@ -0,0 +1,33 @@ +{ + "namings": "ini,reconstruction,single", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"], + "vectPot": false, + "single": false, + "reconstruction": [2,3], + "mpi": false, + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk3.ini","idefix-hlld-rk3.ini"], + "vectPot": false, + "single": false, + "reconstruction": 4, + "mpi": false, + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"], + "vectPot": false, + "reconstruction": 2, + "mpi": false, + "single": true, + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/MHD/sphBragTDiffusion/testme.json b/test/MHD/sphBragTDiffusion/testme.json new file mode 100644 index 000000000..c4cb9a334 --- /dev/null +++ b/test/MHD/sphBragTDiffusion/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "tolerance": 2e-15 + } + ] +} diff --git a/test/MHD/sphBragViscosity/testme.json b/test/MHD/sphBragViscosity/testme.json new file mode 100644 index 000000000..900a34a77 --- /dev/null +++ b/test/MHD/sphBragViscosity/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "standardTest": false, + "tolerance": 1e-15 + } + ] +} diff --git a/test/Planet/Planet3Body/testme.json b/test/Planet/Planet3Body/testme.json new file mode 100644 index 000000000..2b2fad202 --- /dev/null +++ b/test/Planet/Planet3Body/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "tolerance": 2e-11 + } + ] +} diff --git a/test/Planet/PlanetMigration2D/testme.json b/test/Planet/PlanetMigration2D/testme.json new file mode 100644 index 000000000..6e1e08537 --- /dev/null +++ b/test/Planet/PlanetMigration2D/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 1e-13 + } + ] +} diff --git a/test/Planet/PlanetPlanetRK42D/testme.json b/test/Planet/PlanetPlanetRK42D/testme.json new file mode 100644 index 000000000..0181b072f --- /dev/null +++ b/test/Planet/PlanetPlanetRK42D/testme.json @@ -0,0 +1,25 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0002.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 1e-13, + "multirun": [ + { + "nonRegressionTest": true, + "standardTest": true + },{ + "restart": true, + "restart_no_overwrite": ["dump.0001.dmp", "data.0005.vtk"] + } + ] + } + ] +} diff --git a/test/Planet/PlanetSpiral2D/testme.json b/test/Planet/PlanetSpiral2D/testme.json new file mode 100644 index 000000000..6e1e08537 --- /dev/null +++ b/test/Planet/PlanetSpiral2D/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 1e-13 + } + ] +} diff --git a/test/Planet/PlanetTorque3D/testme.json b/test/Planet/PlanetTorque3D/testme.json new file mode 100644 index 000000000..085fb6c0f --- /dev/null +++ b/test/Planet/PlanetTorque3D/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2","2"], + "tolerance": 1e-13 + } + ] +} diff --git a/test/Planet/PlanetsIsActiveRK52D/testme.json b/test/Planet/PlanetsIsActiveRK52D/testme.json new file mode 100644 index 000000000..30848aecd --- /dev/null +++ b/test/Planet/PlanetsIsActiveRK52D/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk4.ini", "idefix-rk5.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2"], + "nonRegressionTest": false, + "tolerance": 1e-13 + } + ] +} diff --git a/test/SelfGravity/DustyCollapse/testme.json b/test/SelfGravity/DustyCollapse/testme.json new file mode 100644 index 000000000..8afbcdb6d --- /dev/null +++ b/test/SelfGravity/DustyCollapse/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "nonRegressionTest": false, + "tolerance": 1e-13 + } + ] +} diff --git a/test/SelfGravity/JeansInstability/testme.json b/test/SelfGravity/JeansInstability/testme.json new file mode 100644 index 000000000..bf4819389 --- /dev/null +++ b/test/SelfGravity/JeansInstability/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-cg.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2"], + "nonRegressionTest": false, + "tolerance": 1e-13 + } + ] +} diff --git a/test/SelfGravity/RandomSphere/testme.json b/test/SelfGravity/RandomSphere/testme.json new file mode 100644 index 000000000..d5cda3135 --- /dev/null +++ b/test/SelfGravity/RandomSphere/testme.json @@ -0,0 +1,14 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-cg.ini","idefix-minres.ini"], + "noplot": true, + "mpi": [false, true], + "dec": ["2","2","1"], + "nonRegressionTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/SelfGravity/RandomSphereCartesian/testme.json b/test/SelfGravity/RandomSphereCartesian/testme.json new file mode 100644 index 000000000..85f0f3889 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-cg.ini","idefix-minres.ini","idefix-jacobi.ini"], + "noplot": true, + "nonRegressionTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/SelfGravity/UniformCollapse/testme.json b/test/SelfGravity/UniformCollapse/testme.json new file mode 100644 index 000000000..3de00c4df --- /dev/null +++ b/test/SelfGravity/UniformCollapse/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": false, + "standardTest": false, + "tolerance": 0, + "nonRegressionTest": false + } + ] +} diff --git a/test/utils/columnDensity/testme.json b/test/utils/columnDensity/testme.json new file mode 100644 index 000000000..718413c4d --- /dev/null +++ b/test/utils/columnDensity/testme.json @@ -0,0 +1,13 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "mpi": [false,true], + "nonRegressionTest": false, + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/utils/dumpImage/testme.json b/test/utils/dumpImage/testme.json new file mode 100644 index 000000000..718413c4d --- /dev/null +++ b/test/utils/dumpImage/testme.json @@ -0,0 +1,13 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "mpi": [false,true], + "nonRegressionTest": false, + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/utils/lookupTable/testme.json b/test/utils/lookupTable/testme.json new file mode 100644 index 000000000..8ea6f6961 --- /dev/null +++ b/test/utils/lookupTable/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini":["idefix.ini"], + "nonRegressionTest": false, + "standardTest": false, + "tolerance": 0 + } + ] +} From 455ef19f1e430fea6e79b5e53e70713c446bcaea Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Wed, 18 Mar 2026 13:51:30 +0100 Subject: [PATCH 04/12] test: document the semantic of test.py --- doc/source/testing.rst | 12 +- doc/source/testing/testLauncher.rst | 384 ++++++++++++++++++++++++++++ 2 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 doc/source/testing/testLauncher.rst diff --git a/doc/source/testing.rst b/doc/source/testing.rst index 888ac2057..59bd2177a 100644 --- a/doc/source/testing.rst +++ b/doc/source/testing.rst @@ -71,9 +71,17 @@ How tests are driven (testme scripts) Each test directory contains a small Python "testMe" driver that uses the helper Python class documented in the repository: +- See the test launcher documentation: :doc:`test.py ` - See the test helper documentation: :doc:`idfxTest ` -That helper (idfxTest) is responsible for: +The test launcher (test.py) is responsible for: + +- Loading all the test definitions by search all the ``testme.json`` files in the ``test`` + directory. +- Calling the :doc:`idfxTest ` helper to run the particular test. +- Generate reports about success and failures. + +The helper (idfxTest) is responsible for: - parsing TESTME_OPTIONS-like flags (precision, MPI, CUDA, reconstruction, vector potential, etc.), - calling configure / compile / run, @@ -108,10 +116,12 @@ Relevant files - Workflow entry point: .github/workflows/idefix-ci.yml - Reusable jobs: .github/workflows/idefix-ci-jobs.yml +- Test launcher documentation: :doc:`test launcher ` - Test helper documentation: :doc:`idfxTest ` .. toctree:: :maxdepth: 2 :caption: Contents: + testing/testLauncher.rst testing/idfxTest.rst diff --git a/doc/source/testing/testLauncher.rst b/doc/source/testing/testLauncher.rst new file mode 100644 index 000000000..b8d80f005 --- /dev/null +++ b/doc/source/testing/testLauncher.rst @@ -0,0 +1,384 @@ +========================== +Test launcher and reporter +========================== + +Overview +-------- + +The class :doc:`idfxTest ` provides the basement to implement an *Idefix* integration test for validation. +In order to ease launching all the tests, the user might prefer to use directly the ``./test.py`` command at the +root or *Idefix* sources. + +This script will run all the registered variants of *Idefix* and build a report in the terminal. At the +end of the run a standard ``junit.xml`` file is produced. This one can be translated into a browsable +HTML file. + +Depencencies +------------ + +Before using :doc:`idfxTest ` you need to install some Python Depencencies (possibly in a ``virtual env``): + +.. code-block:: shell + + pip install -r test/python_requirements.txt + +Running +------- + +To run the test you can basically : + +.. code-block:: shell + + # Run all tests + ./test.py + + # Run all tests in ./tests/HD + ./test.py -subdir=./tests/HD + + # Select in more details the tests containing the "single" keyword + # See pytest documentation for the exact advanced semantic + ./test.py -subdir=./tests/HD -k single + + # Run in verbose + ./test.py -v + +The result of the execution will be an output like : + +.. code-block:: text + + ============================================ test session starts ============================================= + collected 52 items / 44 deselected / 8 selected + + test.py::test_idefix_build_run_check[HD/sod-iso-idefix.ini-single-reconstruction-2] PASSED [ 12%] + test.py::test_idefix_build_run_check[HD/sod-iso-idefix-hll.ini-single-reconstruction-2] PASSED [ 25%] + test.py::test_idefix_build_run_check[HD/sod-iso-idefix-hllc.ini-single-reconstruction-2] PASSED [ 37%] + test.py::test_idefix_build_run_check[HD/sod-iso-idefix-tvdlf.ini-single-reconstruction-2] PASSED [ 50%] + test.py::test_idefix_build_run_check[HD/sod-idefix.ini-single-reconstruction-2] PASSED [ 62%] + test.py::test_idefix_build_run_check[HD/sod-idefix-hll.ini-single-reconstruction-2] PASSED [ 75%] + test.py::test_idefix_build_run_check[HD/sod-idefix-hllc.ini-single-reconstruction-2] PASSED [ 87%] + test.py::test_idefix_build_run_check[HD/sod-idefix-tvdlf.ini-single-reconstruction-2] PASSED [100%] + + ---------- generated xml file: idefix-tests.junit.xml --------------------------------------------------------- + =============================== 8 passed, 44 deselected in 73.03s (0:01:13) =================================== + +In case of an error, the output of the command will be printed. + +HTML report +----------- + +If you want to to generate an HTML page from the report you can proceed by using the Python package ``junit2html`` : + +.. code-block:: shell + + # install junit2html + pip install junit2html + + # convert the report + junit2html ./idefix-tests.junit.xml ./idefix-tests.junit.html + +Advanced usage of the command +----------------------------- + +Here the options supported by the test script : + +.. code-block:: text + + usage: test.py [-h] [-noplot] [-ploterr] [-cmake CMAKE [CMAKE ...]] [-definitions DEFINITIONS] + [-dec DEC [DEC ...]] [-check] [-cuda] [-intel] [-hip] [-single] [-vectPot] + [-reconstruction RECONSTRUCTION] [-idefixDir IDEFIXDIR] [-mpi] [-all] [-init] + [-Werror] [-ccache] [-restart] [-v] [--help-pytest] [-fake] [-subdir SUBDIR] + + options: + -h, --help show this help message and exit + -noplot disable plotting in standard tests + -ploterr Enable plotting on error in regression tests + -cmake CMAKE [CMAKE ...] + CMake options + -definitions DEFINITIONS + definitions.hpp file + -dec DEC [DEC ...] MPI domain decomposition + -check Only perform regression tests without compilation + -cuda Test on Nvidia GPU using CUDA + -intel Test compiling with Intel OneAPI + -hip Test on AMD GPU using HIP + -single Enable single precision + -vectPot Enable vector potential formulation + -reconstruction RECONSTRUCTION + set reconstruction scheme (2=PLM, 3=LimO3, 4=PPM) + -idefixDir IDEFIXDIR Set directory for idefix source files (default $IDEFIX_DIR) + -mpi Enable MPI + -all Do all test suite (otherwise, just do the test with the current configuration) + -init Reinit reference files for non-regression tests (dangerous!) + -Werror Consider warnings as errors + -ccache Use ccache to reduce the build time over multiple run of the test suite. + -restart Enable creating a restart from a checkpoint. + -v, --verbose Enable verbose mode, by not capturing the output. + --help-pytest Display the options you can transmit directly to pytest in addition to the specific to idefix tests. + -fake Make a fake run by just logging the actions to validate that we generate same command over refactoring. + -subdir SUBDIR Select the test in the given subdir not to run all. + +The script is constructed on top of the ``pytest`` command so you automatically get access +to all the advanced option this command provide. Here a few examples : + +.. code-block:: shell + + # get the pytest help + ./test.py --help-pytest + + # let pytest filtering the tests + ./test.py -k "single and mpi" + + # stop on first failure + ./test.py -x + + # re-run only the last failed tests + ./test.py --lf + +Definition of the tests +----------------------- + +The script is simply searching all the files names ``testme.json`` into the ``test`` directory. +This file describe the combination of parameters to use to produce the list of *Idefix* run to perform. +For a single basic configuration one can use : + +.. code-block:: json + + { + "variants": { + "dumpname": "dump.0001.dmp", + "ini": "idefix.ini", + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "standardTest": false, + "tolerance": 0 + } + } + +Available parameters +-------------------- + +The parameters in the ``variants`` dictionnary correspond to the option supported by the +:doc:`idfxTest ` script to configure the build and run of *Idefix*. + +In addition there is some extra keys which are dedicated to the json interpretation layer : + +.. list-table:: + :header-rows: 1 + + * - Option + - Default + - Description + * - ``dumname`` + - ``dump.0001.dmp`` + - Dump file to use to check the results after the run. + * - ``ini`` + - ``idefix.ini`` + - The configuration file to use. + * - ``tolerance`` + - ``0`` + - The margins to allow when checking the results. + * - ``standardTest`` + - ``true`` + - Runs any Python-based standard tests (e.g., ``testidefix.py``) present in the test directory for additional validation. + * - ``nonRegressionTest`` + - ``true`` + - Compares the output dump file to a reference file using RMSE; fails if the error exceeds the tolerance. + * - ``nonRegressionTestIni`` + - Same than ``ini`` + - When making restart you might want to make the check using the inirial configuration file. + * - ``multirun`` + - ``{}`` + - See the multi-run section below. + +Looping over parameters +----------------------- + +You might want to explore running Idefix within parameter ranges (configuration files, modes). +For this simply list the values you want as a list. The test script will automatically +generate the list of all combinations. + +.. code-block:: json + + { + "variants": { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini"], + "vectPot": [false, true], + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + } + } + +It will automatically produce the tests : + +* HD/sod-iso-idefix.ini +* HD/sod-iso-idefix.ini-vectPot +* HD/sod-iso-idefix.ini-mpi +* HD/sod-iso-idefix.ini-vectPot-mpi +* HD/sod-iso-idefix-hll.ini +* HD/sod-iso-idefix-hll.ini-vectPot +* HD/sod-iso-idefix-hll.ini-mpi +* HD/sod-iso-idefix-hll.ini-vectPot-mpi + +Specific keys +------------- + +There is some keys which are by default some arrays, they will not be considered +as combination rules : + +* ``dec`` +* ``multirun`` +* ``restart_no_overwrite`` +* ``tolerance`` + +Reduce the combinations +----------------------- + +You might not want to see all the combinations but just a few, for this, you +can list several sets as a list. Here using single only on half of the modes. + +.. code-block:: json + + { + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-hll.ini"], + "vectPot": true, + "single": true, + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + } + ] + } + +* HD/sod-iso-idefix.ini +* HD/sod-iso-idefix.ini-mpi +* HD/sod-iso-idefix-hll.ini-single-vectPot +* HD/sod-iso-idefix-hll.ini-single-vectPot-mpi + +Naming the test +--------------- + +If you prefer to see the options appearing in a specific order in the generated test name, +you can provide the key ``namings`` listing as a comma separated list the variables in +the order you want to see them composing the test name. + +.. code-block:: json + + { + "namings": "ini,single,mpi", + "variants": { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini"], + "vectPot": false, + "single": [false, true], + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + } + } + + +By default, the alphabetical order will be used. + +When clauses +------------ + +You can also dynamically override a specific value when a parameter value is selected, +it simplified the writting of some conditions like if you used and IF statement. + +.. code-block:: json + + { + "namings": "ini,single,mpi", + "variants": { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini"], + "vectPot": false, + "single": [false, true], + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + }, + "when": { + "conditions": { + "single": true + }, + "apply": { + "reconstruction": 1 + } + } + } + +In this case, ``reconstrucion`` will be set to 1 when ``single`` equal to ``true``. + +Note that the ``conditions`` field can contains several values which will be treaded +as and AND logical operator. + +You can provide several ``when`` clauses by using a list of them instead of directly +the dictionnary. + +Making multi-run steps +---------------------- + +In order to validate checkpoint restart, or continuing a simulation with dirfferent +tunnings the script supports decribing multi-run configurations. + +They are described like : + +.. code-block:: json + + { + "variants": { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "standardTest": false, + "tolerance": 0, + "dec": [2,2,2], + "multirun": [ + { + + },{ + "mpi": true, + "restart": true, + "restart_no_overwrite": ["dump.0001.dmp", "data.0005.vtk"] + } + ] + }, + } + +Using the idfxTest options +-------------------------- + +As the ``test.py`` uses the class described in :doc:`idfxTest ` it also supports all the +command line options it offers. + +Most usefull might be the enabling of ``ccache`` to reduce the compilation time from one +run to another. + +.. code-block:: shell + + ./test.py -ccache From 5069ab3a9c9ae6c06db477330d2ceb94442c8d6c Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Wed, 18 Mar 2026 13:54:41 +0100 Subject: [PATCH 05/12] test: document -ccache option of idfx_test.py --- doc/source/testing/idfxTest.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/testing/idfxTest.rst b/doc/source/testing/idfxTest.rst index b425d3fe2..96e364465 100644 --- a/doc/source/testing/idfxTest.rst +++ b/doc/source/testing/idfxTest.rst @@ -74,6 +74,9 @@ The constructor parses command-line arguments using ``argparse``. These options * - ``-Werror`` - ``Werror`` - Treat compiler warnings as errors. + * - ``-ccache`` + - ``ccache`` + - Enable usage of ccache to build the tests and reduce the build time. Main Methods ------------ From 55714e2335d1ef12e84f2729c3b81d131545f87b Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Thu, 19 Mar 2026 10:09:32 +0100 Subject: [PATCH 06/12] test: fix some comments --- pytools/idfx_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytools/idfx_test.py b/pytools/idfx_test.py index a87b2de44..9d6327e47 100644 --- a/pytools/idfx_test.py +++ b/pytools/idfx_test.py @@ -121,6 +121,7 @@ def __init__ (self, current_test_file, name=""): help="Make a fake run by just logging the actions to validate that we generate same command over refactoring.", action="store_true") + # this option is not used directly by direct users of idfxTest but by idx_test_gen parser.add_argument("-subdir", default="./test", help="Select the test in the given subdir not to run all.", @@ -308,7 +309,7 @@ def clean(self): shutil.rmtree(self.buildDir) # remove previous build configuration cache in the test itself - # otherwise cmake take in in the build subdir + # otherwise cmake takes it in the build subdir testDirCMakeCache = os.path.join(self.testDir, "CMakeCache.txt") if os.path.exists(testDirCMakeCache): os.unlink(testDirCMakeCache) From 062dbf0b82b8b849eb73714dfb58d3653ddc5431 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Thu, 19 Mar 2026 10:23:08 +0100 Subject: [PATCH 07/12] test: remove the CMakeCache.txt check, it is not required --- pytools/idfx_test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pytools/idfx_test.py b/pytools/idfx_test.py index 9d6327e47..be86a72e8 100644 --- a/pytools/idfx_test.py +++ b/pytools/idfx_test.py @@ -6,6 +6,7 @@ import re import json import pytest +import warnings import numpy as np import matplotlib.pyplot as plt @@ -308,12 +309,6 @@ def clean(self): if os.path.exists(self.buildDir): shutil.rmtree(self.buildDir) - # remove previous build configuration cache in the test itself - # otherwise cmake takes it in the build subdir - testDirCMakeCache = os.path.join(self.testDir, "CMakeCache.txt") - if os.path.exists(testDirCMakeCache): - os.unlink(testDirCMakeCache) - # recreate os.makedirs(self.buildDir, exist_ok=False) From 6952bd1dc8f6c3bafc3015ad4a1dfe1fa58b3e68 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Thu, 19 Mar 2026 11:35:38 +0100 Subject: [PATCH 08/12] test: some fixes in documentation --- doc/source/testing/testLauncher.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/source/testing/testLauncher.rst b/doc/source/testing/testLauncher.rst index b8d80f005..e6039672d 100644 --- a/doc/source/testing/testLauncher.rst +++ b/doc/source/testing/testLauncher.rst @@ -7,9 +7,9 @@ Overview The class :doc:`idfxTest ` provides the basement to implement an *Idefix* integration test for validation. In order to ease launching all the tests, the user might prefer to use directly the ``./test.py`` command at the -root or *Idefix* sources. +root of the *Idefix* sources. -This script will run all the registered variants of *Idefix* and build a report in the terminal. At the +This script will run all the listed variants of *Idefix* and build a report in the terminal. At the end of the run a standard ``junit.xml`` file is produced. This one can be translated into a browsable HTML file. @@ -61,7 +61,7 @@ The result of the execution will be an output like : ---------- generated xml file: idefix-tests.junit.xml --------------------------------------------------------- =============================== 8 passed, 44 deselected in 73.03s (0:01:13) =================================== -In case of an error, the output of the command will be printed. +When seeing an error, the output of the command will be printed at the end. HTML report ----------- @@ -117,7 +117,7 @@ Here the options supported by the test script : -fake Make a fake run by just logging the actions to validate that we generate same command over refactoring. -subdir SUBDIR Select the test in the given subdir not to run all. -The script is constructed on top of the ``pytest`` command so you automatically get access +The script is built on top of the ``pytest`` command so you automatically get access to all the advanced option this command provide. Here a few examples : .. code-block:: shell @@ -159,7 +159,7 @@ For a single basic configuration one can use : Available parameters -------------------- -The parameters in the ``variants`` dictionnary correspond to the option supported by the +The parameters in the ``variants`` dictionnary correspond to the options supported by the :doc:`idfxTest ` script to configure the build and run of *Idefix*. In addition there is some extra keys which are dedicated to the json interpretation layer : @@ -197,7 +197,7 @@ Looping over parameters You might want to explore running Idefix within parameter ranges (configuration files, modes). For this simply list the values you want as a list. The test script will automatically -generate the list of all combinations. +generate all combinations. .. code-block:: json @@ -329,7 +329,7 @@ it simplified the writting of some conditions like if you used and IF statement. } } -In this case, ``reconstrucion`` will be set to 1 when ``single`` equal to ``true``. +In this case, ``reconstruction`` will be set to 1 when ``single`` is equal to ``true``. Note that the ``conditions`` field can contains several values which will be treaded as and AND logical operator. From 1afed8fdd65247ee77fd05504deff135cedb391a Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Thu, 19 Mar 2026 13:01:35 +0100 Subject: [PATCH 09/12] test: fix test paddings to pass linter in CI --- .pre-commit-config.yaml | 1 + doc/source/testing/testLauncher.rst | 5 ++- pytools/idfx_test.py | 16 ++++----- pytools/idfx_test_gen.py | 12 +++---- pytools/idfx_test_run.py | 8 ++--- pytools/pytest.ini | 2 +- pytools/tests/test/pb1/testme.json | 18 +++++----- pytools/tests/test/pb2/testme.json | 32 +++++++++--------- pytools/tests/test_idfx_test_run.py | 2 +- test/Dust/DustEnergy/testme.json | 20 +++++------ test/Dust/DustyShock/testme.json | 18 +++++----- test/Dust/DustyWave/testme.json | 20 +++++------ test/HD/FargoPlanet/testme.json | 46 ++++++++++++------------- test/HD/MachReflection/testme.json | 26 +++++++-------- test/HD/SedovBlastWave/testme.json | 52 ++++++++++++++--------------- 15 files changed, 138 insertions(+), 140 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 080bc9ac3..522249fed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,7 @@ repos: rev: v6.1.2 hooks: - id: inifix-format + files: ^(test/).*\.(ini)$ # want to skip pytest.ini - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.5.5 diff --git a/doc/source/testing/testLauncher.rst b/doc/source/testing/testLauncher.rst index e6039672d..637d2e218 100644 --- a/doc/source/testing/testLauncher.rst +++ b/doc/source/testing/testLauncher.rst @@ -302,8 +302,8 @@ By default, the alphabetical order will be used. When clauses ------------ -You can also dynamically override a specific value when a parameter value is selected, -it simplified the writting of some conditions like if you used and IF statement. +You can also dynamically override a specific value when a parameter value is selected. +It is just like if you used and IF statement. .. code-block:: json @@ -360,7 +360,6 @@ They are described like : "dec": [2,2,2], "multirun": [ { - },{ "mpi": true, "restart": true, diff --git a/pytools/idfx_test.py b/pytools/idfx_test.py index be86a72e8..fd28ccb61 100644 --- a/pytools/idfx_test.py +++ b/pytools/idfx_test.py @@ -5,8 +5,6 @@ import sys import re import json -import pytest -import warnings import numpy as np import matplotlib.pyplot as plt @@ -105,11 +103,11 @@ def __init__ (self, current_test_file, name=""): parser.add_argument("-ccache", help="Use ccache to reduce the build time over multiple run of the test suite.", action="store_true") - + parser.add_argument("-restart", help="Enable creating a restart from a checkpoint.", action='store_true') - + parser.add_argument("-v", "--verbose", help="Enable verbose mode, by not capturing the output.", action="store_true") @@ -145,7 +143,7 @@ def __init__ (self, current_test_file, name=""): self.lastCmakeCmd="" # subdir self.filterSubdir=args.subdir - # save + # save self.cmdArgs = vars(args) self.cmdArgs.update({ "restart_no_overwrite": [], @@ -153,12 +151,12 @@ def __init__ (self, current_test_file, name=""): self.log=[] # when making a restart we should not overrite those files (will be checked) self.restart_no_overwrite=[] - + # forward args for pytest if args.verbose: - unknown.append("--capture=no") + unknown.append("--capture=no") if args.help_pytest: - unknown.append("--help") + unknown.append("--help") # remaining args self.remainingArgs=unknown @@ -178,7 +176,7 @@ def applyConfig(self, config: dict={}): newArgs = {} newArgs.update(self.cmdArgs) newArgs.update(config) - + # replace in dict self.__dict__.update(newArgs) diff --git a/pytools/idfx_test_gen.py b/pytools/idfx_test_gen.py index 5a7f620b1..ab00fc6b6 100644 --- a/pytools/idfx_test_gen.py +++ b/pytools/idfx_test_gen.py @@ -21,7 +21,7 @@ def __init__(self, currentTestFile: str, name: str = ""): Constructor or the class. Args: - currentTestFile (str): + currentTestFile (str): Path the the current python file. Normally you simply pass __file__ to this parameter. name (str): @@ -55,7 +55,7 @@ def genTestConfigs(self, names:str, params, whenClauses = {}) -> list: # get name ordering list nameList = names.split(',') - # gen list of complete configs + # gen list of complete configs all_configs = [] if isinstance(params, dict): all_configs += self._genOneConfigSeries(names, params) @@ -64,7 +64,7 @@ def genTestConfigs(self, names:str, params, whenClauses = {}) -> list: all_configs += self._genOneConfigSeries(names, p) else: raise Exception("Should never be called !") - + # convert as parametrize with nice name result = [] for config in all_configs: @@ -98,7 +98,7 @@ def extractNamingParameters(self, params) -> list: Args: params (list|dict): The list of configuration sets to scan or a single set. - + Returns: The list of names of the variable parameters. ''' @@ -129,7 +129,7 @@ def extractNamingParameters(self, params) -> list: # TODO make something better by assigning a priority to vars variables.sort() - # ok + # ok return ','.join(variables) def _genNextLevelCombinations(self, input: list, paramName: str, paramValues: list) -> list: @@ -139,7 +139,7 @@ def _genNextLevelCombinations(self, input: list, paramName: str, paramValues: li Args: input (list): The incoming list of combinations already unpacked before this call. - paramName (str): + paramName (str): Name of the parameter to unpack. paramValues (list): The list of values to unpack and to build combinations for. diff --git a/pytools/idfx_test_run.py b/pytools/idfx_test_run.py index c30e4b928..9269b11b9 100644 --- a/pytools/idfx_test_run.py +++ b/pytools/idfx_test_run.py @@ -55,7 +55,7 @@ def _makeVariableArgAsList(self, namings: str, variants) -> list: # split namings namings = namings.split(",") - # loop on each + # loop on each for variant in variants: for name in namings: if name in variant and isinstance(variant[name], list) == False: @@ -101,10 +101,10 @@ def genTests(self) -> list: result += idefixTestGenerator.genTestConfigs(namings, test['variants'], test.get('when', {})) except Exception as e: raise Exception(f"Fail to generate tests from {testfileRelPath} : {e}") - + # ok return result - + def run(self, config: dict) -> None: # clone before modify to not modity for caller config = copy.deepcopy(config) @@ -239,7 +239,7 @@ def main(self, all: bool = False): sys.argv.append("-all") idefixTest = tst.idfxTest(self.parentScritFile, name="main") os.environ["IDEFIX_TEST_FILTER_SUBDIR"] = idefixTest.filterSubdir - + if idefixTest.all: pytest.main(['-v', '--no-header', '--junit-xml=idefix-tests.junit.xml', '--tb=short'] + idefixTest.remainingArgs + [self.parentScritFile]) else: diff --git a/pytools/pytest.ini b/pytools/pytest.ini index 25e73fa4e..d1f826516 100644 --- a/pytools/pytest.ini +++ b/pytools/pytest.ini @@ -1,2 +1,2 @@ [pytest] -python_files=test_*.py +python_files="test_*.py" diff --git a/pytools/tests/test/pb1/testme.json b/pytools/tests/test/pb1/testme.json index b82d1bcba..d8ddad44f 100644 --- a/pytools/tests/test/pb1/testme.json +++ b/pytools/tests/test/pb1/testme.json @@ -1,11 +1,11 @@ { - "variants": [ - { - "dumpname": "dump.0001.dmp", - "ini": ["idefix.ini","idefix-implicit.ini"], - "noplot": true, - "reconstruction": 2, - "tolerance": 1e-14 - } - ] + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] } diff --git a/pytools/tests/test/pb2/testme.json b/pytools/tests/test/pb2/testme.json index f7b8b1b5b..96c1753ab 100644 --- a/pytools/tests/test/pb2/testme.json +++ b/pytools/tests/test/pb2/testme.json @@ -1,18 +1,18 @@ { - "variants": [ - { - "dumpname": "dump.0001.dmp", - "ini": ["idefix.ini","idefix-implicit.ini"], - "noplot": true, - "reconstruction": 2, - "tolerance": 1e-14 - }, - { - "dumpname": "dump.0001.dmp", - "ini": ["idefix.ini","idefix-implicit.ini"], - "noplot": false, - "reconstruction": 2, - "tolerance": 1e-14 - } - ] + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + }, + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": false, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] } diff --git a/pytools/tests/test_idfx_test_run.py b/pytools/tests/test_idfx_test_run.py index 2aa788416..8843705ee 100644 --- a/pytools/tests/test_idfx_test_run.py +++ b/pytools/tests/test_idfx_test_run.py @@ -85,4 +85,4 @@ def test_genTests(): 'testname': 'pb2' }, marks=(), id='pb2-idefix-implicit.ini' ), - ] \ No newline at end of file + ] diff --git a/test/Dust/DustEnergy/testme.json b/test/Dust/DustEnergy/testme.json index c980b42cc..9a78834b9 100644 --- a/test/Dust/DustEnergy/testme.json +++ b/test/Dust/DustEnergy/testme.json @@ -1,12 +1,12 @@ { - "namings": "ini", - "variants": [ - { - "dumpname": "dump.0001.dmp", - "ini": ["idefix.ini","idefix-implicit.ini"], - "noplot": true, - "reconstruction": 2, - "tolerance": 0 - } - ] + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 0 + } + ] } diff --git a/test/Dust/DustyShock/testme.json b/test/Dust/DustyShock/testme.json index b82d1bcba..d8ddad44f 100644 --- a/test/Dust/DustyShock/testme.json +++ b/test/Dust/DustyShock/testme.json @@ -1,11 +1,11 @@ { - "variants": [ - { - "dumpname": "dump.0001.dmp", - "ini": ["idefix.ini","idefix-implicit.ini"], - "noplot": true, - "reconstruction": 2, - "tolerance": 1e-14 - } - ] + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] } diff --git a/test/Dust/DustyWave/testme.json b/test/Dust/DustyWave/testme.json index 6f5c0b9ac..0050f364e 100644 --- a/test/Dust/DustyWave/testme.json +++ b/test/Dust/DustyWave/testme.json @@ -1,12 +1,12 @@ { - "namings": "ini", - "variants": [ - { - "dumpname": "dump.0001.dmp", - "ini": ["idefix.ini","idefix-implicit.ini"], - "noplot": true, - "reconstruction": 2, - "tolerance": 1e-14 - } - ] + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] } diff --git a/test/HD/FargoPlanet/testme.json b/test/HD/FargoPlanet/testme.json index 1e38e4ddb..e1d735828 100644 --- a/test/HD/FargoPlanet/testme.json +++ b/test/HD/FargoPlanet/testme.json @@ -1,25 +1,25 @@ { - "namings": "ini,mpi", - "variants": [ - { - "dumpname": "dump.0001.dmp", - "ini": ["idefix.ini", "idefix-rkl.ini"], - "noplot": true, - "reconstruction": 2, - "single": false, - "mpi": [false, true], - "dec": [2, 2], - "tolerance": 1e-13 - } - ], - "when": [ - { - "conditions": { - "ini": "idefix-rkl.ini" - }, - "apply": { - "tolerance": 1e-12 - } - } - ] + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini", "idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": [2, 2], + "tolerance": 1e-13 + } + ], + "when": [ + { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply": { + "tolerance": 1e-12 + } + } + ] } diff --git a/test/HD/MachReflection/testme.json b/test/HD/MachReflection/testme.json index 67256aafb..be2d8d950 100644 --- a/test/HD/MachReflection/testme.json +++ b/test/HD/MachReflection/testme.json @@ -1,15 +1,15 @@ { - "namings": "ini,mpi", - "variants": [ - { - "dumpname": "dump.0001.dmp", - "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], - "noplot": true, - "reconstruction": 2, - "single": false, - "mpi": [false, true], - "dec": [2, 2], - "tolerance": 0 - } - ] + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": [2, 2], + "tolerance": 0 + } + ] } diff --git a/test/HD/SedovBlastWave/testme.json b/test/HD/SedovBlastWave/testme.json index 24f3d1c42..11327598b 100644 --- a/test/HD/SedovBlastWave/testme.json +++ b/test/HD/SedovBlastWave/testme.json @@ -1,28 +1,28 @@ { - "namings": "definitionFile,ini", - "variants": [ - { - "dumpname": "dump.0001.dmp", - "definitionFile": "definitions.hpp", - "ini": ["idefix.ini"], - "vectPot": false, - "reconstruction": 2, - "single": false, - "mpi": true, - "dec": [2, 2, 2 ], - "standardTest": false, - "tolerance": 0 - },{ - "dumpname": "dump.0001.dmp", - "definitionFile": "definitions-spherical.hpp", - "ini": ["idefix-spherical.ini"], - "vectPot": false, - "reconstruction": 2, - "single": false, - "mpi": true, - "dec": [2, 2, 2 ], - "standardTest": false, - "tolerance": 0 - } - ] + "namings": "definitionFile,ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "definitionFile": "definitions.hpp", + "ini": ["idefix.ini"], + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": true, + "dec": [2, 2, 2 ], + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "definitionFile": "definitions-spherical.hpp", + "ini": ["idefix-spherical.ini"], + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": true, + "dec": [2, 2, 2 ], + "standardTest": false, + "tolerance": 0 + } + ] } From c732ca269f07f18a92f0c9d7382c18d6527df66e Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Thu, 19 Mar 2026 13:31:04 +0100 Subject: [PATCH 10/12] test: change configuration of the root pytest.ini to run the tests in pytools by default --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 0b8cb7ff8..b3482664f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] markers= default: Test to run by default. -python_files=./test.py +python_files="test_*.py" junit_logging="system-out" junit_log_passing_tests=false From ae0fc3f092544f501ad86958bc78c15c61179263 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Tue, 21 Apr 2026 10:20:39 +0200 Subject: [PATCH 11/12] test: fix PR remarks about the doc --- doc/source/testing/testLauncher.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/testing/testLauncher.rst b/doc/source/testing/testLauncher.rst index 637d2e218..aea7f5c3d 100644 --- a/doc/source/testing/testLauncher.rst +++ b/doc/source/testing/testLauncher.rst @@ -5,7 +5,7 @@ Test launcher and reporter Overview -------- -The class :doc:`idfxTest ` provides the basement to implement an *Idefix* integration test for validation. +The class :doc:`idfxTest ` provides the toolbox to implement an *Idefix* integration test for validation. In order to ease launching all the tests, the user might prefer to use directly the ``./test.py`` command at the root of the *Idefix* sources. @@ -16,7 +16,7 @@ HTML file. Depencencies ------------ -Before using :doc:`idfxTest ` you need to install some Python Depencencies (possibly in a ``virtual env``): +Before using :doc:`idfxTest ` you need to install some Python depencencies (possibly in a ``virtual env``): .. code-block:: shell @@ -61,7 +61,7 @@ The result of the execution will be an output like : ---------- generated xml file: idefix-tests.junit.xml --------------------------------------------------------- =============================== 8 passed, 44 deselected in 73.03s (0:01:13) =================================== -When seeing an error, the output of the command will be printed at the end. +When an error is detected, the output of the command will be printed at the end. HTML report ----------- @@ -137,8 +137,8 @@ to all the advanced option this command provide. Here a few examples : Definition of the tests ----------------------- -The script is simply searching all the files names ``testme.json`` into the ``test`` directory. -This file describe the combination of parameters to use to produce the list of *Idefix* run to perform. +The script looks for all the files named ``testme.json`` in the ``test`` directory and all its sub-directories. +This file describes the combination of parameters used to produce the list of *Idefix* runs to check. For a single basic configuration one can use : .. code-block:: json @@ -297,7 +297,7 @@ the order you want to see them composing the test name. } -By default, the alphabetical order will be used. +By default, the alphabetical order is used. When clauses ------------ From a0aedbafdfb80f1fc66273ef207c5ead521bd90b Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Tue, 21 Apr 2026 14:25:25 +0200 Subject: [PATCH 12/12] test: adapt the new IO/ tests to the new semantic and add their testme.json --- pytools/idfx_test_gen.py | 2 +- pytools/idfx_test_run.py | 6 ++++++ test/IO/dump/testme.json | 28 ++++++++++++++++++++++++++++ test/IO/dump/testme.py | 2 +- test/IO/pydefix/testme.json | 17 +++++++++++++++++ test/IO/pydefix/testme.py | 2 +- test/IO/xdmf/testme.json | 17 +++++++++++++++++ test/IO/xdmf/testme.py | 2 +- test/MHD/LinearWaveTest/testme.json | 4 ++-- test/MHD/OrszagTang3D/testme.json | 12 ++---------- 10 files changed, 76 insertions(+), 16 deletions(-) create mode 100644 test/IO/dump/testme.json create mode 100644 test/IO/pydefix/testme.json create mode 100644 test/IO/xdmf/testme.json diff --git a/pytools/idfx_test_gen.py b/pytools/idfx_test_gen.py index ab00fc6b6..e552c321d 100644 --- a/pytools/idfx_test_gen.py +++ b/pytools/idfx_test_gen.py @@ -8,7 +8,7 @@ import copy import pytest -DO_NOT_LOOP_ON = ['restart_no_overwrite', "dec", "multirun"] +DO_NOT_LOOP_ON = ['restart_no_overwrite', "dec", "multirun", "check_file_produced"] class IdefixDirTestGenerator: ''' diff --git a/pytools/idfx_test_run.py b/pytools/idfx_test_run.py index 9269b11b9..d14de0b41 100644 --- a/pytools/idfx_test_run.py +++ b/pytools/idfx_test_run.py @@ -123,6 +123,7 @@ def run(self, config: dict) -> None: standardTest = config.get("standardTest", True) nonRegressionTest = config.get("nonRegressionTest", True) nonRegressionTestIni = config.get("nonRegressionTestIni", None) + check_file_produced = config.get("check_file_produced", []) problemDir = os.path.dirname(testfile) # cleanup some keyword not handled at the @@ -148,6 +149,11 @@ def run(self, config: dict) -> None: with moveInDir(problemDir): self._runNonRegression(dumpname, config['ini'], config, tolerance=tolerance, definitionFile=definitionFile, standardTest=standardTest, nonReg=nonRegressionTest, nonRegIni=nonRegressionTestIni) + # check produced + for file in check_file_produced: + if not os.path.exists(file): + raise Exception(f"Don't find expected file to be produced by the run : {file} !") + def _runNonRegression(self, dumpname, ini, config_override, tolerance=0, definitionFile="", nonReg=True, nonRegIni=None, standardTest=True, first_run_ini=None,first_run_dumpname=None,configure_and_compile=True): if 'multirun' in config_override: self._runNonRegMultirun(dumpname, ini, config_override, tolerance=tolerance, nonReg=nonReg, nonRegIni=nonRegIni, standardTest=standardTest, configure_and_compile=configure_and_compile, definitionFile=definitionFile, first_run_ini=first_run_ini, first_run_dumpname=first_run_dumpname) diff --git a/test/IO/dump/testme.json b/test/IO/dump/testme.json new file mode 100644 index 000000000..cb69f946f --- /dev/null +++ b/test/IO/dump/testme.json @@ -0,0 +1,28 @@ +{ + "namings": "ini,single,vectPot,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": [false, true], + "reconstruction": 2, + "single": [false], + "mpi": [false, true], + "dec": ["2","2","2"], + "standardTest": false, + "nonRegressionTest": false, + "tolerance": 1e-13 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": [false], + "reconstruction": 2, + "mpi": [false, true], + "single": [true], + "dec": ["2","2","2"], + "standardTest": false, + "nonRegressionTest": false, + "tolerance": 1e-13 + } + ] +} diff --git a/test/IO/dump/testme.py b/test/IO/dump/testme.py index 35affdba2..25e84433b 100755 --- a/test/IO/dump/testme.py +++ b/test/IO/dump/testme.py @@ -21,7 +21,7 @@ def testMe(test): test.run("idefix.ini") -test=tst.idfxTest() +test=tst.idfxTest(__file__) # if no decomposition is specified, use that one if not test.dec: diff --git a/test/IO/pydefix/testme.json b/test/IO/pydefix/testme.json new file mode 100644 index 000000000..78760f724 --- /dev/null +++ b/test/IO/pydefix/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,single,vectPot,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": ["2","2"], + "standardTest": false, + "tolerance": 1e-12 + } + ] +} diff --git a/test/IO/pydefix/testme.py b/test/IO/pydefix/testme.py index 0a3ad0432..2940e56dd 100755 --- a/test/IO/pydefix/testme.py +++ b/test/IO/pydefix/testme.py @@ -21,7 +21,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/IO/xdmf/testme.json b/test/IO/xdmf/testme.json new file mode 100644 index 000000000..8198bb7bb --- /dev/null +++ b/test/IO/xdmf/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,single,vectPot,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": false, + "dec": ["2","2","2"], + "standardTest": false, + "nonRegressionTest": false, + "check_file_produced": [ "data.0001.flt.xmf", "data.0001.flt.h5" ] + } + ] +} diff --git a/test/IO/xdmf/testme.py b/test/IO/xdmf/testme.py index a1e333430..6273bbe0e 100755 --- a/test/IO/xdmf/testme.py +++ b/test/IO/xdmf/testme.py @@ -12,7 +12,7 @@ name="dump.0001.dmp" -test=tst.idfxTest() +test=tst.idfxTest(__file__) def check_xdmf_exists(): # verify that the expected XDMF sidecar and data file exist. diff --git a/test/MHD/LinearWaveTest/testme.json b/test/MHD/LinearWaveTest/testme.json index 9a65e72dc..a3f997e34 100644 --- a/test/MHD/LinearWaveTest/testme.json +++ b/test/MHD/LinearWaveTest/testme.json @@ -9,7 +9,7 @@ "reconstruction": [2], "mpi": [false,true], "dec": ["2","2","2"], - "tolerance": 1e-14 + "tolerance": 2e-13 },{ "dumpname": "dump.0001.dmp", "ini": ["idefix-fast.ini","idefix-slow.ini","idefix-alfven.ini","idefix-entropy.ini"], @@ -18,7 +18,7 @@ "reconstruction": [3, 4], "mpi": [false], "dec": ["2","2","2"], - "tolerance": 1e-14 + "tolerance": 2e-13 } ] } diff --git a/test/MHD/OrszagTang3D/testme.json b/test/MHD/OrszagTang3D/testme.json index 867fe18b3..a37b54164 100644 --- a/test/MHD/OrszagTang3D/testme.json +++ b/test/MHD/OrszagTang3D/testme.json @@ -10,11 +10,7 @@ "mpi": [false, true], "dec": ["2","2","2"], "standardTest": false, - "tolerance": 1e-13, - "multirun": [ - {}, - {"ini": "idefix-checkrestart.ini", "dumpname": "dump.0002.dmp", "nonRegressionTestIni": "idefix.ini"} - ] + "tolerance": 1e-13 },{ "dumpname": "dump.0001.dmp", "ini": ["idefix.ini"], @@ -24,11 +20,7 @@ "single": [true], "dec": ["2","2","2"], "standardTest": false, - "tolerance": 1e-13, - "multirun": [ - {}, - {"ini": "idefix-checkrestart.ini", "dumpname": "dump.0002.dmp", "nonRegressionTestIni": "idefix.ini"} - ] + "tolerance": 1e-13 } ], "when": {