diff --git a/.github/workflows/CI_parallel_GPU_OPENMP.yml b/.github/workflows/CI_parallel_GPU_OPENMP.yml new file mode 100644 index 000000000..c06f8c557 --- /dev/null +++ b/.github/workflows/CI_parallel_GPU_OPENMP.yml @@ -0,0 +1,1477 @@ +name: CI PARALLEL (MPI) GPU OPENMP + +######################################################################## +on: + # Manual trigger only + workflow_dispatch: + +jobs: + wait-guard: + name: Runner availability guard + runs-on: ubuntu-latest + timeout-minutes: 2880 # 48 hours + steps: + # This job serves as a guard to prevent the workflow from pending indefinitely if the self-hosted GPU runner is not available. + - run: echo "GPU runner did not appear within 48 hours" + + parallel-gpu-tests: + # Uses self-hosted runner on nibbler cluster with GPU access + name: Parallel GPU Tests (NVFORTRAN) + runs-on: [self-hosted, nibbler, gpu] + # Prevent runaway jobs from consuming cluster resources indefinitely + timeout-minutes: 120 + + env: + METIS_HOME: ${{ github.workspace }}/metis-5.1.0/build/Linux-x86_64 + + steps: + - uses: actions/checkout@v4 + + # Fix ownership so the SLURM job user can read all checked-out files. + # Root (or the runner service account) owns files after checkout; a.ballout + # needs read access to mesh files, binaries, and control files at runtime. + - name: Fix workspace ownership + run: sudo chown -R a.ballout:a.ballout $GITHUB_WORKSPACE + + # Install METIS library (required for parallel partitioning) + - name: Install METIS + run: | + module purge + module load cmake + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + if [ ! -d "metis-5.1.0" ]; then + wget "https://src.fedoraproject.org/lookaside/pkgs/metis/metis-5.1.0.tar.gz/5465e67079419a69e0116de24fce58fe/metis-5.1.0.tar.gz" + tar -xvf metis-5.1.0.tar.gz + cd metis-5.1.0 + # Update CMake minimum required version to 3.5 + sed -i 's/cmake_minimum_required(VERSION [0-9.]\+)/cmake_minimum_required(VERSION 3.5)/' CMakeLists.txt + make config cc=gcc + make + mkdir -p build/Linux-x86_64/lib build/Linux-x86_64/bin + cp -r build/Linux-x86_64/libmetis/* build/Linux-x86_64/lib/ 2>/dev/null || true + cp -r build/Linux-x86_64/programs/* build/Linux-x86_64/bin/ 2>/dev/null || true + fi + + - name: Configure Horses3D + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cd Solver + ./configure + + - name: Compile horses3d + working-directory: ./Solver + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make allclean COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + make ns ins mu COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + +######################################################################## +####### NAVIER-STOKES TESTS ######## +######################################################################## + +# +# 1) CYLINDER +# ----------- + + - name: Build NSCylinder + working-directory: ./Solver/test/NavierStokes/Cylinder/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Fix test directory ownership + run: sudo chown -R a.ballout:a.ballout $GITHUB_WORKSPACE + if: '!cancelled()' + + - name: Run NSCylinder + working-directory: ./Solver/test/NavierStokes/Cylinder + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinder + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns Cylinder.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 5) CYLINDER SMAGORINSKY +# ----------------------- + + # - name: Build NSCylinderSmagorinsky + # working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + # make clean || true + # make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + # if: '!cancelled()' + + # - name: Run NSCylinderSmagorinsky + # working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=NSCylinderSmagorinsky + # #SBATCH --ntasks=2 + # #SBATCH --partition=gpu + # #SBATCH --mem=32G + # #SBATCH --gres=gpu:2 # number of gpus max 4 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + # mpirun -np 2 --bind-to none ./horses3d.ns CylinderSmagorinsky.control + # EOF + + # job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + +# +# 6) CYLINDER WALE +# ------------------ + + - name: Build NSCylinderWALE + working-directory: ./Solver/test/NavierStokes/CylinderWALE/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run NSCylinderWALE + working-directory: ./Solver/test/NavierStokes/CylinderWALE + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderWALE + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderWALE.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 7) CYLINDER VREMAN +# ------------------ + + - name: Build NSCylinderVreman + working-directory: ./Solver/test/NavierStokes/CylinderVreman/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run NSCylinderVreman + working-directory: ./Solver/test/NavierStokes/CylinderVreman + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderVreman + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderVreman.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 8) CYLINDER ChandrasekarRoe +# ------------------ + + - name: Build NSCylinderChandrasekarRoe + working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run NSCylinderChandrasekarRoe + working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderChandrasekarRoe + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderChandrasekarRoe.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# # +# # 9) TAYLOR GREEN VORTEX +# # ---------------------- + +# - name: Build TaylorGreen +# working-directory: ./Solver/test/NavierStokes/TaylorGreen/SETUP +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi +# make clean || true +# make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES +# if: '!cancelled()' + +# - name: Run TaylorGreen +# working-directory: ./Solver/test/NavierStokes/TaylorGreen +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + +# cat > run_test.sh << 'EOF' +# #!/bin/bash +# #SBATCH --job-name=TaylorGreen +# #SBATCH --ntasks=2 +# #SBATCH --partition=gpu +# #SBATCH --mem=32G +# #SBATCH --gres=gpu:2 # number of gpus max 4 +# #SBATCH --time=00:15:00 + +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi +# mpirun -np 2 --bind-to none ./horses3d.ns TaylorGreen.control +# EOF + +# job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) +# while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + +# cat slurm-${job_id}.out + +# # Check exit code (ExitCode:Signal, e.g., "0:0" means success) +# exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) +# if [ "$exit_code" != "0" ]; then +# echo "Test failed with exit code: $exit_code" +# exit 1 +# fi +# if: '!cancelled()' + +# # +# # 18) NACA0012 Unsteady Dual Time Stepping +# # ---------------------------- + +# - name: Build DualTimeStepping +# working-directory: ./Solver/test/NavierStokes/DualTimeStepping/SETUP +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi +# make clean || true +# make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES +# if: '!cancelled()' + +# - name: Run DualTimeStepping +# working-directory: ./Solver/test/NavierStokes/DualTimeStepping +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + +# cat > run_test.sh << 'EOF' +# #!/bin/bash +# #SBATCH --job-name=DualTimeStepping +# #SBATCH --ntasks=2 +# #SBATCH --partition=gpu +# #SBATCH --mem=32G +# #SBATCH --gres=gpu:2 # number of gpus max 4 +# #SBATCH --time=00:15:00 + +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi +# mpirun -np 2 --bind-to none ./horses3d.ns DualTimeStepping.control +# EOF + +# job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) +# while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + +# cat slurm-${job_id}.out + +# exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) +# if [ "$exit_code" != "0" ]; then +# echo "Test failed with exit code: $exit_code" +# exit 1 +# fi +# if: '!cancelled()' + +# +# 26) IBM CYLINDER +# ------------------- + + - name: Build IBM_Cylinder + working-directory: ./Solver/test/NavierStokes/IBM_Cylinder/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run IBM_Cylinder + working-directory: ./Solver/test/NavierStokes/IBM_Cylinder + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=IBM_Cylinder + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns IBMCylinder.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 40) CYLINDER EntropyConservingCentral +# ------------------------------------- + - name: Build NSCylinderEntropyConservingCentral + working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderEntropyConservingCentral + working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderEntropyConservingCentral + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderEntropyConservingCentral.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 41) CYLINDER RoePikePirozzoli +# ---------------------------- + - name: Build NSCylinderRoePikePirozzoli + working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderRoePikePirozzoli + working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderRoePikePirozzoli + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderRoePikePirozzoli.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 42) CYLINDER KennedyGruberLaxFriedrichs +# --------------------------------------- + - name: Build NSCylinderKennedyGruberLaxFriedrichs + working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderKennedyGruberLaxFriedrichs + working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderKennedyGruberLaxFriedrichs + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderKennedyGruberLaxFriedrichs.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 43) CYLINDER MorinishiLowDissipationRoe +# ---------------------------------------- + - name: Build NSCylinderMorinishiLowDissipationRoe + working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderMorinishiLowDissipationRoe + working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderMorinishiLowDissipationRoe + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderMorinishiLowDissipationRoe.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 44) CYLINDER DucrosMatrixDissipation +# ------------------------------------- + - name: Build NSCylinderDucrosMatrixDissipation + working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderDucrosMatrixDissipation + working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderDucrosMatrixDissipation + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderDucrosMatrixDissipation.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 45) CYLINDER ViscousStandard +# ------------------------- + - name: Build NSCylinderViscousStandard + working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderViscousStandard + working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderViscousStandard + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderViscousStandard.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 46) CYLINDER RusanovStandard +# ------------------------- + - name: Build NSCylinderRusanovStandard + working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderRusanovStandard + working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderRusanovStandard + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderRusanovStandard.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 47) CYLINDER UdissStandard +# ------------------------- + - name: Build NSCylinderUdissStandard + working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderUdissStandard + working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderUdissStandard + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderUdissStandard.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 48) CYLINDER sprk33 +# ------------------- + - name: Build NSCylinderssprk33 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk33/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderssprk33 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk33 + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderssprk33 + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns Cylinderssprk33.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 49) CYLINDER sprk43 +# ------------------- + - name: Build NSCylinderssprk43 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk43/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderssprk43 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk43 + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderssprk43 + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns Cylinderssprk43.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 50) Limiter Test +# ------------------- + - name: Build NSLimiterTest + working-directory: ./Solver/test/NavierStokes/LimiterTest/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSLimiterTest + working-directory: ./Solver/test/NavierStokes/LimiterTest + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSLimiterTest + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns LimiterTest.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +######################################################################## +####### INCOMPRESSIBLE_NS TESTS ######## +######################################################################## + +### Sometimes this test passes and sometimes it failscc +# +# 1) NACA0012 +# ---------------------------- + + # - name: Build iNS-NACA0012 + # working-directory: ./Solver/test/IncompressibleNS/NACA0012/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + # make clean || true + # make ins COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + # if: '!cancelled()' + + # - name: Run iNS-NACA0012 + # working-directory: ./Solver/test/IncompressibleNS/NACA0012 + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=NACA0012 + # #SBATCH --ntasks=2 + # #SBATCH --partition=gpu + # #SBATCH --cpus-per-task=1 + # #SBATCH --mem=32G + # #SBATCH --gres=gpu:2 # number of gpus max 4 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + # mpirun -np 2 --bind-to none ./horses3d.ins NACA0012.control + # EOF + + # job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + + +######################################################################## +####### MULTIPHASE TESTS ######## +######################################################################## + +# +# 4) Entropy conserving test +# ---------------------------- + + - name: Build MultiphaseEntropyConservingTest + working-directory: ./Solver/test/Multiphase/EntropyConservingTest/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make mu COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run MultiphaseEntropyConservingTest + working-directory: ./Solver/test/Multiphase/EntropyConservingTest + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=EntropyConservingTest + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.mu EntropyConservingTest.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' \ No newline at end of file diff --git a/.github/workflows/CI_parallel_MU.yml b/.github/workflows/CI_parallel_MU.yml index 05bd52861..1161b0811 100644 --- a/.github/workflows/CI_parallel_MU.yml +++ b/.github/workflows/CI_parallel_MU.yml @@ -34,7 +34,7 @@ jobs: compiler: ['gfortran'] mode: ['DEBUG'] comm: ['PARALLEL'] - enable_threads: ['NO'] + enable_threads: ['NO','YES'] name: MU - ${{ matrix.compiler }} - ${{ matrix.mode }} - ${{ matrix.comm }} - ${{ matrix.enable_threads }} diff --git a/.github/workflows/CI_parallel_NS.yml b/.github/workflows/CI_parallel_NS.yml index 1cff15029..9fe72f8fc 100644 --- a/.github/workflows/CI_parallel_NS.yml +++ b/.github/workflows/CI_parallel_NS.yml @@ -34,7 +34,7 @@ jobs: compiler: ['gfortran'] mode: ['DEBUG'] comm: ['PARALLEL'] - enable_threads: ['NO'] + enable_threads: ['NO','YES'] mkl: ['NO'] hdf5: ['YES'] diff --git a/.github/workflows/CI_parallel_iNS.yml b/.github/workflows/CI_parallel_iNS.yml index bbef28c32..0453d3d74 100644 --- a/.github/workflows/CI_parallel_iNS.yml +++ b/.github/workflows/CI_parallel_iNS.yml @@ -34,7 +34,7 @@ jobs: compiler: ['gfortran'] mode: ['DEBUG'] comm: ['PARALLEL'] - enable_threads: ['NO'] + enable_threads: ['NO','YES'] name: iNS - ${{ matrix.compiler }} - ${{ matrix.mode }} - ${{ matrix.comm }} - ${{ matrix.enable_threads }} diff --git a/.github/workflows/CI_serial_GPU_OPENMP.yml b/.github/workflows/CI_serial_GPU_OPENMP.yml new file mode 100644 index 000000000..f12f945e1 --- /dev/null +++ b/.github/workflows/CI_serial_GPU_OPENMP.yml @@ -0,0 +1,1309 @@ +name: CI SERIAL GPU OPENMP + +######################################################################## +on: + # Manual trigger only + workflow_dispatch: + +jobs: + wait-guard: + name: Runner availability guard + runs-on: ubuntu-latest + timeout-minutes: 2880 # 48 hours + steps: + # This job serves as a guard to prevent the workflow from pending indefinitely if the self-hosted GPU runner is not available. + - run: echo "GPU runner did not appear within 48 hours" + + serial-gpu-tests: + # Uses self-hosted runner on nibbler with GPU access + name: Serial GPU Tests (NVFORTRAN) + runs-on: [self-hosted, nibbler, gpu] + # Prevent runaway jobs from consuming cluster resources indefinitely + timeout-minutes: 120 + + env: + METIS_HOME: ${{ github.workspace }}/metis-5.1.0/build/Linux-x86_64 + + steps: + - uses: actions/checkout@v4 + + - name: Configure Horses3D + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cd Solver + ./configure + + - name: Compile horses3d + working-directory: ./Solver + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make allclean COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + make ns ins mu COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + +######################################################################## +####### NAVIER-STOKES TESTS ######## +######################################################################## + +# +# 1) CYLINDER +# ----------- + + - name: Build NSCylinder + working-directory: ./Solver/test/NavierStokes/Cylinder/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run NSCylinder + working-directory: ./Solver/test/NavierStokes/Cylinder + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinder_Serial + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns Cylinder.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 5) CYLINDER SMAGORINSKY +# ----------------------- + + # - name: Build NSCylinderSmagorinsky + # working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # make clean || true + # make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + # if: '!cancelled()' + + # - name: Run NSCylinderSmagorinsky + # working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=NSCylinderSmagorinsky + # #SBATCH --ntasks=1 + # #SBATCH --mem=20G + # #SBATCH --partition=gpu + # #SBATCH --gres=gpu:1 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # ./horses3d.ns CylinderSmagorinsky.control + # EOF + + # job_id=$(sbatch --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + +# +# 6) CYLINDER WALE +# ------------------ + + - name: Build NSCylinderWALE + working-directory: ./Solver/test/NavierStokes/CylinderWALE/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run NSCylinderWALE + working-directory: ./Solver/test/NavierStokes/CylinderWALE + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderWALE + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderWALE.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 7) CYLINDER VREMAN +# ------------------ + + - name: Build NSCylinderVreman + working-directory: ./Solver/test/NavierStokes/CylinderVreman/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run NSCylinderVreman + working-directory: ./Solver/test/NavierStokes/CylinderVreman + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderVreman + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderVreman.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 8) CYLINDER ChandrasekarRoe +# ------------------ + + - name: Build NSCylinderChandrasekarRoe + working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run NSCylinderChandrasekarRoe + working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderChandrasekarRoe + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderChandrasekarRoe.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 9) TAYLOR GREEN VORTEX +# ---------------------- + + # - name: Build TaylorGreen + # working-directory: ./Solver/test/NavierStokes/TaylorGreen/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # make clean || true + # make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + # if: '!cancelled()' + + # - name: Run TaylorGreen + # working-directory: ./Solver/test/NavierStokes/TaylorGreen + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=TaylorGreen_Serial + # #SBATCH --ntasks=1 + # #SBATCH --mem=20G + # #SBATCH --partition=gpu + # #SBATCH --gres=gpu:1 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # ./horses3d.ns TaylorGreen.control + # EOF + + # job_id=$(sbatch --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + +# # +# # 18) NACA0012 Unsteady Dual Time Stepping +# # ---------------------------- + +# - name: Build DualTimeStepping +# working-directory: ./Solver/test/NavierStokes/DualTimeStepping/SETUP +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# make clean || true +# make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES +# if: '!cancelled()' + +# - name: Run DualTimeStepping +# working-directory: ./Solver/test/NavierStokes/DualTimeStepping +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + +# cat > run_test.sh << 'EOF' +# #!/bin/bash +# #SBATCH --job-name=DualTimeStepping +# #SBATCH --ntasks=1 +# #SBATCH --mem=20G +# #SBATCH --partition=gpu +# #SBATCH --gres=gpu:1 +# #SBATCH --time=00:15:00 + +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# ./horses3d.ns DualTimeStepping.control +# EOF + +# job_id=$(sbatch --parsable run_test.sh) +# while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + +# cat slurm-${job_id}.out + +# exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) +# if [ "$exit_code" != "0" ]; then +# echo "Test failed with exit code: $exit_code" +# exit 1 +# fi +# if: '!cancelled()' + +# +# 26) IBM CYLINDER +# ------------------- + + - name: Build IBM_Cylinder + working-directory: ./Solver/test/NavierStokes/IBM_Cylinder/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run IBM_Cylinder + working-directory: ./Solver/test/NavierStokes/IBM_Cylinder + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=IBM_Cylinder + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns IBMCylinder.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 40) CYLINDER EntropyConservingCentral +# ------------------------------------- + - name: Build NSCylinderEntropyConservingCentral + working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderEntropyConservingCentral + working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderEntropyConservingCentral + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderEntropyConservingCentral.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 41) CYLINDER RoePikePirozzoli +# ---------------------------- + - name: Build NSCylinderRoePikePirozzoli + working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderRoePikePirozzoli + working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderRoePikePirozzoli + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderRoePikePirozzoli.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 42) CYLINDER KennedyGruberLaxFriedrichs +# --------------------------------------- + - name: Build NSCylinderKennedyGruberLaxFriedrichs + working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderKennedyGruberLaxFriedrichs + working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderKennedyGruberLaxFriedrichs + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderKennedyGruberLaxFriedrichs.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 43) CYLINDER MorinishiLowDissipationRoe +# ---------------------------------------- + - name: Build NSCylinderMorinishiLowDissipationRoe + working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderMorinishiLowDissipationRoe + working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderMorinishiLowDissipationRoe + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderMorinishiLowDissipationRoe.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 44) CYLINDER DucrosMatrixDissipation +# ------------------------------------- + - name: Build NSCylinderDucrosMatrixDissipation + working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderDucrosMatrixDissipation + working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderDucrosMatrixDissipation + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderDucrosMatrixDissipation.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 45) CYLINDER ViscousStandard +# ------------------------- + - name: Build NSCylinderViscousStandard + working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderViscousStandard + working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderViscousStandard + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderViscousStandard.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 46) CYLINDER RusanovStandard +# ------------------------- + - name: Build NSCylinderRusanovStandard + working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderRusanovStandard + working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderRusanovStandard + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderRusanovStandard.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 47) CYLINDER UdissStandard +# ------------------------- + - name: Build NSCylinderUdissStandard + working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderUdissStandard + working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderUdissStandard + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderUdissStandard.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 48) CYLINDER sprk33 +# ------------------- + - name: Build NSCylinderssprk33 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk33/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderssprk33 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk33 + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderssprk33 + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns Cylinderssprk33.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 49) CYLINDER sprk43 +# ------------------- + - name: Build NSCylinderssprk43 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk43/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderssprk43 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk43 + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderssprk43 + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns Cylinderssprk43.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + + +# +# 50) Limiter Test +# ------------------- + - name: Build NSLimiterTest + working-directory: ./Solver/test/NavierStokes/LimiterTest/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSLimiterTest + working-directory: ./Solver/test/NavierStokes/LimiterTest + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSLimiterTest + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns LimiterTest.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +######################################################################## +####### INCOMPRESSIBLE_NS TESTS ######## +######################################################################## + +### Sometimes this test passes and sometimes it failscc +# +# 1) NACA0012 +# ---------------------------- + + # - name: Build iNS-NACA0012 + # working-directory: ./Solver/test/IncompressibleNS/NACA0012/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # make clean || true + # make ins COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + # if: '!cancelled()' + + # - name: Run iNS-NACA0012 + # working-directory: ./Solver/test/IncompressibleNS/NACA0012 + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=NACA0012 + # #SBATCH --ntasks=1 + # #SBATCH --mem=20G + # #SBATCH --partition=gpu + # #SBATCH --gres=gpu:1 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # ./horses3d.ins NACA0012.control + # EOF + + # job_id=$(sbatch --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + + +######################################################################## +####### MULTIPHASE TESTS ######## +######################################################################## + +# +# 4) Entropy conserving test +# ---------------------------- + + - name: Build MultiphaseEntropyConservingTest + working-directory: ./Solver/test/Multiphase/EntropyConservingTest/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make mu COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run MultiphaseEntropyConservingTest + working-directory: ./Solver/test/Multiphase/EntropyConservingTest + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=MultiphaseEntropyConservingTest + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.mu EntropyConservingTest.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' \ No newline at end of file diff --git a/.github/workflows/CI_serial_MU.yml b/.github/workflows/CI_serial_MU.yml index ae58d4241..bfe650722 100644 --- a/.github/workflows/CI_serial_MU.yml +++ b/.github/workflows/CI_serial_MU.yml @@ -34,7 +34,7 @@ jobs: compiler: ['gfortran'] mode: ['DEBUG'] comm: ['SEQUENTIAL'] - enable_threads: ['NO'] + enable_threads: ['NO','YES'] name: MU - ${{ matrix.compiler }} - ${{ matrix.mode }} - ${{ matrix.comm }} - ${{ matrix.enable_threads }} diff --git a/.gitignore b/.gitignore index 281295a2e..bcad5fda0 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,15 @@ Solver/include/ horsesConverter #ford -doc/doc_output \ No newline at end of file +doc/doc_output +*.aux +*.fls +*.out +*.fdb_latexmk +*.gz +*.log +*.toc +*.blg +*.bbl +*.diff +*.patch diff --git a/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 b/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 index f9ec0cd98..498cd6f44 100644 --- a/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 +++ b/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 @@ -258,7 +258,9 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) ! Prolongation of the solution to the faces ! ----------------------------------------- ! +#ifndef _OPENACC !$omp parallel shared(mesh, time) +#endif call HexMesh_ProlongSolToFaces(mesh, NCONS) ! ---------------- ! Update MPI Faces @@ -273,16 +275,16 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) ! ------------------------------------------ ! Apply the Boundary conditions to the state ! ------------------------------------------ -! This was done in the compute boundary flux before +! This was done in the compute boundary flux before ! but it was called twice because we call it once in this file ! and one in the Elliptic discretisation. So now we compute it ! only once at the begining of time derivative and store it -! +! nZones = size(mesh % zones) do zoneID=1, nZones - CALL BCs(zoneID) % bc % FlowState(mesh, mesh % zones(zoneID)) + CALL BCs(zoneID) % bc % FlowState(mesh, mesh % zones(zoneID)) enddo -! +! ! ----------------- ! Compute gradients ! ----------------- @@ -293,7 +295,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) if ( computeGradients ) then call ViscousDiscretization % ComputeGradient( NCONS, NGRAD, mesh, time, GetGradients, HO_Elements) end if - + ! call ComputeArtificialViscosity(mesh) ! ! ----------------------- @@ -304,13 +306,15 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) call TimeDerivative_ComputeQDotHO(mesh = mesh , & particles = particles, & t = time) - else + else call TimeDerivative_ComputeQDot(mesh = mesh , & particles = particles, & t = time) - + end if +#ifndef _OPENACC !$omp end parallel +#endif ! END SUBROUTINE ComputeTimeDerivative ! @@ -343,7 +347,9 @@ SUBROUTINE ComputeTimeDerivativeIsolated( mesh, particles, time, mode, HO_Elemen ! Prolongation of the solution to the faces ! ----------------------------------------- ! +#ifndef _OPENACC !$omp parallel shared(mesh, time) +#endif call mesh % ProlongSolutionToFaces(NCONS) ! ! ----------------------------------------------------- @@ -365,7 +371,9 @@ SUBROUTINE ComputeTimeDerivativeIsolated( mesh, particles, time, mode, HO_Elemen ! call TimeDerivative_ComputeQDotIsolated(mesh = mesh , & t = time ) +#ifndef _OPENACC !$omp end parallel +#endif ! END SUBROUTINE ComputeTimeDerivativeIsolated @@ -405,24 +413,33 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! *********************************************** ! if (flowIsNavierStokes) then -!$omp do schedule(runtime) private(i,j,k) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) async(1) +#else +!$omp do schedule(runtime) private(i,j,k) +#endif do eID = 1, size(mesh % elements) !$acc loop vector collapse(3) do k = 0, mesh % elements(eID) % Nxyz(3) ; do j = 0, mesh % elements(eID) % Nxyz(2) ; do i = 0, mesh % elements(eID) % Nxyz(1) call get_laminar_mu_kappa(mesh % elements(eID) % storage % Q(:,i,j,k), & - mesh % elements(eID) % storage % mu_NS(1,i,j,k),& + mesh % elements(eID) % storage % mu_NS(1,i,j,k),& mesh % elements(eID) % storage % mu_NS(2,i,j,k)) end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if if ( LESModel % active) then -!$omp do schedule(runtime) private(i,j,k,delta,mu_smag) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, LESModel) async(1) +#else +!$omp do schedule(runtime) private(i,j,k,delta,mu_smag) +#endif do eID = 1, size(mesh % elements) delta = (mesh % elements(eID) % geom % Volume / product(mesh % elements(eID) % Nxyz + 1)) ** (1.0_RP / 3.0_RP) !$acc loop vector collapse(3) @@ -440,8 +457,11 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) mesh % elements(eID) % storage % mu_turb_NS(i,j,k) * dimensionless % mu_to_kappa end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if ! ! Compute viscosity at interior and boundary faces @@ -472,23 +492,35 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! Compute Riemann solver of non-shared faces ! ****************************************** ! -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang collapse(2) present(mesh) async(1) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_interior) ; do side = 1,2 fID = mesh % faces_interior(iFace) call computeElementInterfaceFlux_viscous(mesh % faces(fID), side) - end do ; end do + end do ; end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) async(1) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_interior) fID = mesh % faces_interior(iFace) call computeElementInterfaceFlux(mesh % faces(fID)) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do nowait +#endif call computeBoundaryFlux(mesh, t) ! @@ -496,14 +528,20 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! Surface integrals and scaling of elements with non-shared faces ! *************************************************************** ! -!$omp do schedule(runtime) private(i,j,k,eID) +#ifdef _OPENACC !$acc parallel loop gang num_gangs(size(mesh % elements_sequential)) vector_length(128) present(mesh) async(1) +#else +!$omp do schedule(runtime) private(i,j,k,eID) +#endif do iEl = 1, size(mesh % elements_sequential) eID = mesh % elements_sequential(iEl) call TimeDerivative_FacesContribution(mesh % elements(eID), mesh) end do -!$acc end parallel loop +#ifdef _OPENACC +!$acc end parallel loop +#else !$omp end do +#endif ! ! **************************** ! Wait until messages are sent @@ -536,27 +574,39 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! Compute Riemann solver of shared faces ! ************************************** ! -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang num_gangs(size(mesh % faces_mpi)) present(mesh) private(fID) async(1) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_mpi) fID = mesh % faces_mpi(iFace) call computeMPIFaceFlux(mesh % faces(fID)) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif ! ! *********************************************************** ! Surface integrals and scaling of elements with shared faces ! *********************************************************** ! -!$omp do schedule(runtime) private(i,j,k,eID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) async(1) +#else +!$omp do schedule(runtime) private(i,j,k,eID) +#endif do iEl = 1, size(mesh % elements_mpi) eID = mesh % elements_mpi(iEl) call TimeDerivative_FacesContribution(mesh % elements(eID), mesh) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif ! ! Add an MPI Barrier ! ------------------ @@ -646,42 +696,55 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! *********************** ! Now add the source term ! *********************** -!$omp do schedule(runtime) private(i,j,k) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) present(mesh) async(1) +#else +!$omp do schedule(runtime) private(i,j,k) +#endif do eID = 1, mesh % no_of_elements !$acc loop vector collapse(4) do k = 0, mesh % elements(eID) % Nxyz(3) ; do j = 0, mesh % elements(eID) % Nxyz(2) ; do i = 0, mesh % elements(eID) % Nxyz(1) ; do eq = 1, NCONS mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) + mesh % elements(eID) % storage % S_NS(eq,i,j,k) end do ; end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif ! ! ********************* ! Add IBM source term ! ********************* if( mesh % IBM % active ) then if( .not. mesh % IBM % semiImplicit ) then -!$omp do schedule(runtime) private(i,j,k,Source,Q_target) +#ifdef _OPENACC ! Check if update(t) is required !$acc parallel loop gang present(mesh) copyin(t) async(1) - do eID = 1, mesh % no_of_elements +#else +!$omp do schedule(runtime) private(i,j,k,Source,Q_target) + ! Check if update(t) is required +#endif + do eID = 1, mesh % no_of_elements !$acc loop vector collapse(3) private(Source, Q_target) do k = 0, mesh % elements(eID) % Nxyz(3) ; do j = 0, mesh % elements(eID) % Nxyz(2) ; do i = 0, mesh % elements(eID) % Nxyz(1) if( mesh % elements(eID) % isInsideBody(i,j,k) ) then - if( mesh % IBM % stl(mesh % elements(eID) % STL(i,j,k)) % move ) then + if( mesh % IBM % stl(mesh % elements(eID) % STL(i,j,k)) % move ) then ! Q_target = IBM_MaskVelocity(mesh%IBM, mesh % elements(eID)% storage% Q(:,i,j,k), NCONS, mesh % elements(eID)% STL(i,j,k), mesh % elements(eID)% geom% x(:,i,j,k), t ) call IBM_MaskVelocity(mesh % IBM, mesh % elements(eID) % storage % Q(:,i,j,k), NCONS, mesh % elements(eID) % STL(i,j,k), mesh % elements(eID) % geom % x(:,i,j,k), t, Q_target ) call IBM_SourceTerm(mesh % IBM, eID = eID, Q = mesh % elements(eID) % storage % Q(:,i,j,k), Q_target = Q_target, Source = Source, wallfunction = .false. ) - else + else call IBM_SourceTerm(mesh % IBM, eID = eID, Q = mesh % elements(eID) % storage % Q(:,i,j,k), Source = Source, wallfunction = .false. ) - end if + end if mesh % elements(eID) % storage % QDot(:,i,j,k) = mesh % elements(eID) % storage % QDot(:,i,j,k) + Source end if end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do +#endif if( mesh % IBM % Wallfunction ) then !$omp single call mesh % IBM % GetBandRegionStates( mesh % elements ) @@ -750,8 +813,11 @@ subroutine TimeDerivative_ComputeQDotHO( mesh , particles, t) end if if ( LESModel % active) then - !$omp do schedule(runtime) private(i,j,k,delta,mu_smag) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, LESModel) +#else + !$omp do schedule(runtime) private(i,j,k,delta,mu_smag) +#endif do eID = 1, size(mesh % elements) delta = (mesh % elements(eID) % geom % Volume / product(mesh % elements(eID) % Nxyz + 1)) ** (1.0_RP / 3.0_RP) !$acc loop vector collapse(3) @@ -771,8 +837,11 @@ subroutine TimeDerivative_ComputeQDotHO( mesh , particles, t) mesh % elements(eID) % storage % mu_turb_NS(i,j,k) * dimensionless % mu_to_kappa end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if ! ! Compute viscosity at interior and boundary faces @@ -1043,8 +1112,11 @@ subroutine compute_viscosity_at_faces(no_of_faces, no_of_sides, face_ids, mesh) real(kind=RP) :: delta, mu_smag if (flowIsNavierStokes) then -!$omp do schedule(runtime) private(i,j) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, face_ids) async(1) +#else +!$omp do schedule(runtime) private(i,j) +#endif do iFace = 1, no_of_faces fID = face_ids(iFace) !$acc loop vector collapse(3) @@ -1056,13 +1128,19 @@ subroutine compute_viscosity_at_faces(no_of_faces, no_of_sides, face_ids, mesh) end do end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if if ( LESModel % Active ) then -!$omp do schedule(runtime) private(i,j,delta,mu_smag) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, LESModel, face_ids) async(1) +#else +!$omp do schedule(runtime) private(i,j,delta,mu_smag) +#endif do iFace = 1, no_of_faces delta = sqrt(mesh % faces(face_ids(iFace)) % geom % surface / product(mesh % faces(face_ids(iFace)) % Nf + 1)) !$acc loop vector collapse(3) @@ -1079,8 +1157,11 @@ subroutine compute_viscosity_at_faces(no_of_faces, no_of_sides, face_ids, mesh) end do end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if end subroutine compute_viscosity_at_faces @@ -1309,8 +1390,11 @@ subroutine TimeDerivative_VolumetricContribution(mesh) ! ! Compute inviscid - viscous contravariant flux ! --------------------------------------------- - !$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) num_gangs(9700) present(mesh) async(1) +#else + !$omp do schedule(runtime) +#endif do eID = 1 , size(mesh % elements) !$acc loop vector collapse(3) private(inviscidFlux, viscousFlux) @@ -1353,8 +1437,11 @@ subroutine TimeDerivative_VolumetricContribution(mesh) mesh % elements(eID) % storage % QDot) end do - !$acc end parallel loop +#ifdef _OPENACC + !$acc end parallel loop +#else !$omp end do +#endif end subroutine TimeDerivative_VolumetricContribution diff --git a/Solver/src/NavierStokesSolver/split_template.inc b/Solver/src/NavierStokesSolver/split_template.inc new file mode 100644 index 000000000..40ddac7bc --- /dev/null +++ b/Solver/src/NavierStokesSolver/split_template.inc @@ -0,0 +1,88 @@ + !$acc parallel present(mesh) vector_length(128) num_gangs(9750) async(1) + !$acc loop gang + do eID = 1 , size(mesh % elements) + + !$acc loop vector collapse(3) private(Flux) + do k = 0, mesh % elements(eID) % Nxyz(3) + do j = 0, mesh % elements(eID) % Nxyz(2) + do i = 0, mesh % elements(eID) % Nxyz(1) + + call ViscousFlux_STATE( NCONS, NGRAD, mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % U_x(:,i,j,k), & + mesh % elements(eID) % storage % U_y(:,i,j,k) , mesh % elements(eID) % storage % U_z(:,i,j,k), & + mesh % elements(eID) % storage % mu_ns(1,i,j,k), 0.0_RP, & + mesh % elements(eID) % storage % mu_ns(2,i,j,k), Flux) + !$acc loop seq + do eq = 1, NCONS + + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IX) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradXi(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradXi(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradXi(IZ,i,j,k) + + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IY) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradEta(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradEta(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradEta(IZ,i,j,k) + + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IZ) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradZeta(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradZeta(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradZeta(IZ,i,j,k) + ! initialize to 0 to accumulate + mesh % elements(eID) % storage % Qdot(eq,i,j,k) = 0.0_RP + end do + end do ; end do ; end do + + call ScalarWeakIntegrals_StdVolumeGreen( mesh % elements(eID) % Nxyz, NCONS, mesh % elements(eID) % storage % contravariantFlux, & + mesh % elements(eID) % storage % QDot) +! +! ************************************* +! Compute interior contravariant fluxes +! ************************************* +! +! Compute inviscid contravariant flux +! ----------------------------------- + !$acc loop vector collapse(3) private(Flux) + do k = 0, mesh % elements(eID) % Nxyz(3) + do j = 0, mesh % elements(eID) % Nxyz(2) + do i = 0, mesh % elements(eID) % Nxyz(1) + if ( mesh % elements(eID) % Nxyz(1) > 0 ) then + !$acc loop seq + do l = 0, mesh % elements(eID) % Nxyz(1) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,l,j,k), mesh % elements(eID) % geom % jGradXi(:,i,j,k), mesh % elements(eID) % geom % jGradXi(:,l,j,k), Flux(:,IX)) + + !$acc loop seq + do eq = 1, NCONS + mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) & + - NodalStorage(mesh % elements(eID) % Nxyz(1)) % sharpD(i,l) * Flux(eq,IX) + end do + end do + end if + + if ( mesh % elements(eID) % Nxyz(2) > 0 ) then + !$acc loop seq + do l = 0, mesh % elements(eID) % Nxyz(2) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,i,l,k), mesh % elements(eID) % geom % jGradEta(:,i,j,k), mesh % elements(eID) % geom % jGradEta(:,i,l,k), Flux(:,IY)) + + !$acc loop seq + do eq = 1, NCONS + mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) & + - NodalStorage(mesh % elements(eID) % Nxyz(2)) % sharpD(j,l) * Flux(eq,IY) + end do + end do + end if + + if ( mesh % elements(eID) % Nxyz(3) > 0 ) then + !$acc loop seq + do l = 0, mesh % elements(eID) % Nxyz(3) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,i,j,l), mesh % elements(eID) % geom % jGradZeta(:,i,j,k), mesh % elements(eID) % geom % jGradZeta(:,i,j,l), Flux(:,IZ)) + + !$acc loop seq + do eq = 1, NCONS + mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) & + - NodalStorage(mesh % elements(eID) % Nxyz(3)) % sharpD(k,l) * Flux(eq,IZ) + end do + end do + end if + + end do ; end do ; end do + + enddo + !$acc end parallel loop diff --git a/Solver/src/addons/probes2tecplot/Makefile b/Solver/src/addons/probes2tecplot/Makefile new file mode 100644 index 000000000..70d430b22 --- /dev/null +++ b/Solver/src/addons/probes2tecplot/Makefile @@ -0,0 +1,109 @@ +##################################################################### +## HORSES - a High-Order Spectral Element Solver +## +## probes2tecplot Makefile +## +##################################################################### + +######################################## +# 1.- User defined parameters +######################################## +INSTALL_DIR?=./ +FFLAGS?=-cpp +AR?=ar +ARFLAGS?=cr +RANLIB?=ranlib +MOD_FLAG?=-J +PLATFORM?=LINUX +COMPILER?=gfortran +MODE?=RELEASE +COMM?=PARALLEL +MAKE=make +AR=ar +RANLIB=ranlib +PROG=./bin/probes2tecplot + +# Directory of shared source files from horses2tecplot +H2T_DIR=../horses2tecplot + +# Files compiled from the horses2tecplot directory (shared code) +H2T_OBJS= Storage \ + OutputVariables \ + sharedSpectralBasis \ + +# Files compiled from the probes2tecplot directory (new code) +LOCAL_OBJS= Probes2tecplot \ + main \ + +H2T_OBJS_BUILD=$(foreach obj,$(H2T_OBJS),./build/$(obj).o) +LOCAL_OBJS_BUILD=$(foreach obj,$(LOCAL_OBJS),./build/$(obj).o) +ALL_OBJS_BUILD=$(H2T_OBJS_BUILD) $(LOCAL_OBJS_BUILD) + + +########################################## +## Include dependencies ## +########################################## +-include ./make.deps + + +########################################################## +## COMPILATION RULES ## +########################################################## + +.DEFAULT_GOAL := $(PROG) + +# Explicit rules for local files (explicit rules always beat pattern rules) +./build/Probes2tecplot.o: ./Probes2tecplot.f90 + @echo + @echo $< + $(FC) $(FFLAGS) $(MACROS) $(EXTLIB_INC) -I./include $(INCLUDE) $(INCLUDE)_ns $(MACROS) -c $< -o $@ $(MOD_FLAG) ./include + +./build/main.o: ./main.f90 + @echo + @echo $< + $(FC) $(FFLAGS) $(MACROS) $(EXTLIB_INC) -I./include $(INCLUDE) $(INCLUDE)_ns $(MACROS) -c $< -o $@ $(MOD_FLAG) ./include + +# Pattern rule for shared files from horses2tecplot +./build/%.o: $(H2T_DIR)/%.f90 + @echo + @echo $< + $(FC) $(FFLAGS) $(MACROS) $(EXTLIB_INC) -I./include $(INCLUDE) $(INCLUDE)_ns $(MACROS) -c $< -o $@ $(MOD_FLAG) ./include + +$(PROG): header mkdirs $(ALL_OBJS_BUILD) + @echo + @echo + @echo "---------------------------------" + @echo "| Linking " $@ "|" + @echo "---------------------------------" + $(FC) $(FFLAGS) $(EXTLIB_INC) -o $(PROG) $(ALL_OBJS_BUILD) $(HORSES_LIBS_EXEC) $(LIBS) + + +install: $(PROG) + cp -p $(PROG) $(INSTALL_DIR)/bin + cp -p ./include/*.mod $(INSTALL_DIR)/include + @echo + @echo + +########################################## +## Extra auxiliary rules ## +########################################## +clean: FORCE + rm -f ./include/*.mod ./lib/*.a ./build/*.o + rm -f $(PROG) + +mkdirs: FORCE + @mkdir -p ./build + @mkdir -p ./include + @mkdir -p ./bin + @mkdir -p ./lib + +header: FORCE + @echo + @echo "================================" + @echo ">> Building probes2tecplot" + @echo "================================" + @echo + @echo + +## Dummy target to make sure a rule is executed +FORCE: \ No newline at end of file diff --git a/Solver/src/addons/probes2tecplot/Probes2tecplot.f90 b/Solver/src/addons/probes2tecplot/Probes2tecplot.f90 new file mode 100644 index 000000000..9cc973053 --- /dev/null +++ b/Solver/src/addons/probes2tecplot/Probes2tecplot.f90 @@ -0,0 +1,813 @@ +#include "Includes.h" +module Probes2tecplotModule +! +! /////////////////////////////////////////////////////////////////////////// +! +! This module implements the probes2tecplot functionality: +! - Reads a probes file with (name, x, y, z) entries +! - Finds each probe point in the mesh using Newton iteration +! accelerated with element bounding boxes +! - Caches probe locations (eID + xi) to a binary file for reuse +! - Interpolates the solution at each probe position using Lagrange basis +! - Writes output variables to a Tecplot .tec file +! +! /////////////////////////////////////////////////////////////////////////// +! + use SMConstants + use Storage + use SolutionFile + use SharedSpectralBasis + use NodalStorageClass + use OutputVariables + use PhysicsStorage + use Headers + implicit none + + private + public ProbeData_t + public ReadProbesFile + public FindAndCacheProbes + public InterpolateAndWriteProbes + + integer, parameter :: MAX_NEWTON = 100 + real(kind=RP), parameter :: NEWTON_TOL = 1.0e-10_RP + real(kind=RP), parameter :: INSIDE_TOL = 1.0e-6_RP + real(kind=RP), parameter :: BBOX_TOL = 1.0e-6_RP + integer, parameter :: MAX_WARN = 20 +#define PRECISION_FORMAT "(E18.10)" + + type ProbeData_t + character(len=LINE_LENGTH) :: name + real(kind=RP) :: x(NDIM) + real(kind=RP) :: xi(NDIM) + integer :: eID + logical :: found + end type ProbeData_t + + type ElementBBox_t + real(kind=RP) :: xmin(NDIM) + real(kind=RP) :: xmax(NDIM) + end type ElementBBox_t + + contains + + pure function getBaseName(fullPath) result(base) + character(len=*), intent(in) :: fullPath + character(len=LINE_LENGTH) :: base + integer :: slashPos, dotPos + + slashPos = index(trim(fullPath), '/', BACK=.true.) + base = fullPath(slashPos+1:) + dotPos = index(trim(base), '.', BACK=.true.) + if (dotPos > 1) base = base(1:dotPos-1) + + end function getBaseName + + pure function getDirName(fullPath) result(dir) + character(len=*), intent(in) :: fullPath + character(len=LINE_LENGTH) :: dir + integer :: slashPos + + slashPos = index(trim(fullPath), '/', BACK=.true.) + if (slashPos > 0) then + dir = fullPath(1:slashPos) + else + dir = "" + end if + + end function getDirName + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Read probe positions from file. +! Format: one probe per line -> name x y z +! Lines starting with # are comments. Name is optional. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine ReadProbesFile(fileName, probes, no_of_probes) + implicit none + character(len=*), intent(in) :: fileName + type(ProbeData_t), allocatable, intent(out) :: probes(:) + integer, intent(out) :: no_of_probes +! +! --------------- +! Local variables +! --------------- +! + integer :: fid, io, lineCount, pID, nread + character(len=512) :: line + real(kind=RP) :: x, y, z + character(len=LINE_LENGTH) :: name + + write(STD_OUT,'(/)') + call Section_Header("Reading probes file") + write(STD_OUT,'(30X,A,A)') "-> File: ", trim(fileName) + + open(newunit=fid, file=trim(fileName), action="read", status="old", iostat=io) + if (io .ne. 0) then + write(STD_OUT,'(30X,A,A,A)') "ERROR: Could not open probes file '", trim(fileName), "'" + no_of_probes = 0 + return + end if + + lineCount = 0 + do + read(fid, '(A)', iostat=io) line + if (io .ne. 0) exit + line = adjustl(line) + if (len_trim(line) .eq. 0) cycle + if (line(1:1) .eq. '#') cycle + lineCount = lineCount + 1 + end do + close(fid) + + no_of_probes = lineCount + if (no_of_probes .eq. 0) return + allocate(probes(no_of_probes)) + + open(newunit=fid, file=trim(fileName), action="read", status="old") + pID = 0 + do + read(fid, '(A)', iostat=io) line + if (io .ne. 0) exit + line = adjustl(line) + if (len_trim(line) .eq. 0) cycle + if (line(1:1) .eq. '#') cycle + pID = pID + 1 + read(line, *, iostat=nread) name, x, y, z + if (nread .ne. 0) then + read(line, *, iostat=nread) x, y, z + if (nread .ne. 0) then + write(STD_OUT,'(30X,A,I0,A)') "WARNING: Cannot parse probe line ", pID, ". Skipping." + pID = pID - 1 + cycle + end if + write(probes(pID)%name, '(A,I0)') "probe_", pID + else + probes(pID)%name = trim(name) + end if + probes(pID)%x(1) = x + probes(pID)%x(2) = y + probes(pID)%x(3) = z + probes(pID)%found = .false. + probes(pID)%eID = 0 + probes(pID)%xi = 0.0_RP + end do + no_of_probes = pID + close(fid) + write(STD_OUT,'(30X,A,I0,A)') "-> Found ", no_of_probes, " probe(s)." + + end subroutine ReadProbesFile + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Find all probes in the mesh (or load from cache if available). +! Cache file is named .pcache and stores eID + xi +! for each probe. It is invalidated if meshName or probesFileName change. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine FindAndCacheProbes(mesh, meshName, probesFileName, probes, no_of_probes) + use FileReadingUtilities, only: getFileName + implicit none + type(Mesh_t), intent(in) :: mesh + character(len=*), intent(in) :: meshName + character(len=*), intent(in) :: probesFileName + type(ProbeData_t), allocatable, intent(out) :: probes(:) + integer, intent(out) :: no_of_probes +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: cacheFileName + logical :: cacheLoaded + type(ElementBBox_t), allocatable :: bbox(:) + integer :: eID + integer(kind=8) :: t0, t1, count_rate + + cacheFileName = trim(getFileName(probesFileName)) // ".pcache" + + write(STD_OUT,'(/)') + call Section_Header("Probe location lookup") + write(STD_OUT,'(30X,A,A)') "-> Cache file: ", trim(cacheFileName) + + call system_clock(t0, count_rate) + call ReadProbeCache(cacheFileName, meshName, probesFileName, probes, no_of_probes, cacheLoaded) + + if (cacheLoaded) then + call system_clock(t1) + write(STD_OUT,'(30X,A,I0,A)') "-> Loaded ", no_of_probes, " probe locations from cache." + write(STD_OUT,'(30X,A,F8.3,A)') "-> Cache load time: ", & + real(t1-t0,RP)/real(count_rate,RP), " s" + return + end if + + write(STD_OUT,'(30X,A)') "-> Cache not found or stale. Searching mesh..." +! +! Read probe positions from file +! -------------------------------- + call ReadProbesFile(probesFileName, probes, no_of_probes) + if (no_of_probes .eq. 0) return +! +! Initialize spectral basis for mesh geometry orders +! --------------------------------------------------- + do eID = 1, mesh % no_of_elements + associate(e => mesh % elements(eID)) + call addNewSpectralBasis(spA, e % Nmesh, mesh % nodeType) + end associate + end do +! +! Build element bounding boxes +! ---------------------------- + call BuildBoundingBoxes(mesh, bbox) +! +! Find each probe in the mesh +! ---------------------------- + call Section_Header("Finding probes in mesh") + call system_clock(t0, count_rate) + call FindAllProbesInMesh(mesh, bbox, probes, no_of_probes) + call system_clock(t1) + write(STD_OUT,'(30X,A,F8.3,A)') "-> Mesh search time: ", & + real(t1-t0,RP)/real(count_rate,RP), " s" + + deallocate(bbox) +! +! Save cache for future runs +! -------------------------- + call SaveProbeCache(cacheFileName, meshName, probesFileName, probes, no_of_probes) + + end subroutine FindAndCacheProbes + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Read probe location cache from binary file. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine ReadProbeCache(cacheFileName, meshName, probesFileName, probes, no_of_probes, success) + implicit none + character(len=*), intent(in) :: cacheFileName + character(len=*), intent(in) :: meshName + character(len=*), intent(in) :: probesFileName + type(ProbeData_t), allocatable, intent(out) :: probes(:) + integer, intent(out) :: no_of_probes + logical, intent(out) :: success +! +! --------------- +! Local variables +! --------------- +! + integer :: fid, io, pID + character(len=LINE_LENGTH) :: stored_mesh, stored_probes + + success = .false. + no_of_probes = 0 + + open(newunit=fid, file=trim(cacheFileName), action="read", form="unformatted", & + access="stream", status="old", iostat=io) + if (io .ne. 0) return + + read(fid, iostat=io) stored_mesh ; if (io .ne. 0) then ; close(fid) ; return ; end if + read(fid, iostat=io) stored_probes; if (io .ne. 0) then ; close(fid) ; return ; end if + + if (trim(stored_mesh) .ne. trim(meshName) .or. & + trim(stored_probes) .ne. trim(probesFileName)) then + write(STD_OUT,'(30X,A)') "-> Cache is stale (mesh or probes file changed). Recomputing." + close(fid) + return + end if + + read(fid, iostat=io) no_of_probes ; if (io .ne. 0) then ; close(fid) ; return ; end if + + allocate(probes(no_of_probes)) + do pID = 1, no_of_probes + read(fid, iostat=io) probes(pID) % name, probes(pID) % x, & + probes(pID) % eID, probes(pID) % xi, probes(pID) % found + if (io .ne. 0) then + write(STD_OUT,'(30X,A)') "WARNING: Cache read error. Recomputing." + deallocate(probes) + no_of_probes = 0 + close(fid) + return + end if + end do + + close(fid) + success = .true. + + end subroutine ReadProbeCache + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Save probe location cache to binary file. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine SaveProbeCache(cacheFileName, meshName, probesFileName, probes, no_of_probes) + implicit none + character(len=*), intent(in) :: cacheFileName + character(len=*), intent(in) :: meshName + character(len=*), intent(in) :: probesFileName + type(ProbeData_t), intent(in) :: probes(:) + integer, intent(in) :: no_of_probes +! +! --------------- +! Local variables +! --------------- +! + integer :: fid, io, pID + character(len=LINE_LENGTH) :: stored_mesh, stored_probes + + stored_mesh = trim(meshName) + stored_probes = trim(probesFileName) + + open(newunit=fid, file=trim(cacheFileName), action="write", form="unformatted", & + access="stream", status="replace", iostat=io) + if (io .ne. 0) then + write(STD_OUT,'(30X,A,A)') "WARNING: Could not write cache file: ", trim(cacheFileName) + return + end if + + write(fid) stored_mesh + write(fid) stored_probes + write(fid) no_of_probes + do pID = 1, no_of_probes + write(fid) probes(pID) % name, probes(pID) % x, & + probes(pID) % eID, probes(pID) % xi, probes(pID) % found + end do + close(fid) + + write(STD_OUT,'(30X,A,A)') "-> Cache saved: ", trim(cacheFileName) + + end subroutine SaveProbeCache + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Build axis-aligned bounding boxes for all elements. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine BuildBoundingBoxes(mesh, bbox) + implicit none + type(Mesh_t), intent(in) :: mesh + type(ElementBBox_t), allocatable, intent(out) :: bbox(:) +! +! --------------- +! Local variables +! --------------- +! + integer :: e, i, j, k + + allocate(bbox(mesh % no_of_elements)) + + do e = 1, mesh % no_of_elements + associate(elem => mesh % elements(e)) + bbox(e) % xmin = huge(1.0_RP) + bbox(e) % xmax = -huge(1.0_RP) + do k = 0, elem % Nmesh(3) + do j = 0, elem % Nmesh(2) + do i = 0, elem % Nmesh(1) + bbox(e) % xmin = min(bbox(e) % xmin, elem % x(:,i,j,k)) + bbox(e) % xmax = max(bbox(e) % xmax, elem % x(:,i,j,k)) + end do + end do + end do + end associate + end do + + end subroutine BuildBoundingBoxes + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Find all probes in the mesh using Newton iteration with bounding box filter. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine FindAllProbesInMesh(mesh, bbox, probes, no_of_probes) + implicit none + type(Mesh_t), intent(in) :: mesh + type(ElementBBox_t), intent(in) :: bbox(:) + type(ProbeData_t), intent(inout) :: probes(:) + integer, intent(in) :: no_of_probes +! +! --------------- +! Local variables +! --------------- +! + integer :: pID, found_count, warn_count, processed_count, new_count, milestone + integer(kind=8) :: t0, t1, count_rate + + found_count = 0 + warn_count = 0 + processed_count = 0 + milestone = max(1, no_of_probes / 20) ! 5% steps + + write(STD_OUT,'(30X,A,I0,A)') "-> Searching ", no_of_probes, " probes..." + + call system_clock(t0, count_rate) + + !$OMP PARALLEL DO SCHEDULE(DYNAMIC,100) REDUCTION(+:found_count) DEFAULT(SHARED) PRIVATE(new_count) + do pID = 1, no_of_probes + call FindProbeInMesh(mesh, bbox, probes(pID) % x, & + probes(pID) % eID, probes(pID) % xi, probes(pID) % found) + if (probes(pID) % found) found_count = found_count + 1 + !$OMP ATOMIC CAPTURE + processed_count = processed_count + 1 + new_count = processed_count + !$OMP END ATOMIC + if (mod(new_count, milestone) .eq. 0) then + !$OMP CRITICAL (progress_bar) + write(STD_OUT,'(30X,A,I0,A,I0,A,F5.1,A)') & + " [", new_count, "/", no_of_probes, & + "] ", 100.0_RP * new_count / no_of_probes, "%" + !$OMP END CRITICAL (progress_bar) + end if + end do + !$OMP END PARALLEL DO + + call system_clock(t1) + write(STD_OUT,'(30X,A)') " [done]" + write(STD_OUT,'(30X,A,F8.3,A)') "-> Parallel search time: ", & + real(t1-t0,RP)/real(count_rate,RP), " s" + + ! Print warnings sequentially after the parallel search + do pID = 1, no_of_probes + if (.not. probes(pID) % found) then + warn_count = warn_count + 1 + if (warn_count .le. MAX_WARN) then + write(STD_OUT,'(30X,A,A,A,3(ES12.4,1X),A)') "-> WARNING: Probe '", & + trim(probes(pID) % name), "' at (", probes(pID) % x, ") not found in mesh." + end if + end if + end do + + if (warn_count .gt. MAX_WARN) then + write(STD_OUT,'(30X,A,I0,A)') " (", warn_count - MAX_WARN, " additional not-found probes suppressed)" + end if + write(STD_OUT,'(30X,A,I0,A,I0,A)') "-> ", found_count, " of ", no_of_probes, " probes found in mesh." + + end subroutine FindAllProbesInMesh + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Search all elements for x_probe, using bounding box pre-filter. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine FindProbeInMesh(mesh, bbox, x_probe, eID, xi, found) + implicit none + type(Mesh_t), intent(in) :: mesh + type(ElementBBox_t), intent(in) :: bbox(:) + real(kind=RP), intent(in) :: x_probe(NDIM) + integer, intent(out) :: eID + real(kind=RP), intent(out) :: xi(NDIM) + logical, intent(out) :: found +! +! --------------- +! Local variables +! --------------- +! + integer :: e + + found = .false. + eID = 0 + + do e = 1, mesh % no_of_elements + if (any(x_probe < bbox(e) % xmin - BBOX_TOL)) cycle + if (any(x_probe > bbox(e) % xmax + BBOX_TOL)) cycle + call FindPointInElement(mesh % elements(e), mesh % is2D, x_probe, xi, found) + if (found) then + eID = e + return + end if + end do + + end subroutine FindProbeInMesh + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Newton iteration to find reference coordinates (xi,eta,zeta) of x_probe +! inside element e. Returns found=.true. if the point is inside the element. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine FindPointInElement(e, is2D, x_probe, xi, found) + implicit none + type(Element_t), intent(in) :: e + logical, intent(in) :: is2D + real(kind=RP), intent(in) :: x_probe(NDIM) + real(kind=RP), intent(out) :: xi(NDIM) + logical, intent(out) :: found +! +! --------------- +! Local variables +! --------------- +! + real(kind=RP) :: Lxi (0:e % Nmesh(1)) + real(kind=RP) :: Leta (0:e % Nmesh(2)) + real(kind=RP) :: Lzeta (0:e % Nmesh(3)) + real(kind=RP) :: dLxi (0:e % Nmesh(1)) + real(kind=RP) :: dLeta (0:e % Nmesh(2)) + real(kind=RP) :: dLzeta(0:e % Nmesh(3)) + real(kind=RP) :: x_calc(NDIM), F(NDIM), Jac(NDIM,NDIM), dxi_step(NDIM) + integer :: iter, i, j, k + + xi = 0.0_RP + found = .false. + + do iter = 1, MAX_NEWTON + + Lxi = spA(e % Nmesh(1)) % lj(xi(1)) + Leta = spA(e % Nmesh(2)) % lj(xi(2)) + dLxi = spA(e % Nmesh(1)) % dlj(xi(1)) + dLeta = spA(e % Nmesh(2)) % dlj(xi(2)) + + if (is2D) then + Lzeta(0) = 1.0_RP + dLzeta(0) = 0.0_RP + else + Lzeta = spA(e % Nmesh(3)) % lj(xi(3)) + dLzeta = spA(e % Nmesh(3)) % dlj(xi(3)) + end if + + x_calc = 0.0_RP + Jac = 0.0_RP + do k = 0, e % Nmesh(3) + do j = 0, e % Nmesh(2) + do i = 0, e % Nmesh(1) + x_calc = x_calc + e % x(:,i,j,k) * Lxi(i) * Leta(j) * Lzeta(k) + Jac(:,1) = Jac(:,1) + e % x(:,i,j,k) * dLxi(i)* Leta(j) * Lzeta(k) + Jac(:,2) = Jac(:,2) + e % x(:,i,j,k) * Lxi(i) * dLeta(j) * Lzeta(k) + Jac(:,3) = Jac(:,3) + e % x(:,i,j,k) * Lxi(i) * Leta(j) * dLzeta(k) + end do + end do + end do + + F = x_calc - x_probe + if (norm2(F) < NEWTON_TOL) exit + + dxi_step = Solve3x3(Jac, F) + xi = xi - dxi_step + + xi(1) = max(min(xi(1), 2.0_RP), -2.0_RP) + xi(2) = max(min(xi(2), 2.0_RP), -2.0_RP) + if (.not. is2D) xi(3) = max(min(xi(3), 2.0_RP), -2.0_RP) + + end do + + if (is2D) then + found = (abs(xi(1)) <= 1.0_RP + INSIDE_TOL) .and. & + (abs(xi(2)) <= 1.0_RP + INSIDE_TOL) + else + found = all(abs(xi) <= 1.0_RP + INSIDE_TOL) + end if + + end subroutine FindPointInElement + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Interpolate conservative variables Q at reference coordinates xi. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + function InterpolateQAtProbe(e, xi, nodeType) result(Q_probe) + implicit none + type(Element_t), intent(in) :: e + real(kind=RP), intent(in) :: xi(NDIM) + integer, intent(in) :: nodeType + real(kind=RP) :: Q_probe(NVARS) +! +! --------------- +! Local variables +! --------------- +! + real(kind=RP) :: Lxi (0:e % Nsol(1)) + real(kind=RP) :: Leta (0:e % Nsol(2)) + real(kind=RP) :: Lzeta(0:e % Nsol(3)) + integer :: i, j, k + + Lxi = spA(e % Nsol(1)) % lj(xi(1)) + Leta = spA(e % Nsol(2)) % lj(xi(2)) + + if (e % Nsol(3) .eq. 0) then + Lzeta(0) = 1.0_RP + else + Lzeta = spA(e % Nsol(3)) % lj(xi(3)) + end if + + Q_probe = 0.0_RP + do k = 0, e % Nsol(3) + do j = 0, e % Nsol(2) + do i = 0, e % Nsol(1) + Q_probe = Q_probe + e % Q(:,i,j,k) * Lxi(i) * Leta(j) * Lzeta(k) + end do + end do + end do + + end function InterpolateQAtProbe + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! For each solution: interpolate Q at all probe locations, compute output +! variables and write a Tecplot .tec file. The mock element is allocated +! once and reused across probes. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine InterpolateAndWriteProbes(mesh, probes, no_of_probes, solutionName, probesFileName) + use FileReadingUtilities, only: getFileName + implicit none + type(Mesh_t), intent(inout) :: mesh + type(ProbeData_t), intent(in) :: probes(:) + integer, intent(in) :: no_of_probes + character(len=*), intent(in) :: solutionName + character(len=*), intent(in) :: probesFileName +! +! --------------- +! Local variables +! --------------- +! + type(Element_t) :: mock_e + real(kind=RP), allocatable :: output(:,:,:,:) + real(kind=RP), allocatable :: outVars(:,:) + real(kind=RP), allocatable :: Q_all(:,:) + integer :: pID, fid, eID, found_count + integer(kind=8) :: t_total, t_end, count_rate + character(len=LINE_LENGTH) :: outputFile, formatout + character(len=1024) :: title + + write(STD_OUT,'(/)') + call Section_Header("Probes extraction") + call system_clock(t_total, count_rate) + found_count = count(probes(1:no_of_probes) % found) + write(STD_OUT,'(30X,A,I0,A,I0,A)') "-> Extracting ", found_count, & + " of ", no_of_probes, " probes..." +! +! Initialize spectral basis for solution orders +! ----------------------------------------------- + do eID = 1, mesh % no_of_elements + associate(e => mesh % elements(eID)) + call addNewSpectralBasis(spA, e % Nsol, mesh % nodeType) + end associate + end do +! +! Set up output variable metadata +! --------------------------------- + call getOutputVariables() + allocate(output (no_of_outputVariables, 0:0, 0:0, 0:0)) + allocate(outVars(no_of_outputVariables, no_of_probes)) + outVars = 0.0_RP +! +! Allocate mock element once — reused for every probe +! ----------------------------------------------------- + mock_e % Nout = 0 + allocate(mock_e % Qout (NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % xOut (NDIM, 0:0, 0:0, 0:0)) + allocate(mock_e % QDot_out(NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % U_xout (NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % U_yout (NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % U_zout (NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % statsout (1, 0:0, 0:0, 0:0)) + allocate(mock_e % mu_NSout (1, 0:0, 0:0, 0:0)) + allocate(mock_e % ut_NSout (1, 0:0, 0:0, 0:0)) + allocate(mock_e % wallYout (1, 0:0, 0:0, 0:0)) + allocate(mock_e % mu_sgsout(1, 0:0, 0:0, 0:0)) + allocate(mock_e % wallY (1, 0:0, 0:0, 0:0)) + allocate(mock_e % ut_NS (1, 0:0, 0:0, 0:0)) + + mock_e % QDot_out = 0.0_RP + mock_e % U_xout = 0.0_RP + mock_e % U_yout = 0.0_RP + mock_e % U_zout = 0.0_RP + mock_e % wallY = 0.0_RP + mock_e % ut_NS = 0.0_RP + mock_e % statsout = 0.0_RP + mock_e % mu_NSout = 0.0_RP + mock_e % ut_NSout = 0.0_RP + mock_e % wallYout = 0.0_RP + mock_e % mu_sgsout = 0.0_RP + mock_e % sensor = 0.0_RP +! +! Phase 1: interpolate Q at all probe locations (parallel) +! ---------------------------------------------------------- + allocate(Q_all(NVARS, no_of_probes)) + Q_all = 0.0_RP + + !$OMP PARALLEL DO SCHEDULE(DYNAMIC,256) DEFAULT(SHARED) + do pID = 1, no_of_probes + if (.not. probes(pID) % found) cycle + Q_all(:,pID) = InterpolateQAtProbe(mesh % elements(probes(pID) % eID), & + probes(pID) % xi, mesh % nodeType) + end do + !$OMP END PARALLEL DO +! +! Phase 2: compute output variables (sequential — shared mock element) +! --------------------------------------------------------------------- + do pID = 1, no_of_probes + if (.not. probes(pID) % found) cycle + + mock_e % Qout(:,0,0,0) = Q_all(:,pID) + mock_e % xOut(:,0,0,0) = probes(pID) % x + + call ComputeOutputVariables(no_of_outputVariables, outputVariableNames, & + mock_e % Nout, mock_e, output, mesh % refs, & + .false., .false., .false.) + outVars(:,pID) = output(:,0,0,0) + end do + + deallocate(Q_all) +! +! Deallocate mock element +! ------------------------ + deallocate(mock_e % Qout, mock_e % xOut, mock_e % QDot_out) + deallocate(mock_e % U_xout, mock_e % U_yout, mock_e % U_zout) + deallocate(mock_e % statsout, mock_e % mu_NSout, mock_e % ut_NSout) + deallocate(mock_e % wallYout, mock_e % mu_sgsout) + deallocate(mock_e % wallY, mock_e % ut_NS) + deallocate(output) +! +! Write output .tec file +! ----------------------- + outputFile = trim(getDirName(solutionName)) // trim(getBaseName(solutionName)) // & + "." // trim(getBaseName(probesFileName)) // ".tec" + write(STD_OUT,'(30X,A,A)') "-> Output file: ", trim(outputFile) + + open(newunit=fid, file=trim(outputFile), action="write", status="unknown") + + write(title,'(A,A,A)') '"Probes extracted from ', trim(solutionName), '"' + write(fid,'(A,A)') "TITLE = ", trim(title) + write(fid,'(A,A)') 'VARIABLES = "x","y","z"', trim(getOutputVariablesLabel()) + write(fid,'(A,I0,A)') 'ZONE T="Probes", I=', no_of_probes, ', J=1, K=1, F=POINT' + + write(formatout,'(A,I0,A,A)') "(", 3 + no_of_outputVariables, PRECISION_FORMAT, ")" + + do pID = 1, no_of_probes + write(fid, trim(formatout)) probes(pID) % x, outVars(:,pID) + end do + + close(fid) + call system_clock(t_end) + write(STD_OUT,'(30X,A,I0,A,I0,A,F8.3,A)') "-> Written ", found_count, & + " of ", no_of_probes, " probes in ", & + real(t_end-t_total,8)/real(count_rate,8), " s" + + deallocate(outVars) + + end subroutine InterpolateAndWriteProbes + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Solve a 3x3 linear system A*x = b using Cramer's rule. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + pure function Solve3x3(A, b) result(x) + implicit none + real(kind=RP), intent(in) :: A(3,3), b(3) + real(kind=RP) :: x(3) +! +! --------------- +! Local variables +! --------------- +! + real(kind=RP) :: det, inv_det + + det = A(1,1)*(A(2,2)*A(3,3) - A(2,3)*A(3,2)) & + - A(1,2)*(A(2,1)*A(3,3) - A(2,3)*A(3,1)) & + + A(1,3)*(A(2,1)*A(3,2) - A(2,2)*A(3,1)) + + if (abs(det) < 1.0e-30_RP) then + x = 0.0_RP + return + end if + + inv_det = 1.0_RP / det + + x(1) = inv_det * ( b(1)*(A(2,2)*A(3,3) - A(2,3)*A(3,2)) & + - A(1,2)*(b(2)*A(3,3) - A(2,3)*b(3)) & + + A(1,3)*(b(2)*A(3,2) - A(2,2)*b(3)) ) + + x(2) = inv_det * ( A(1,1)*(b(2)*A(3,3) - A(2,3)*b(3)) & + - b(1)*(A(2,1)*A(3,3) - A(2,3)*A(3,1)) & + + A(1,3)*(A(2,1)*b(3) - b(2)*A(3,1)) ) + + x(3) = inv_det * ( A(1,1)*(A(2,2)*b(3) - b(2)*A(3,2)) & + - A(1,2)*(A(2,1)*b(3) - b(2)*A(3,1)) & + + b(1)*(A(2,1)*A(3,2) - A(2,2)*A(3,1)) ) + + end function Solve3x3 + +end module Probes2tecplotModule diff --git a/Solver/src/addons/probes2tecplot/main.f90 b/Solver/src/addons/probes2tecplot/main.f90 new file mode 100644 index 000000000..742c1a310 --- /dev/null +++ b/Solver/src/addons/probes2tecplot/main.f90 @@ -0,0 +1,267 @@ +! +!/////////////////////////////////////////////////////////////////////////////////////// +! +! probes2tecplot - Extract solution variables at probe points from HORSES3D solution files +! +! Usage (control file): +! probes2tecplot control_file.convert +! +! Usage (command line): +! probes2tecplot mesh.hmesh solution.hsol probes.dat [--output-variables=p,u,v,w] +! +! Control file format: +! hmesh file= Mesh.hmesh +! hsol file= Solution.hsol +! probes file= probes.dat +! output variables= p, u, v, w +! flow equations= ns +! boundary file= optional +! partition file= optional +! +! Probes file format (one probe per line, lines starting with # are comments): +! probe_name x y z +! +! Probe location cache: +! On the first run, probe locations (element ID + reference coordinates) are +! saved to .pcache. Subsequent runs with the same mesh and probes +! file load directly from cache, skipping the mesh search entirely. +! +!/////////////////////////////////////////////////////////////////////////////////////// +! +program probes2tecplot + use SMConstants + use Storage + use SolutionFile + use SharedSpectralBasis + use Headers + use MPI_Process_Info + use OutputVariables, only: outScale, hasVariablesFlag, askedVariables, Lreference, getNoOfCommas + use Probes2tecplotModule + implicit none + + integer :: no_of_solutions + character(len=LINE_LENGTH) :: meshName + character(len=LINE_LENGTH) :: probesFileName + character(len=LINE_LENGTH), allocatable :: solutionNames(:) + integer :: iSol + type(Mesh_t) :: mesh + type(ProbeData_t), allocatable :: probes(:) + integer :: no_of_probes + integer(kind=8) :: t_sol, t_end, sol_rate + + call MPI_Process % Init + call Main_Header("HORSES Probes to TecPlot extraction utility", __DATE__, __TIME__) +! +! Parse the control file or command line arguments +! ------------------------------------------------- + call getProbes2tecplotArgs(meshName, probesFileName, no_of_solutions, solutionNames) +! +! Construct spectral basis +! ------------------------ + call ConstructSpectralBasis +! +! Read mesh +! --------- + call mesh % ReadMesh(meshName) +! +! Find probes in mesh (or load from cache) — done only once +! ---------------------------------------------------------- + call FindAndCacheProbes(mesh, trim(meshName), trim(probesFileName), probes, no_of_probes) +! +! Process each solution file +! -------------------------- + if (no_of_probes .gt. 0) then + do iSol = 1, no_of_solutions + write(STD_OUT,'(/,/)') + call Section_Header("Processing solution") + call system_clock(t_sol, sol_rate) + call mesh % ReadSolution(solutionNames(iSol)) + call system_clock(t_end) + write(STD_OUT,'(30X,A,A)') "-> Solution: ", trim(solutionNames(iSol)) + write(STD_OUT,'(30X,A,F8.3,A)') "-> Read time: ", & + real(t_end-t_sol,8)/real(sol_rate,8), " s" + call InterpolateAndWriteProbes(mesh, probes, no_of_probes, solutionNames(iSol), trim(probesFileName)) + call system_clock(t_end) + write(STD_OUT,'(30X,A,F8.3,A)') "-> Total time: ", & + real(t_end-t_sol,8)/real(sol_rate,8), " s" + end do + end if +! +! Cleanup +! ------- + if (allocated(probes)) deallocate(probes) + call DestructSpectralBasis + call MPI_Process % Close + + contains + + subroutine getProbes2tecplotArgs(meshName, probesFileName, no_of_solutions, solutionNames) + use FTValueDictionaryClass, only: FTValueDictionary + use FileReaders , only: ReadControlFile + use FileReadingUtilities , only: getCharArrayFromString + use Utilities , only: toLower + implicit none + character(len=*), intent(out) :: meshName + character(len=*), intent(out) :: probesFileName + integer, intent(out) :: no_of_solutions + character(len=LINE_LENGTH), allocatable, intent(out) :: solutionNames(:) +! +! --------------- +! Local variables +! --------------- +! + integer :: no_of_arguments, i, pos, pos2, fID, io + character(len=LINE_LENGTH) :: arg, inputResultName + type(FTValueDictionary) :: controlVariables + logical :: useControlFile + + no_of_arguments = command_argument_count() + no_of_solutions = 0 +! +! Determine input mode: control file (1 arg) or command line (>1 args) +! --------------------------------------------------------------------- + useControlFile = (no_of_arguments .eq. 1) + + if (no_of_arguments .eq. 0) then + write(STD_OUT,'(A)') "Usage: probes2tecplot control_file.convert" + write(STD_OUT,'(A)') " probes2tecplot mesh.hmesh solution.hsol probes.dat [--output-variables=p,u,v,w]" + call exit(1) + end if +! +! ---- Control file mode ---- +! --------------------------- + if (useControlFile) then + call controlVariables % initWithSize(16) + call ReadControlFile(controlVariables) + + if (.not. controlVariables % containsKey("hmesh file")) then + write(STD_OUT,'(A)') "ERROR: 'hmesh file' not specified in control file." + call exit(1) + end if + meshName = controlVariables % stringValueForKey("hmesh file", LINE_LENGTH) + + if (.not. controlVariables % containsKey("probes file")) then + write(STD_OUT,'(A)') "ERROR: 'probes file' not specified in control file." + call exit(1) + end if + probesFileName = controlVariables % stringValueForKey("probes file", LINE_LENGTH) + + if (.not. controlVariables % containsKey("hsol file")) then + write(STD_OUT,'(A)') "ERROR: 'hsol file' not specified in control file." + call exit(1) + end if + + inputResultName = controlVariables % stringValueForKey("hsol file", LINE_LENGTH) + no_of_solutions = getNoOfCommas(trim(inputResultName)) + 1 + allocate(solutionNames(no_of_solutions)) + + if (no_of_solutions .eq. 1) then + solutionNames(1) = adjustl(trim(inputResultName)) + else + pos = 0 + do i = 1, no_of_solutions - 1 + pos2 = index(trim(inputResultName(pos+1:)), ",") + pos + solutionNames(i) = trim(adjustl(inputResultName(pos+1:pos2-1))) + pos = pos2 + end do + pos = index(trim(inputResultName), ",", BACK=.true.) + solutionNames(no_of_solutions) = adjustl(trim(inputResultName(pos+1:))) + end if + + hasVariablesFlag = controlVariables % containsKey("output variables") + if (hasVariablesFlag) askedVariables = controlVariables % stringValueForKey("output variables", LINE_LENGTH) + outScale = .not. controlVariables % logicalValueForKey("dimensionless") + Lreference = controlVariables % getValueOrDefault("reference length (m)", 1.0_RP) + + hasBoundaries = controlVariables % containsKey("boundary file") + if (hasBoundaries) boundaryFileName = controlVariables % stringValueForKey("boundary file", LINE_LENGTH) + hasMPIranks = controlVariables % containsKey("partition file") + if (hasMPIranks) partitionFileName = controlVariables % stringValueForKey("partition file", LINE_LENGTH) + + if (controlVariables % containsKey("flow equations")) then + flowEq = controlVariables % stringValueForKey("flow equations", LINE_LENGTH) + call toLower(flowEq) + else + flowEq = "ns" + end if +! +! ---- Command line mode ---- +! --------------------------- + else + outScale = .true. + hasVariablesFlag = .false. + hasBoundaries = .false. + hasMPIranks = .false. + flowEq = "ns" + Lreference = 1.0_RP + probesFileName = "" + + do i = 1, no_of_arguments + call get_command_argument(i, arg) + open(newunit=fID, file=trim(arg), action="read", form="unformatted", & + access="stream", iostat=io) + close(fID) + if (io .ne. 0) cycle + if (getSolutionFileType(arg) .eq. MESH_FILE .or. & + getSolutionFileType(arg) .eq. ZONE_MESH_FILE) then + meshName = trim(arg) + exit + end if + end do + + no_of_solutions = 0 + do i = 1, no_of_arguments + call get_command_argument(i, arg) + open(newunit=fID, file=trim(arg), action="read", form="unformatted", & + access="stream", iostat=io) + close(fID) + if (io .ne. 0) cycle + if (getSolutionFileType(arg) .ne. MESH_FILE .and. & + getSolutionFileType(arg) .ne. ZONE_MESH_FILE) then + no_of_solutions = no_of_solutions + 1 + end if + end do + + allocate(solutionNames(max(1, no_of_solutions))) + pos = 0 + do i = 1, no_of_arguments + call get_command_argument(i, arg) + if (arg(1:2) .eq. "--") then + if (index(arg, "--output-variables=") .ne. 0) then + hasVariablesFlag = .true. + askedVariables = arg(len("--output-variables=")+1:) + else if (index(arg, "--dimensionless") .ne. 0) then + outScale = .false. + end if + cycle + end if + + open(newunit=fID, file=trim(arg), action="read", form="unformatted", & + access="stream", iostat=io) + close(fID) + + if (io .eq. 0) then + if (getSolutionFileType(arg) .ne. MESH_FILE .and. & + getSolutionFileType(arg) .ne. ZONE_MESH_FILE) then + pos = pos + 1 + solutionNames(pos) = trim(arg) + end if + else + open(newunit=fID, file=trim(arg), action="read", status="old", iostat=io) + close(fID) + if (io .eq. 0 .and. len_trim(probesFileName) .eq. 0) then + probesFileName = trim(arg) + end if + end if + end do + + if (len_trim(probesFileName) .eq. 0) then + write(STD_OUT,'(A)') "ERROR: No probes file found in arguments." + write(STD_OUT,'(A)') "Usage: probes2tecplot mesh.hmesh solution.hsol probes.dat" + call exit(1) + end if + end if + + end subroutine getProbes2tecplotArgs + +end program probes2tecplot \ No newline at end of file diff --git a/Solver/src/libs/discretization/Makefile b/Solver/src/libs/discretization/Makefile index ef42e362d..525127b8c 100644 --- a/Solver/src/libs/discretization/Makefile +++ b/Solver/src/libs/discretization/Makefile @@ -96,6 +96,54 @@ LIB = discretization all: lib$(LIB)_ns.a lib$(LIB)_nssa.a lib$(LIB)_ins.a lib$(LIB)_ch.a lib$(LIB)_mu.a lib$(LIB)_caa.a +# nvfortran GPU: strip -fopenmp from files with mixed OMP/ACC directives +FFLAGS_NO_OMP := $(filter-out -fopenmp -mp,$(FFLAGS)) + +./build_ns/EllipticBR1.o \ +./build_ns/EllipticBR2.o \ +./build_ns/EllipticIP.o \ +./build_ns/DGSEMClass.o: ./build_ns/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -I./include_ns $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_ns -c $< -o $@ $(MOD_FLAG) ./include_ns + +./build_nssa/EllipticBR1.o \ +./build_nssa/EllipticBR2.o \ +./build_nssa/EllipticIP.o \ +./build_nssa/DGSEMClass.o: ./build_nssa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -DSPALARTALMARAS -I./include_nssa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_nssa -c $< -o $@ $(MOD_FLAG) ./include_nssa + +./build_ins/EllipticBR1.o \ +./build_ins/EllipticBR2.o \ +./build_ins/EllipticIP.o \ +./build_ins/DGSEMClass.o: ./build_ins/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DINCNS -I./include_ins $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_ins -c $< -o $@ $(MOD_FLAG) ./include_ins + +./build_ch/EllipticBR1.o \ +./build_ch/EllipticBR2.o \ +./build_ch/EllipticIP.o \ +./build_ch/DGSEMClass.o: ./build_ch/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DCAHNHILLIARD -I./include_ch $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_ch -c $< -o $@ $(MOD_FLAG) ./include_ch + +./build_mu/EllipticBR1.o \ +./build_mu/EllipticBR2.o \ +./build_mu/EllipticIP.o \ +./build_mu/DGSEMClass.o: ./build_mu/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DMULTIPHASE -DCAHNHILLIARD -I./include_mu $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_mu -c $< -o $@ $(MOD_FLAG) ./include_mu + +./build_caa/DGSEMClass.o: ./build_caa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DACOUSTIC -I./include_caa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_caa -c $< -o $@ $(MOD_FLAG) ./include_caa + ./build_ns/%.o: %.f90 @echo @echo $< diff --git a/Solver/src/libs/discretization/SCsensorClass.f90 b/Solver/src/libs/discretization/SCsensorClass.f90 index 06e7ef0af..e55044370 100755 --- a/Solver/src/libs/discretization/SCsensorClass.f90 +++ b/Solver/src/libs/discretization/SCsensorClass.f90 @@ -457,7 +457,6 @@ subroutine Compute_SCsensor(sensor, sem, t) ! Local variables ! --------------- ! - type(Element), pointer :: e integer :: eID real(RP) :: s @@ -469,28 +468,27 @@ subroutine Compute_SCsensor(sensor, sem, t) ! Add 'inertia' to the scaled value ! --------------------------------- if (sensor % min_steps > 1) then ! Enter the loop only if necessary -!$omp parallel do default(private) shared(sem) +!$omp parallel do default(private) shared(sem, sensor) do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) - s = e % storage % sensor + associate(st => sem % mesh % elements(eID) % storage) + s = st % sensor if (s > 0.0_RP) then - if (e % storage % prev_sensor <= 0.0_RP) then - e % storage % first_sensed = 0 - e % storage % prev_sensor = s + if (st % prev_sensor <= 0.0_RP) then + st % first_sensed = 0 + st % prev_sensor = s else - e % storage % first_sensed = e % storage % first_sensed + 1 - e % storage % prev_sensor = s + st % first_sensed = st % first_sensed + 1 + st % prev_sensor = s end if - elseif (e % storage % first_sensed < sensor % min_steps) then - e % storage % first_sensed = e % storage % first_sensed + 1 - e % storage % sensor = e % storage % prev_sensor + elseif (st % first_sensed < sensor % min_steps) then + st % first_sensed = st % first_sensed + 1 + st % sensor = st % prev_sensor end if + end associate end do !$omp end parallel do end if - nullify(e) - end subroutine Compute_SCsensor ! !/////////////////////////////////////////////////////////////////////////////// @@ -564,7 +562,6 @@ subroutine Sensor_integral_sqrt(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e integer :: eID integer :: i integer :: j @@ -574,7 +571,7 @@ subroutine Sensor_integral_sqrt(sensor, sem, t) !$omp parallel do default(private) shared(sem, sensor, NodalStorage) do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) val = 0.0_RP do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) contribution = GetSensedVariable( & @@ -592,12 +589,10 @@ subroutine Sensor_integral_sqrt(sensor, sem, t) end do ; end do ; end do e % storage % sensor = SinRamp(sensor, sqrt(val)) - + end associate end do !$omp end parallel do - nullify(e) - end subroutine Sensor_integral_sqrt ! !/////////////////////////////////////////////////////////////////////////////// @@ -615,7 +610,6 @@ subroutine Sensor_integral(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e integer :: eID integer :: i integer :: j @@ -625,7 +619,7 @@ subroutine Sensor_integral(sensor, sem, t) !$omp parallel do default(private) shared(sem, sensor, NodalStorage) do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) val = 0.0_RP do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) contribution = GetSensedVariable( & @@ -642,11 +636,10 @@ subroutine Sensor_integral(sensor, sem, t) * contribution end do ; end do ; end do e % storage % sensor = SinRamp(sensor, val) + end associate end do !$omp end parallel do - nullify(e) - end subroutine Sensor_integral ! !/////////////////////////////////////////////////////////////////////////////// @@ -669,7 +662,6 @@ subroutine Sensor_modal(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e integer :: eID integer :: i, j, k, r integer :: maxNx, maxNy, maxNz @@ -689,7 +681,7 @@ subroutine Sensor_modal(sensor, sem, t) !$omp parallel do default(private) shared(sem, sensor, NodalStorage) do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) Nx = e % Nxyz(1) Ny = e % Nxyz(2) Nz = e % Nxyz(3) @@ -778,11 +770,10 @@ subroutine Sensor_modal(sensor, sem, t) ! Sensor value as the ratio of num / den e % storage % sensor = SinRamp(sensor, log10( num / den )) - + end associate end do !$omp end parallel do - nullify(e) nullify(Lwx) nullify(Lwy) nullify(Lwz) @@ -821,8 +812,6 @@ subroutine Sensor_truncation(sensor, sem, t) real(RP) :: mTE procedure(UserDefinedSourceTermNS_f) :: UserDefinedSourceTermNS - type(Element), pointer :: e - type(Element), pointer :: ce type(DGSem), pointer :: csem @@ -832,13 +821,10 @@ subroutine Sensor_truncation(sensor, sem, t) ! --------------- !$omp parallel do default(private) shared(sem, csem) do eID = 1, sem % mesh % no_of_elements - - e => sem % mesh % elements(eID) - ce => csem % mesh % elements(eID) - + associate(e => sem % mesh % elements(eID), ce => csem % mesh % elements(eID)) call Interp3DArrays(NCONS, e % Nxyz, e % storage % Q, ce % Nxyz, ce % storage % Q) ce % storage % sensor = e % storage % sensor - + end associate end do !$omp end parallel do @@ -846,12 +832,9 @@ subroutine Sensor_truncation(sensor, sem, t) ! ! Maximum TE computation ! ---------------------- -!$omp parallel do default(private) shared(sem, csem) +!$omp parallel do default(private) shared(sem, csem, sensor) do eID = 1, sem % mesh % no_of_elements - - e => sem % mesh % elements(eID) - ce => csem % mesh % elements(eID) - + associate(e => sem % mesh % elements(eID), ce => csem % mesh % elements(eID)) ! Use G_NS as temporary storage for Qdot call Interp3DArrays(NCONS, e % Nxyz, e % storage % QDot, ce % Nxyz, ce % storage % G_NS) @@ -875,12 +858,10 @@ subroutine Sensor_truncation(sensor, sem, t) else e % storage % sensor = SinRamp(sensor, log10(mTE)) end if - + end associate end do !$omp end parallel do - nullify(e) - nullify(ce) nullify(csem) end subroutine Sensor_truncation @@ -907,7 +888,6 @@ subroutine Sensor_aliasing(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e type(NodalStorage_t), pointer :: spAxi, spAeta, spAzeta integer :: eID logical :: need_dealloc @@ -926,7 +906,7 @@ subroutine Sensor_aliasing(sensor, sem, t) select type (HyperbolicDiscretization) type is (SplitDG_t) - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) spAxi => NodalStorage(e % Nxyz(1)) spAeta => NodalStorage(e % Nxyz(2)) spAzeta => NodalStorage(e % Nxyz(3)) @@ -977,12 +957,12 @@ subroutine Sensor_aliasing(sensor, sem, t) else e % storage % sensor = SinRamp(sensor, log10(e % storage % sensor)) end if + end associate end select end do !$omp end parallel do - nullify(e) nullify(spAxi) nullify(spAeta) nullify(spAzeta) @@ -1011,7 +991,6 @@ subroutine Sensor_GMM(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e integer :: eID integer :: i, j, k integer :: cnt @@ -1028,9 +1007,7 @@ subroutine Sensor_GMM(sensor, sem, t) ! ----------------------------------------------------------------- cnt = 0 do eID = 1, sem % mesh % no_of_elements - - e => sem % mesh % elements(eID) - + associate(e => sem % mesh % elements(eID)) do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) cnt = cnt + 1 @@ -1113,7 +1090,7 @@ subroutine Sensor_GMM(sensor, sem, t) sensor % x(2,cnt) = dp(1)**2 + dp(2)**2 + dp(3)**2 end do ; end do ; end do - + end associate end do ! ! Rescale the values @@ -1131,7 +1108,7 @@ subroutine Sensor_GMM(sensor, sem, t) nclusters = sensor % gmm % nclusters cnt = 0 do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) if (nclusters <= 1) then e % storage % sensor = 0.0_RP else @@ -1140,10 +1117,9 @@ subroutine Sensor_GMM(sensor, sem, t) e % storage % sensor = real(cluster - 1, RP) / (nclusters - 1) end if cnt = cnt + n + end associate end do - nullify(e) - end subroutine Sensor_GMM ! !/////////////////////////////////////////////////////////////////////////////// diff --git a/Solver/src/libs/mesh/Makefile b/Solver/src/libs/mesh/Makefile index 04b73175f..68ab963b3 100644 --- a/Solver/src/libs/mesh/Makefile +++ b/Solver/src/libs/mesh/Makefile @@ -198,6 +198,39 @@ all: lib$(LIB)_ns.a lib$(LIB)_nssa.a lib$(LIB)_ins.a lib$(LIB)_ch.a lib$(LIB)_mu @echo $< $(FC) -cpp -O2 -ffree-line-length-0 -DACOUSTIC -I./include_caa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_caa -c $< -o $@ $(MOD_FLAG) ./include_caa +# nvfortran GPU: strip -fopenmp from files with mixed OMP/ACC directives +FFLAGS_NO_OMP := $(filter-out -fopenmp -mp,$(FFLAGS)) + +./build_ns/HexMesh.o ./build_ns/IBMClass.o: ./build_ns/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -I./include_ns $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ns -c $< -o $@ $(MOD_FLAG) ./include_ns + +./build_nssa/HexMesh.o ./build_nssa/IBMClass.o: ./build_nssa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -DSPALARTALMARAS -I./include_nssa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_nssa -c $< -o $@ $(MOD_FLAG) ./include_nssa + +./build_ins/HexMesh.o ./build_ins/IBMClass.o: ./build_ins/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DINCNS -I./include_ins $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ins -c $< -o $@ $(MOD_FLAG) ./include_ins + +./build_ch/HexMesh.o ./build_ch/IBMClass.o: ./build_ch/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DCAHNHILLIARD -I./include_ch $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ch -c $< -o $@ $(MOD_FLAG) ./include_ch + +./build_mu/HexMesh.o ./build_mu/IBMClass.o: ./build_mu/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DMULTIPHASE -DCAHNHILLIARD -I./include_mu $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_mu -c $< -o $@ $(MOD_FLAG) ./include_mu + +./build_caa/HexMesh.o ./build_caa/IBMClass.o: ./build_caa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DACOUSTIC -I./include_caa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_caa -c $< -o $@ $(MOD_FLAG) ./include_caa + ./build_ns/%.o: %.f90 @echo @echo $< diff --git a/Solver/src/libs/mesh/StorageClass.f90 b/Solver/src/libs/mesh/StorageClass.f90 index 182541918..161e54e00 100644 --- a/Solver/src/libs/mesh/StorageClass.f90 +++ b/Solver/src/libs/mesh/StorageClass.f90 @@ -1293,7 +1293,7 @@ end subroutine ElementStorage_SetStorageToCH_mu ! Interpolate solution to another element storage ! this % Q -> other % Q ! ----------------------------------------------- - impure elemental subroutine ElementStorage_InterpolateSolution(this,other,nodes,with_gradients) + subroutine ElementStorage_InterpolateSolution(this,other,nodes,with_gradients) implicit none !-arguments---------------------------------------------- class(ElementStorage_t), intent(in) :: this diff --git a/Solver/src/libs/monitors/Makefile b/Solver/src/libs/monitors/Makefile index 2f2f10635..b38152fea 100644 --- a/Solver/src/libs/monitors/Makefile +++ b/Solver/src/libs/monitors/Makefile @@ -88,6 +88,40 @@ LIB = monitors all: lib$(LIB)_ns.a lib$(LIB)_nssa.a lib$(LIB)_ins.a lib$(LIB)_ch.a lib$(LIB)_mu.a lib$(LIB)_caa.a +# nvfortran GPU: strip -fopenmp from files with mixed OMP/ACC directives +# (Probe.f90 and Monitors.f90 keep -fopenmp via the generic rule below) +FFLAGS_NO_OMP := $(filter-out -fopenmp -mp,$(FFLAGS)) + +./build_ns/SurfaceIntegrals.o ./build_ns/VolumeIntegrals.o: ./build_ns/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -I./include_ns $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ns -c $< -o $@ $(MOD_FLAG) ./include_ns + +./build_nssa/SurfaceIntegrals.o ./build_nssa/VolumeIntegrals.o: ./build_nssa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -DSPALARTALMARAS -I./include_nssa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_nssa -c $< -o $@ $(MOD_FLAG) ./include_nssa + +./build_ins/SurfaceIntegrals.o ./build_ins/VolumeIntegrals.o: ./build_ins/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DINCNS -I./include_ins $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ins -c $< -o $@ $(MOD_FLAG) ./include_ins + +./build_ch/SurfaceIntegrals.o ./build_ch/VolumeIntegrals.o: ./build_ch/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DCAHNHILLIARD -I./include_ch $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ch -c $< -o $@ $(MOD_FLAG) ./include_ch + +./build_mu/SurfaceIntegrals.o ./build_mu/VolumeIntegrals.o: ./build_mu/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DMULTIPHASE -DCAHNHILLIARD -I./include_mu $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_mu -c $< -o $@ $(MOD_FLAG) ./include_mu + +./build_caa/SurfaceIntegrals.o ./build_caa/VolumeIntegrals.o: ./build_caa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DACOUSTIC -I./include_caa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_caa -c $< -o $@ $(MOD_FLAG) ./include_caa + ./build_ns/%.o: %.f90 @echo @echo $< diff --git a/Solver/src/libs/monitors/Monitors.f90 b/Solver/src/libs/monitors/Monitors.f90 index 641be3051..eaa4f1cff 100644 --- a/Solver/src/libs/monitors/Monitors.f90 +++ b/Solver/src/libs/monitors/Monitors.f90 @@ -27,10 +27,17 @@ module MonitorsClass ! type Monitor_t character(len=LINE_LENGTH) :: solution_file + character(len=LINE_LENGTH) :: probes_solution_file = "" integer :: no_of_probes integer :: no_of_surfaceMonitors integer :: no_of_volumeMonitors integer :: no_of_loadBalancingMonitors + integer :: no_of_fileProbes = 0 + character(len=LINE_LENGTH) :: probesFileName = "" + character(len=STR_LEN_MONITORS), allocatable :: probesVariables(:) + real(kind=RP) :: probeFileSaveTimestep = 0.0_RP + character(len=8) :: probeFileOutputFormat = "ASCII" + real(kind=RP) :: fp_lastSavedTime = -huge(0.0_RP) integer :: bufferLine integer , allocatable :: iter(:) integer :: dt_restriction @@ -55,6 +62,7 @@ module MonitorsClass procedure :: WriteValues => Monitor_WriteValues procedure :: UpdateValues => Monitor_UpdateValues procedure :: WriteToFile => Monitor_WriteToFile + procedure :: WriteProbesFileSummary => Monitor_WriteProbesFileSummary procedure :: destruct => Monitor_Destruct procedure :: copy => Monitor_Assign generic :: assignment(=) => copy @@ -66,9 +74,10 @@ module MonitorsClass ! !/////////////////////////////////////////////////////////////////////////////////////// ! - subroutine Monitors_Construct( Monitors, mesh, controlVariables ) + subroutine Monitors_Construct( Monitors, mesh, controlVariables ) use FTValueDictionaryClass use mainKeywordsModule + use MPI_Process_Info implicit none class(Monitor_t) :: Monitors class(HexMesh), intent(in) :: mesh @@ -82,9 +91,19 @@ subroutine Monitors_Construct( Monitors, mesh, controlVariables ) integer :: fID , io integer :: i character(len=STR_LEN_MONITORS) :: line - character(len=STR_LEN_MONITORS) :: solution_file + character(len=STR_LEN_MONITORS) :: solution_file logical, save :: FirstCall = .TRUE. logical :: saveGradients + character(len=LINE_LENGTH) :: probesFileName + character(len=LINE_LENGTH) :: probesVariablesLine + character(len=STR_LEN_MONITORS), allocatable :: probesVariables(:) + integer :: no_of_fileProbes + integer :: no_of_probesVariables + real(kind=RP) :: probeFileSaveTimestep + character(len=8) :: probeFileOutputFormat + character(len=LINE_LENGTH) :: probes_solution_file + character(len=LINE_LENGTH) :: probes_dir + integer :: last_slash ! ! Setup the buffer ! ---------------- @@ -106,8 +125,24 @@ subroutine Monitors_Construct( Monitors, mesh, controlVariables ) solution_file = trim(getFileName(solution_file)) Monitors % solution_file = trim(solution_file) ! +! Build the probes output subdirectory: /probes/ +! ---------------------------------------------------------- + last_slash = index(trim(solution_file), '/', back=.true.) + if (last_slash .gt. 0) then + probes_dir = trim(solution_file(1:last_slash)) // "probes" + probes_solution_file = trim(solution_file(1:last_slash)) // "probes/" // trim(solution_file(last_slash+1:)) + else + probes_dir = "probes" + probes_solution_file = "probes/" // trim(solution_file) + end if + if (MPI_Process % isRoot) then + call execute_command_line('mkdir -p "' // trim(probes_dir) // '"', wait=.true.) + end if + Monitors % probes_solution_file = trim(probes_solution_file) +! ! Search in case file for probes, surface monitors, and volume monitors ! --------------------------------------------------------------------- + no_of_fileProbes = 0 if (mesh % child) then ! Return doing nothing if this is a child mesh Monitors % no_of_probes = 0 Monitors % no_of_surfaceMonitors = 0 @@ -115,6 +150,25 @@ subroutine Monitors_Construct( Monitors, mesh, controlVariables ) Monitors % no_of_loadBalancingMonitors = 0 else call getNoOfMonitors( Monitors % no_of_probes, Monitors % no_of_surfaceMonitors, Monitors % no_of_volumeMonitors, Monitors % no_of_loadBalancingMonitors ) +! +! Check for an additional probes definition file (one probe per +! line: "x y z variable1 [variable2 ...]"), allowing several +! variables to be sampled and saved for the same probe location +! --------------------------------------------------------------- +#ifdef FLOW + call readProbesFileBlock( probesFileName, probesVariablesLine, no_of_probesVariables, probeFileSaveTimestep, probeFileOutputFormat ) + + if ( len_trim(probesFileName) .gt. 0 ) then + call countProbesInFile( trim(probesFileName), no_of_fileProbes ) + + if ( len_trim(probesVariablesLine) .gt. 0 ) then + call splitIntoTokens( trim(probesVariablesLine), probesVariables, no_of_probesVariables ) + else + write(STD_OUT,*) "Error: 'variables' must be specified inside the '#define probe file' block." + stop + end if + end if +#endif end if ! ! Initialize the Monitors class in the GPU @@ -142,11 +196,31 @@ subroutine Monitors_Construct( Monitors, mesh, controlVariables ) end do #ifdef FLOW - allocate ( Monitors % probes ( Monitors % no_of_probes ) ) + allocate ( Monitors % probes ( Monitors % no_of_probes + no_of_fileProbes ) ) !$acc update device(Monitors) do i = 1 , Monitors % no_of_probes - call Monitors % probes(i) % Initialization ( mesh , i, solution_file , FirstCall ) + call Monitors % probes(i) % Initialization ( mesh , i, probes_solution_file , FirstCall ) end do + + if ( no_of_fileProbes .gt. 0 ) then + call InitializeProbesFromFile( trim(probesFileName), Monitors % probes, Monitors % no_of_probes, & + mesh, probes_solution_file, FirstCall, probesVariables, probeFileSaveTimestep, & + probeFileOutputFormat ) + Monitors % probesFileName = trim(probesFileName) + Monitors % probeFileSaveTimestep = probeFileSaveTimestep + Monitors % probeFileOutputFormat = probeFileOutputFormat + Monitors % fp_lastSavedTime = -huge(0.0_RP) + allocate( Monitors % probesVariables(size(probesVariables)) ) + Monitors % probesVariables = probesVariables +#ifdef HAS_HDF5 + if ( trim(probeFileOutputFormat) .eq. "HDF5" ) then + call Monitor_InitFileProbesHDF5( Monitors, no_of_fileProbes ) + end if +#endif + end if + + Monitors % no_of_fileProbes = no_of_fileProbes + Monitors % no_of_probes = Monitors % no_of_probes + no_of_fileProbes #endif #if defined(NAVIERSTOKES) || defined(INCNS) @@ -210,9 +284,9 @@ subroutine Monitor_WriteLabel ( self ) #ifdef FLOW ! -! Write probes labels -! ------------------- - do i = 1 , self % no_of_probes +! Write probes labels (file-based probes excluded for readability) +! --------------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes call self % probes(i) % WriteLabel end do #endif @@ -282,9 +356,9 @@ subroutine Monitor_WriteUnderlines( self ) #ifdef FLOW ! -! Print dashes for probes -! ----------------------- - do i = 1 , self % no_of_probes +! Print dashes for probes (file-based probes excluded for readability) +! ---------------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes if ( self % probes(i) % active ) then write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % probes(i) % monitorName ) + 2 ) ) end if @@ -349,9 +423,9 @@ subroutine Monitor_WriteValues ( self ) #ifdef FLOW ! -! Print probes -! ------------ - do i = 1 , self % no_of_probes +! Print probes (file-based probes excluded for readability) +! ---------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes call self % probes(i) % WriteValues ( self % bufferLine ) end do #endif @@ -433,11 +507,20 @@ subroutine Monitor_UpdateValues ( self, mesh, t , iter, maxResiduals, Autosave, #ifdef FLOW ! -! Update probes -! ------------- - do i = 1 , self % no_of_probes +! Update standard probes (GPU path with per-probe MPI) +! ----------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes call self % probes(i) % Update( mesh , self % bufferLine ) end do +! +! Update file probes: compute into slot 1, then flush immediately. +! File probes use a size-1 values buffer to avoid large memory +! allocations when the number of probes is O(1e6). +! ---------------------------------------------------------- + if ( self % no_of_fileProbes .gt. 0 ) then + call Monitor_UpdateFileProbes( self, mesh, 1 ) + call Monitor_FlushFileProbesNow( self, t, iter ) + end if #endif #if defined(NAVIERSTOKES) || defined(INCNS) @@ -500,12 +583,12 @@ subroutine Monitor_WriteToFile ( self , mesh, force) end do #ifdef FLOW - do i = 1 , self % no_of_probes + do i = 1 , self % no_of_probes - self % no_of_fileProbes call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) end do #endif - -#if defined(NAVIERSTOKES) || defined(INCNS) + +#if defined(NAVIERSTOKES) || defined(INCNS) do i = 1 , self % no_of_surfaceMonitors call self % surfaceMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) end do @@ -541,8 +624,8 @@ subroutine Monitor_WriteToFile ( self , mesh, force) end do #ifdef FLOW - do i = 1 , self % no_of_probes - call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + do i = 1 , self % no_of_probes - self % no_of_fileProbes + call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) end do #endif @@ -560,7 +643,47 @@ subroutine Monitor_WriteToFile ( self , mesh, force) end if end subroutine Monitor_WriteToFile - + + subroutine Monitor_WriteProbesFileSummary ( self ) +! +! ******************************************************************** +! Prints a startup-log summary of the bulk probes-file monitor, +! shown instead of including the file-based probes in the +! per-iteration screen log. +! ******************************************************************** +! + use Headers + use MPI_Process_Info + implicit none + class(Monitor_t) :: self +! +! --------------- +! Local variables +! --------------- +! + integer :: j + + if ( .not. MPI_Process % isRoot ) return + if ( self % no_of_fileProbes .le. 0 ) return + + write(STD_OUT,'(/)') + call Section_Header("Probes file monitor") + write(STD_OUT,'(/)') + + write(STD_OUT,'(30X,A,A28,A)') "->" , "File: " , trim(self % probesFileName) + write(STD_OUT,'(30X,A,A28,I10)') "->" , "Number of probes: " , self % no_of_fileProbes + write(STD_OUT,'(30X,A,A28)',advance="no") "->" , "Variables: " + do j = 1 , size(self % probesVariables) + write(STD_OUT,'(A)',advance="no") trim(self % probesVariables(j)) // " " + end do + write(STD_OUT,*) + if ( self % probeFileSaveTimestep .gt. 0.0_RP ) then + write(STD_OUT,'(30X,A,A28,ES14.6)') "->" , "Save timestep: " , self % probeFileSaveTimestep + end if + write(STD_OUT,'(30X,A,A28,A)') "->" , "Output format: " , trim(self % probeFileOutputFormat) + + end subroutine Monitor_WriteProbesFileSummary + subroutine Monitor_Destruct (self) implicit none class(Monitor_t) :: self @@ -600,10 +723,21 @@ impure elemental subroutine Monitor_Assign ( to, from ) !------------------------------------------------ to % solution_file = from % solution_file + to % probes_solution_file = from % probes_solution_file to % no_of_probes = from % no_of_probes to % no_of_surfaceMonitors = from % no_of_surfaceMonitors to % no_of_volumeMonitors = from % no_of_volumeMonitors to % no_of_loadBalancingMonitors = from % no_of_loadBalancingMonitors + to % no_of_fileProbes = from % no_of_fileProbes + to % probesFileName = from % probesFileName + to % probeFileSaveTimestep = from % probeFileSaveTimestep + to % probeFileOutputFormat = from % probeFileOutputFormat + to % fp_lastSavedTime = from % fp_lastSavedTime + if ( allocated(from % probesVariables) ) then + safedeallocate ( to % probesVariables ) + allocate ( to % probesVariables ( size(from % probesVariables) ) ) + to % probesVariables = from % probesVariables + end if to % bufferLine = from % bufferLine safedeallocate ( to % iter ) @@ -716,7 +850,12 @@ subroutine getNoOfMonitors(no_of_probes, no_of_surfaceMonitors, no_of_volumeMoni ! --------- line = getSquashedLine( line ) - if ( index ( line , '#defineprobe' ) .gt. 0 ) then + if ( index ( line , '#defineprobefile' ) .gt. 0 ) then +! +! The probe-file block is not an individual probe definition +! ----------------------------------------------------------- + + elseif ( index ( line , '#defineprobe' ) .gt. 0 ) then no_of_probes = no_of_probes + 1 elseif ( index ( line , '#definesurfacemonitor' ) .gt. 0 ) then @@ -740,7 +879,625 @@ subroutine getNoOfMonitors(no_of_probes, no_of_surfaceMonitors, no_of_volumeMoni end subroutine getNoOfMonitors +! +!/////////////////////////////////////////////////////////////////////////////////// +! +! Probes-from-file auxiliary routines +! +! The probes file is a plain text file. Blank lines and lines starting +! with "#" are ignored. Every remaining line holds the coordinates of +! one probe: +! +! x y z +! +! The list of variables to sample (shared by every probe in the file) +! is given through the "probes file variables" control-file keyword. +! +! A separate output file ".probe_.probe" is created for +! each probe, with one column per variable, and one row written per +! saved time step. +! +!/////////////////////////////////////////////////////////////////////////////////// +! + subroutine readProbesFileBlock(fileName, variablesLine, no_of_probesVariables, saveTimestep, outputFormat) +! +! ****************************************************************** +! Reads the "#define probe file ... #end" block from the case +! file, if present: +! +! #define probe file +! file = Probe.dat +! variables = u +! probe save timestep = 1.0E-3 +! output format = HDF5 +! #end +! +! Note: this is a hand-rolled parser (rather than readValueInRegion) +! because readValueInRegion lower-cases every line it reads, which +! would corrupt a case-sensitive file path. +! ****************************************************************** +! + use ParamfileRegions, only: getSquashedLine + implicit none + character(len=LINE_LENGTH), intent(out) :: fileName + character(len=LINE_LENGTH), intent(out) :: variablesLine + integer, intent(out) :: no_of_probesVariables + real(kind=RP), intent(out) :: saveTimestep + character(len=8), intent(out) :: outputFormat +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: paramFile + character(len=LINE_LENGTH) :: line, squashed, valStr + integer :: fID, io, position + + logical :: inside + + fileName = "" + variablesLine = "" + no_of_probesVariables = 0 + saveTimestep = 0.0_RP + outputFormat = "ASCII" + inside = .false. + + call get_command_argument(1, paramFile) + + open ( newunit = fID , file = trim(paramFile) , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + squashed = getSquashedLine(line) + + if ( squashed .eq. getSquashedLine("#define probe file") ) then + inside = .true. + cycle + elseif ( squashed .eq. getSquashedLine("#end") ) then + inside = .false. + cycle + end if + + if ( .not. inside ) cycle +! +! Strip a trailing comment, keeping the value's original case +! ------------------------------------------------------------- + position = index(line , '!') + if ( position .gt. 0 ) line = line(1:position-1) + + position = max( index(line,'='), index(line,':') ) + if ( position .eq. 0 ) cycle + + if ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("file") ) then + fileName = adjustl( removeQuotes( line(position+1:) ) ) + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("variables") ) then + variablesLine = adjustl( removeQuotes( line(position+1:) ) ) + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("probe save timestep") ) then + valStr = adjustl( removeQuotes( line(position+1:) ) ) + read( valStr , * ) saveTimestep + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("output format") ) then + valStr = adjustl( removeQuotes( line(position+1:) ) ) + valStr = adjustl( getSquashedLine(valStr) ) + if ( index(valStr, "HDF5") .gt. 0 ) then + outputFormat = "HDF5" + else + outputFormat = "ASCII" + end if + end if + end do + + close(fID) + + end subroutine readProbesFileBlock + + function removeQuotes(str) result(res) + implicit none + character(len=*), intent(in) :: str + character(len=LINE_LENGTH) :: res + character(len=LINE_LENGTH) :: auxstr + integer :: i, j + + auxstr = trim(adjustl(str)) + res = "" + j = 0 + + do i = 1 , len_trim(auxstr) + if ( auxstr(i:i) .eq. '"' .or. auxstr(i:i) .eq. "'" ) cycle + j = j + 1 + res(j:j) = auxstr(i:i) + end do + + end function removeQuotes + + subroutine countProbesInFile(fileName, n) + implicit none + character(len=*), intent(in) :: fileName + integer, intent(out) :: n +! +! --------------- +! Local variables +! --------------- +! + integer :: fID, io + character(len=LINE_LENGTH) :: line + + n = 0 + open ( newunit = fID , file = fileName , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + line = adjustl(line) + if ( len_trim(line) .eq. 0 ) cycle + if ( line(1:1) .eq. '#' ) cycle + + n = n + 1 + end do + + close(fID) + + end subroutine countProbesInFile + + subroutine splitIntoTokens(line, tokens, n) + implicit none + character(len=*), intent(in) :: line + character(len=STR_LEN_MONITORS), allocatable, intent(out) :: tokens(:) + integer, intent(out) :: n +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: auxline + character(len=STR_LEN_MONITORS) :: buffer(64) + integer :: pos + + auxline = adjustl(line) + n = 0 + + do while ( len_trim(auxline) .gt. 0 ) + pos = index(trim(auxline), " ") + n = n + 1 + + if ( pos .eq. 0 ) then + buffer(n) = trim(auxline) + auxline = "" + else + buffer(n) = auxline(1:pos-1) + auxline = adjustl(auxline(pos+1:)) + end if + end do + + allocate( tokens(n) ) + tokens(1:n) = buffer(1:n) + + end subroutine splitIntoTokens + +#ifdef FLOW + subroutine Monitor_FlushFileProbesNow(self, t_now, iter_now) +! +! Writes the current (single-slot) file-probe values to disk, +! applying the probeFileSaveTimestep filter. Called every timestep +! from Monitor_UpdateValues after Monitor_UpdateFileProbes. +! + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + real(kind=RP), intent(in) :: t_now + integer, intent(in) :: iter_now +! +! --------------- +! Local variables +! --------------- +! + integer :: i + integer :: iter_arr(1) + real(kind=RP) :: t_arr(1) + + iter_arr(1) = iter_now + t_arr(1) = t_now + +#ifdef HAS_HDF5 + if ( trim(self % probeFileOutputFormat) .eq. "HDF5" ) then + call Monitor_WriteFileProbesHDF5( self, iter_arr, t_arr, 1 ) + else +#endif + !$omp parallel do schedule(dynamic,16) default(shared) + do i = self % no_of_probes - self % no_of_fileProbes + 1, self % no_of_probes + call self % probes(i) % WriteToFile( iter_arr, t_arr, 1 ) + end do + !$omp end parallel do +#ifdef HAS_HDF5 + end if +#endif + + end subroutine Monitor_FlushFileProbesNow + + subroutine Monitor_UpdateFileProbes(self, mesh, bufferPos) +! +! Evaluates all file-probes using OpenMP threads (CPU path, no OpenACC), +! then performs a single MPI_Allreduce(SUM) to share results across ranks. +! This replaces N_fileProbes individual MPI_Bcast calls with one collective. +! + use MPI_Process_Info +#ifdef _HAS_MPI_ + use mpi +#endif + implicit none + class(Monitor_t), intent(inout) :: self + class(HexMesh), intent(in) :: mesh + integer, intent(in) :: bufferPos +! +! --------------- +! Local variables +! --------------- +! + integer :: i, v, j, nfp, nv, fp_offset, ierr + real(kind=RP), allocatable :: buf(:) + + nfp = self % no_of_fileProbes + nv = size(self % probesVariables) + fp_offset = self % no_of_probes - nfp + + ! Parallel CPU computation — each probe writes to its own values(:,bufferPos). + ! Non-owning ranks store 0 so MPI_Allreduce(SUM) gives the correct result. + !$omp parallel do schedule(dynamic,16) default(shared) + do i = fp_offset + 1, self % no_of_probes + call self % probes(i) % ComputeLocal(mesh, bufferPos) + end do + !$omp end parallel do + +#ifdef _HAS_MPI_ + if ( MPI_Process % doMPIAction ) then + ! Pack all file-probe values into a flat buffer + allocate( buf(nfp * nv) ) + j = 0 + do i = fp_offset + 1, self % no_of_probes + do v = 1, nv + j = j + 1 + buf(j) = self % probes(i) % values(v, bufferPos) + end do + end do + + ! Single collective instead of nfp individual Bcasts + call MPI_Allreduce(MPI_IN_PLACE, buf, nfp * nv, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_WORLD, ierr) + + ! Unpack + j = 0 + do i = fp_offset + 1, self % no_of_probes + do v = 1, nv + j = j + 1 + self % probes(i) % values(v, bufferPos) = buf(j) + end do + end do + deallocate(buf) + end if +#endif + + end subroutine Monitor_UpdateFileProbes + + subroutine InitializeProbesFromFile(fileName, probes, offset, mesh, solution_file, FirstCall, variables, saveTimestep, outputFormat) + implicit none + character(len=*), intent(in) :: fileName + class(Probe_t), intent(inout) :: probes(:) + integer, intent(in) :: offset + class(HexMesh), intent(in) :: mesh + character(len=*), intent(in) :: solution_file + logical, intent(in) :: FirstCall + character(len=*), intent(in) :: variables(:) + real(kind=RP), intent(in) :: saveTimestep + character(len=*), intent(in) :: outputFormat +! +! --------------- +! Local variables +! --------------- +! + integer :: fID, io, idx, nTok + character(len=LINE_LENGTH) :: line + real(kind=RP) :: x(NDIM) + character(len=STR_LEN_MONITORS), allocatable :: tokens(:) + character(len=STR_LEN_MONITORS) :: pname + + idx = offset + open ( newunit = fID , file = fileName , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + line = adjustl(line) + if ( len_trim(line) .eq. 0 ) cycle + if ( line(1:1) .eq. '#' ) cycle + + call splitIntoTokens(line, tokens, nTok) + + if ( nTok .lt. 3 ) then + deallocate(tokens) + cycle + end if + + read(tokens(1),*) x(1) + read(tokens(2),*) x(2) + read(tokens(3),*) x(3) + + idx = idx + 1 + write(pname,'(A,I0)') "probe_", idx + + call probes(idx) % Initialization( mesh, idx, solution_file, FirstCall, & + x_in = x, variables_in = variables, name_in = trim(pname), & + isFileProbe_in = .true., outputFormat_in = trim(outputFormat) ) + probes(idx) % saveTimestep = saveTimestep + + deallocate(tokens) + end do + + close(fID) + + end subroutine InitializeProbesFromFile + +! ============================================================ +! HDF5 collective I/O for file-based probes +! Compiled only when HAS_HDF5 is defined. +! ============================================================ + +#ifdef HAS_HDF5 + subroutine Monitor_InitFileProbesHDF5(self, no_of_fileProbes) +! +! Creates the HDF5 output file for bulk file-probes with +! the following structure: +! +! /coordinates (3, nProbes) — fixed, written once +! /time (extendible) — one value per saved step +! /iteration (extendible) — one value per saved step +! / (nProbes, ext) — one row per saved step +! + use HDF5 + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + integer, intent(in) :: no_of_fileProbes +! +! --------------- +! Local variables +! --------------- +! + integer(HID_T) :: file_id, dset_id, dspace_id, dcpl_id + integer(HSIZE_T) :: dims2(2), maxdims2(2), chunk2(2) + integer(HSIZE_T) :: dims1(1), maxdims1(1), chunk1(1) + integer :: iError, j, v, fp_offset, nv + character(len=LINE_LENGTH) :: fname + real(kind=RP), allocatable :: coords(:,:) + + if ( .not. MPI_Process % isRoot ) return + + nv = size(self % probesVariables) + fp_offset = self % no_of_probes ! probes are allocated 1..no_of_probes+no_of_fileProbes + ! but at this point no_of_probes has not yet been bumped + + write(fname,'(A,A)') trim(self % probes_solution_file), ".probes.h5" + + call h5open_f(iError) + call h5fcreate_f(trim(fname), H5F_ACC_TRUNC_F, file_id, iError) + + ! /coordinates (3, nProbes) — fixed at creation + allocate( coords(3, no_of_fileProbes) ) + do j = 1, no_of_fileProbes + coords(:, j) = self % probes(fp_offset + j) % x + end do + dims2 = [ int(3, HSIZE_T), int(no_of_fileProbes, HSIZE_T) ] + call h5screate_simple_f(2, dims2, dspace_id, iError) + call h5dcreate_f(file_id, "coordinates", H5T_NATIVE_DOUBLE, dspace_id, dset_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, coords, dims2, iError) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + deallocate(coords) + + ! /time (extendible 1-D) + dims1(1) = 0 + maxdims1(1) = H5S_UNLIMITED_F + chunk1(1) = int(max(BUFFER_SIZE, 1), HSIZE_T) + call h5screate_simple_f(1, dims1, dspace_id, iError, maxdims1) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 1, chunk1, iError) + call h5dcreate_f(file_id, "time", H5T_NATIVE_DOUBLE, dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + + ! /iteration (extendible 1-D) + call h5screate_simple_f(1, dims1, dspace_id, iError, maxdims1) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 1, chunk1, iError) + call h5dcreate_f(file_id, "iteration", H5T_NATIVE_INTEGER, dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + + ! / (extendible × nProbes) — h5ls shows {nProbes, Inf} + ! chunk2(1)=1: one time step per chunk avoids pre-allocation waste when + ! the save-timestep filter makes n_write << BUFFER_SIZE per flush. + dims2(1) = 0 + dims2(2) = int(no_of_fileProbes, HSIZE_T) + maxdims2(1) = H5S_UNLIMITED_F + maxdims2(2) = int(no_of_fileProbes, HSIZE_T) + chunk2(1) = int(1, HSIZE_T) + chunk2(2) = int(no_of_fileProbes, HSIZE_T) + + do v = 1, nv + call h5screate_simple_f(2, dims2, dspace_id, iError, maxdims2) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 2, chunk2, iError) + call h5dcreate_f(file_id, trim(self % probesVariables(v)), H5T_NATIVE_DOUBLE, & + dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + end do + + call h5fclose_f(file_id, iError) + call h5close_f(iError) + + end subroutine Monitor_InitFileProbesHDF5 + + subroutine Monitor_WriteFileProbesHDF5(self, iter, t, no_of_lines) +! +! Appends one buffer of file-probe data to the HDF5 output file. +! Only lines that satisfy the saveTimestep filter are written. +! Called from Monitor_WriteToFile in place of the per-probe ASCII loop. +! + use HDF5 + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + integer, intent(in) :: iter(:) + real(kind=RP), intent(in) :: t(:) + integer, intent(in) :: no_of_lines +! +! --------------- +! Local variables +! --------------- +! + integer(HID_T) :: file_id, dset_id, dspace_id, mspace_id + integer(HSIZE_T) :: cur1(1), max1(1), off1(1), cnt1(1) + integer(HSIZE_T) :: new1(1) + integer(HSIZE_T) :: cur2(2), max2(2), off2(2), cnt2(2) + integer(HSIZE_T) :: new2(2) + integer :: iError, i, j, k, v, nfp, nv, fp_offset, n_write + character(len=LINE_LENGTH) :: fname + logical, allocatable :: wmask(:) + integer, allocatable :: ibuf(:) + real(kind=RP), allocatable :: rbuf(:), vbuf(:) + + if ( .not. MPI_Process % isRoot ) return + + nfp = self % no_of_fileProbes + nv = size(self % probesVariables) + fp_offset = self % no_of_probes - nfp + + ! Build write mask (apply saveTimestep filter) + allocate( wmask(no_of_lines) ) + wmask = .false. + do i = 1, no_of_lines + if ( self % probeFileSaveTimestep .gt. 0.0_RP ) then + if ( t(i) .lt. self % fp_lastSavedTime + self % probeFileSaveTimestep ) cycle + end if + wmask(i) = .true. + self % fp_lastSavedTime = t(i) + end do + n_write = count(wmask) + if ( n_write .eq. 0 ) then + deallocate(wmask) + return + end if + + write(fname,'(A,A)') trim(self % probes_solution_file), ".probes.h5" + + call h5open_f(iError) + call h5fopen_f(trim(fname), H5F_ACC_RDWR_F, file_id, iError) + + ! Query current extent of /time to get append offset + call h5dopen_f(file_id, "time", dset_id, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sget_simple_extent_dims_f(dspace_id, cur1, max1, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + ! cur1(1) = number of time steps already written + + cnt1(1) = int(n_write, HSIZE_T) + off1(1) = cur1(1) + new1(1) = cur1(1) + cnt1(1) + + ! Pack filtered iteration values + allocate( ibuf(n_write) ) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + ibuf(j) = iter(i) + end do + + call h5dopen_f(file_id, "iteration", dset_id, iError) + call h5dextend_f(dset_id, new1, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off1, cnt1, iError) + call h5screate_simple_f(1, cnt1, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_INTEGER, ibuf, cnt1, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + deallocate(ibuf) + + ! Pack filtered time values + allocate( rbuf(n_write) ) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + rbuf(j) = t(i) + end do + + call h5dopen_f(file_id, "time", dset_id, iError) + call h5dextend_f(dset_id, new1, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off1, cnt1, iError) + call h5screate_simple_f(1, cnt1, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, rbuf, cnt1, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + deallocate(rbuf) + + ! Write each variable: one extend + one write per flush (avoids chunk waste). + ! vbuf2(nfp, n_write) packs all filtered timesteps into a contiguous 2D block. + off2(1) = cur1(1) ! append after existing time steps + off2(2) = int(0, HSIZE_T) + cnt2(1) = int(n_write, HSIZE_T) + cnt2(2) = int(nfp, HSIZE_T) + new2(1) = cur1(1) + int(n_write, HSIZE_T) + new2(2) = int(nfp, HSIZE_T) + + allocate( vbuf(nfp * n_write) ) + + do v = 1, nv + call h5dopen_f(file_id, trim(self % probesVariables(v)), dset_id, iError) + + ! Pack into column-major 2D layout vbuf(n_write, nfp): + ! time index j varies fastest (Fortran column-major with cnt2=[n_write,nfp]) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + do k = 1, nfp + vbuf( j + (k-1)*n_write ) = self % probes(fp_offset + k) % values(v, i) + end do + end do + + call h5dextend_f(dset_id, new2, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off2, cnt2, iError) + call h5screate_simple_f(2, cnt2, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, vbuf, cnt2, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + + call h5dclose_f(dset_id, iError) + end do + + deallocate(vbuf) + deallocate(wmask) + + call h5fclose_f(file_id, iError) + call h5close_f(iError) + + end subroutine Monitor_WriteFileProbesHDF5 +#endif ! HAS_HDF5 + +#endif ! FLOW + end module MonitorsClass ! !/////////////////////////////////////////////////////////////////////////////////// -! \ No newline at end of file +! diff --git a/Solver/src/libs/monitors/Probe.f90 b/Solver/src/libs/monitors/Probe.f90 index 5e193c457..e91279dbc 100644 --- a/Solver/src/libs/monitors/Probe.f90 +++ b/Solver/src/libs/monitors/Probe.f90 @@ -24,20 +24,25 @@ module ProbeClass ! type Probe_t logical :: active + logical :: isFileProbe integer :: rank integer :: ID integer :: eID + integer :: nVars real(kind=RP) :: x(NDIM) real(kind=RP) :: xi(NDIM) - real(kind=RP), allocatable :: values(:) + real(kind=RP), allocatable :: values(:,:) real(kind=RP), allocatable :: lxi(:) , leta(:), lzeta(:) real(kind=RP), allocatable :: var(:,:,:) + real(kind=RP) :: saveTimestep + real(kind=RP) :: lastSavedTime character(len=STR_LEN_MONITORS) :: fileName character(len=STR_LEN_MONITORS) :: monitorName - character(len=STR_LEN_MONITORS) :: variable + character(len=STR_LEN_MONITORS), allocatable :: variableNames(:) contains procedure :: Initialization => Probe_Initialization procedure :: Update => Probe_Update + procedure :: ComputeLocal => Probe_ComputeLocal procedure :: WriteLabel => Probe_WriteLabel procedure :: WriteValues => Probe_WriteValue procedure :: WriteToFile => Probe_WriteToFile @@ -49,7 +54,7 @@ module ProbeClass contains - subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) + subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall, x_in, variables_in, name_in, isFileProbe_in, outputFormat_in) use ParamfileRegions use MPI_Process_Info use Utilities, only: toLower @@ -59,45 +64,82 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) integer :: ID character(len=*) :: solution_file logical, intent(in) :: FirstCall + real(kind=RP), intent(in), optional :: x_in(NDIM) + character(len=*), intent(in), optional :: variables_in(:) + character(len=*), intent(in), optional :: name_in + logical, intent(in), optional :: isFileProbe_in + character(len=*), intent(in), optional :: outputFormat_in ! ! --------------- ! Local variables ! --------------- ! - integer :: i, j, k, fid + integer :: i, j, k, v, fid character(len=STR_LEN_MONITORS) :: in_label character(len=STR_LEN_MONITORS) :: fileName character(len=STR_LEN_MONITORS) :: paramFile character(len=STR_LEN_MONITORS) :: coordinates - + character(len=STR_LEN_MONITORS) :: variable + character(len=STR_LEN_MONITORS) :: outputFormat + + self % isFileProbe = .false. + if ( present(isFileProbe_in) ) self % isFileProbe = isFileProbe_in + + outputFormat = "ASCII" + if ( present(outputFormat_in) ) outputFormat = trim(outputFormat_in) + if (FirstCall) then ! -! Allocate memory -! --------------- - allocate ( self % values(BUFFER_SIZE) ) -! ! Get monitor ID ! -------------- self % ID = ID ! -! Search for the parameters in the case file -! ------------------------------------------ - write(in_label , '(A,I0)') "#define probe " , self % ID - - call get_command_argument(1, paramFile) - call readValueInRegion(trim(paramFile), "name" , self % monitorName, in_label, "# end" ) - call readValueInRegion(trim(paramFile), "variable", self % variable , in_label, "# end" ) - call readValueInRegion(trim(paramFile), "position", coordinates , in_label, "# end" ) +! Get the probe definition, either from a probes file (x_in/variables_in +! provided) or from a "#define probe" block in the case file +! ------------------------------------------------------------------------ + if ( present(x_in) ) then + self % monitorName = name_in + self % x = x_in + self % nVars = size(variables_in) + allocate( self % variableNames(self % nVars) ) + self % variableNames = variables_in + + else +! +! Search for the parameters in the case file +! ------------------------------------------ + write(in_label , '(A,I0)') "#define probe " , self % ID + + call get_command_argument(1, paramFile) + call readValueInRegion(trim(paramFile), "name" , self % monitorName, in_label, "# end" ) + call readValueInRegion(trim(paramFile), "variable", variable , in_label, "# end" ) + call readValueInRegion(trim(paramFile), "position", coordinates , in_label, "# end" ) ! -! Get the coordinates -! ------------------- - self % x = getRealArrayFromString(coordinates) +! Get the coordinates +! ------------------- + self % x = getRealArrayFromString(coordinates) + + self % nVars = 1 + allocate( self % variableNames(1) ) + self % variableNames(1) = variable + end if +! +! Allocate memory +! --------------- + if (self % isFileProbe) then + allocate ( self % values(self % nVars, 1) ) + else + allocate ( self % values(self % nVars, BUFFER_SIZE) ) + end if + self % saveTimestep = 0.0_RP + self % lastSavedTime = -huge(self % lastSavedTime) ! -! Check the variable -! ------------------ - call tolower(self % variable) +! Check the variables +! -------------------- + do v = 1, self % nVars + call tolower(self % variableNames(v)) - select case ( trim(self % variable) ) + select case ( trim(self % variableNames(v)) ) #ifdef NAVIERSTOKES case ("pressure") case ("velocity") @@ -106,8 +148,9 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) case ("w") case ("mach") case ("k") + case ("rho") case default - print*, 'Probe variable "',trim(self % variable),'" not implemented.' + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' print*, "Options available are:" print*, " * pressure" print*, " * velocity" @@ -116,6 +159,7 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) print*, " * w" print*, " * Mach" print*, " * K" + print*, " * rho" end select #endif #ifdef INCNS @@ -124,21 +168,23 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) case ("u") case ("v") case ("w") + case ("rho") case default - print*, 'Probe variable "',trim(self % variable),'" not implemented.' + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' print*, "Options available are:" print*, " * pressure" print*, " * velocity" print*, " * u" print*, " * v" print*, " * w" + print*, " * rho" end select #endif #ifdef MULTIPHASE case ("static-pressure") case default - print*, 'Probe variable "',trim(self % variable),'" not implemented.' + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' print*, "Options available are:" print*, " * static-pressure" @@ -151,7 +197,7 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) case ("v") case ("w") case default - print*, 'Probe variable "',trim(self % variable),'" not implemented.' + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' print*, "Options available are:" print*, " * pressure" print*, " * velocity" @@ -160,7 +206,8 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) print*, " * w" end select #endif - + end do + ! ! Find the requested point in the mesh ! ------------------------------------ @@ -218,33 +265,36 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) safedeallocate(self % var ) ; allocate( self % var(0 : e % Nxyz(1),0 : e % Nxyz(2),0 : e % Nxyz(3)) ) self % var = 0.0_RP - !$acc enter data copyin(self) - !$acc enter data copyin(self % eiD) - !$acc enter data copyin(self % id) - !$acc enter data copyin(self % var) - !$acc enter data copyin(self % lxi) - !$acc enter data copyin(self % leta) - !$acc enter data copyin(self % lzeta) + if ( .not. self % isFileProbe ) then + !$acc enter data copyin(self) + !$acc enter data copyin(self % eiD) + !$acc enter data copyin(self % id) + !$acc enter data copyin(self % var) + !$acc enter data copyin(self % lxi) + !$acc enter data copyin(self % leta) + !$acc enter data copyin(self % lzeta) + end if ! ! **************** ! Prepare the file ! **************** ! -! Create file -! ----------- - if (FirstCall) then - open ( newunit = fID , file = trim(self % fileName) , status = "unknown" , action = "write" ) +! Create file (skip for file-probes using HDF5 output) +! ----------------------------------------------------- + if (FirstCall .and. .not. (self % isFileProbe .and. trim(outputFormat) .eq. "HDF5")) then + open ( newunit = fID , file = trim(self % fileName) , status = "unknown" , action = "write" ) ! ! Write the file headers ! ---------------------- write( fID , '(A20,A )') "Monitor name: ", trim(self % monitorName) - write( fID , '(A20,A )') "Selected variable: " , trim(self % variable) - write( fID , '(A20,ES24.10)') "x coordinate: ", self % x(1) - write( fID , '(A20,ES24.10)') "y coordinate: ", self % x(2) - write( fID , '(A20,ES24.10)') "z coordinate: ", self % x(3) + write( fID , '(A25,ES24.10,2(4X,ES24.10))') "x, y, z coordinates: ", self % x(1), self % x(2), self % x(3) write( fID , * ) - write( fID , '(A10,2X,A24,2X,A24)' ) "Iteration" , "Time" , trim(self % variable) + write( fID , '(A10,2X,A24)' , advance = "no") "Iteration" , "Time" + do v = 1 , self % nVars + write( fID , '(2X,A24)' , advance = "no") trim(self % variableNames(v)) + end do + write( fID , * ) close ( fID ) end if @@ -264,59 +314,176 @@ subroutine Probe_Update(self, mesh, bufferPosition) ! Local variables ! --------------- ! - integer :: i, j, k, ierr + integer :: i, j, k, v, ierr real(kind=RP) :: value - if ( .not. self % active ) return + if ( .not. self % active ) return if ( MPI_Process % rank .eq. self % rank ) then ! ! Update the probe ! ---------------- - - select case (trim(self % variable)) + do v = 1, self % nVars + + if ( self % isFileProbe ) then +! +! CPU path for file-probes (no OpenACC to avoid runtime table overflow) +! ---------------------------------------------------------------------- + select case (trim(self % variableNames(v))) +#ifdef NAVIERSTOKES + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("mach") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) + self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& + (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) + end do ; end do ; end do + case("k") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef INCNS + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef MULTIPHASE + case("static-pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & + - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & + - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) + end do ; end do ; end do +#endif +#ifdef ACOUSTIC + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAP,i,j,k) + end do ; end do ; end do + case("density") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAARHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAU,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAV,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAW,i,j,k) + end do ; end do ; end do +#endif + end select + + value = 0.0_RP + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) + end do ; end do ; end do + + else +! +! GPU path for standard probes (OpenACC) +! --------------------------------------- + select case (trim(self % variableNames(v))) #ifdef NAVIERSTOKES case("pressure") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) end do ; end do ; end do !$acc end parallel loop - + case("velocity") !$acc parallel loop collapse(3) present(mesh, self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("u") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop case("v") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("w") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("mach") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) ! Vabs**2 @@ -324,93 +491,107 @@ subroutine Probe_Update(self, mesh, bufferPosition) (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) end do ; end do ; end do !$acc end parallel loop - + case("k") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop + + case("rho") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop #endif #ifdef INCNS case("pressure") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("velocity") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & POW2( mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("u") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("v") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("w") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop + + case("rho") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop #endif #ifdef MULTIPHASE case("static-pressure") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) end do ; end do ; end do !$acc end parallel loop -#endif +#endif #ifdef ACOUSTIC case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAAP,i,j,k) end do ; end do ; end do case("density") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAARHO,i,j,k) end do ; end do ; end do - + case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAAU,i,j,k) end do ; end do ; end do - + case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAAV,i,j,k) end do ; end do ; end do - + case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAAW,i,j,k) end do ; end do ; end do #endif end select - + value = 0.0_RP !$acc parallel loop collapse(3) present(mesh, self) reduction(+:value) async(self % ID) do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) @@ -420,14 +601,18 @@ subroutine Probe_Update(self, mesh, bufferPosition) !$acc wait - self % values(bufferPosition) = value - -#ifdef _HAS_MPI_ + end if ! isFileProbe + + self % values(v, bufferPosition) = value + + end do +! +#ifdef _HAS_MPI_ if ( MPI_Process % doMPIAction ) then ! ! Share the result with the rest of the processes -! ----------------------------------------------- - call mpi_bcast(value, 1, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) +! ----------------------------------------------- + call mpi_bcast(self % values(:,bufferPosition), self % nVars, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) end if #endif @@ -437,12 +622,160 @@ subroutine Probe_Update(self, mesh, bufferPosition) ! -------------------------------------------------------- #ifdef _HAS_MPI_ if ( MPI_Process % doMPIAction ) then - call mpi_bcast(self % values(bufferPosition), 1, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) + call mpi_bcast(self % values(:,bufferPosition), self % nVars, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) end if #endif end if end subroutine Probe_Update + subroutine Probe_ComputeLocal(self, mesh, bufferPosition) +! +! ************************************************************* +! CPU-only computation for file-probes, no MPI. +! Non-owning ranks set values to 0 so a caller can +! accumulate results with a single MPI_Allreduce(SUM). +! ************************************************************* +! + use Physics + use MPI_Process_Info + implicit none + class(Probe_t) :: self + type(HexMesh) :: mesh + integer :: bufferPosition +! +! --------------- +! Local variables +! --------------- +! + integer :: i, j, k, v + real(kind=RP) :: value + + if ( .not. self % active ) then + self % values(:, bufferPosition) = 0.0_RP + return + end if + + if ( MPI_Process % rank .ne. self % rank ) then + self % values(:, bufferPosition) = 0.0_RP + return + end if + + do v = 1, self % nVars + + select case (trim(self % variableNames(v))) +#ifdef NAVIERSTOKES + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("mach") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) + self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& + (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) + end do ; end do ; end do + case("k") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef INCNS + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef MULTIPHASE + case("static-pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & + - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & + - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) + end do ; end do ; end do +#endif +#ifdef ACOUSTIC + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAP,i,j,k) + end do ; end do ; end do + case("density") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAARHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAU,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAV,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAW,i,j,k) + end do ; end do ; end do +#endif + end select + + value = 0.0_RP + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) + end do ; end do ; end do + + self % values(v, bufferPosition) = value + + end do + + end subroutine Probe_ComputeLocal + subroutine Probe_WriteLabel ( self ) ! ! ************************************************************* @@ -469,9 +802,9 @@ subroutine Probe_WriteValue ( self , bufferLine ) class(Probe_t) :: self integer :: bufferLine - write(STD_OUT , '(1X,A,1X,ES10.3)' , advance = "no") "|" , self % values ( bufferLine ) + write(STD_OUT , '(1X,A,1X,ES10.3)' , advance = "no") "|" , self % values ( 1 , bufferLine ) - end subroutine Probe_WriteValue + end subroutine Probe_WriteValue subroutine Probe_WriteToFile ( self , iter , t , no_of_lines) ! @@ -489,22 +822,34 @@ subroutine Probe_WriteToFile ( self , iter , t , no_of_lines) ! Local variables ! --------------- ! - integer :: i + integer :: i, v integer :: fID + if ( .not. self % active ) then + if ( no_of_lines .ne. 0 ) self % values(:,1) = self % values(:,no_of_lines) + return + end if + if ( MPI_Process % isRoot ) then open( newunit = fID , file = trim ( self % fileName ) , action = "write" , access = "append" , status = "old" ) - - do i = 1 , no_of_lines - write( fID , '(I10,2X,ES24.16,2X,ES24.16)' ) iter(i) , t(i) , self % values(i) + do i = 1 , no_of_lines + if ( self % saveTimestep > 0.0_RP ) then + if ( t(i) < self % lastSavedTime + self % saveTimestep ) cycle + end if + write( fID , '(I10,2X,ES24.16)' , advance = "no" ) iter(i) , t(i) + do v = 1 , self % nVars + write( fID , '(2X,ES24.16)' , advance = "no" ) self % values(v,i) + end do + write( fID , * ) + self % lastSavedTime = t(i) end do - + close ( fID ) end if - - if ( no_of_lines .ne. 0 ) self % values(1) = self % values(no_of_lines) + + if ( no_of_lines .ne. 0 ) self % values(:,1) = self % values(:,no_of_lines) end subroutine Probe_WriteToFile @@ -566,6 +911,7 @@ elemental subroutine Probe_Destruct (self) safedeallocate (self % lxi) safedeallocate (self % leta) safedeallocate (self % lzeta) + safedeallocate (self % variableNames) end subroutine Probe_Destruct elemental subroutine Probe_Assign (to, from) @@ -574,32 +920,47 @@ elemental subroutine Probe_Assign (to, from) type(Probe_t) , intent(in) :: from to % active = from % active + to % isFileProbe = from % isFileProbe to % rank = from % rank to % ID = from % ID - to % eID = from % eID + to % eID = from % eID + to % nVars = from % nVars to % x = from % x to % xi = from % xi + + if ( allocated(from % values) ) then + safedeallocate ( to % values ) + allocate ( to % values ( size(from % values, 1), size(from % values, 2) ) ) + to % values = from % values + end if + + if ( allocated(from % lxi) ) then + safedeallocate ( to % lxi ) + allocate ( to % lxi ( size(from % lxi) ) ) + to % lxi = from % lxi + end if + + if ( allocated(from % leta) ) then + safedeallocate ( to % leta ) + allocate ( to % leta ( size(from % leta) ) ) + to % leta = from % leta + end if + + if ( allocated(from % lzeta) ) then + safedeallocate ( to % lzeta ) + allocate ( to % lzeta ( size(from % lzeta) ) ) + to % lzeta = from % lzeta + end if - safedeallocate ( to % values ) - allocate ( to % values ( size(from % values) ) ) - to % values = from % values - - safedeallocate ( to % lxi ) - allocate ( to % lxi ( size(from % lxi) ) ) - to % lxi = from % lxi - - safedeallocate ( to % leta ) - allocate ( to % leta ( size(from % leta) ) ) - to % leta = from % leta - - safedeallocate ( to % lzeta ) - allocate ( to % lzeta ( size(from % lzeta) ) ) - to % lzeta = from % lzeta - + to % saveTimestep = from % saveTimestep + to % lastSavedTime = from % lastSavedTime to % fileName = from % fileName to % monitorName = from % monitorName - to % variable = from % variable - + + safedeallocate ( to % variableNames ) + allocate ( to % variableNames ( size(from % variableNames) ) ) + to % variableNames = from % variableNames + end subroutine Probe_Assign end module ProbeClass diff --git a/Solver/src/libs/sources/ActuatorLine.f90 b/Solver/src/libs/sources/ActuatorLine.f90 index ed549004c..baf7149a7 100644 --- a/Solver/src/libs/sources/ActuatorLine.f90 +++ b/Solver/src/libs/sources/ActuatorLine.f90 @@ -711,7 +711,7 @@ subroutine UpdateFarm(self,time, mesh) ! pointsToFind = 0 newPointToFind = 0 -!$omp do schedule(runtime)private(ii,jj,kk,eID,Q,Qtemp,delta_temp,xi,found,allfound) +!$omp do schedule(runtime)private(ii,jj,kk,eID,delta_temp,xi,found,allfound) do kk = 1, self%num_turbines do jj = 1, self%turbine_t(kk)%num_blades @@ -751,7 +751,7 @@ subroutine UpdateFarm(self,time, mesh) end do enddo enddo - +!$omp end do ! update MPI partitions and look for points that have changed ! ----------------------------------------------------------- if ( (MPI_Process % doMPIAction) ) then @@ -832,7 +832,7 @@ subroutine UpdateFarm(self,time, mesh) enddo enddo -!$omp end do + ! send local forces arrays and angle to device do kk=1, self % num_turbines do jj=1, self % turbine_t(kk) % num_blades diff --git a/Solver/src/libs/timeintegrator/AnalyticalJacobian.f90 b/Solver/src/libs/timeintegrator/AnalyticalJacobian.f90 index 2a27395ce..6ba5cbdb0 100644 --- a/Solver/src/libs/timeintegrator/AnalyticalJacobian.f90 +++ b/Solver/src/libs/timeintegrator/AnalyticalJacobian.f90 @@ -230,6 +230,7 @@ subroutine AnJacobian_Compute(this, sem, nEqn, time, Matrix, TimeDerivative, Tim ! ************************************************************************ ! call ComputeNumericalFluxJacobian(sem % mesh,nEqn,time) +!$omp end parallel ! ! *************** ! Diagonal blocks @@ -242,7 +243,6 @@ subroutine AnJacobian_Compute(this, sem, nEqn, time, Matrix, TimeDerivative, Tim ! ******************* ! if (.not. BlockDiagonal) call AnalyticalJacobian_OffDiagonalBlocks(sem % mesh,Matrix) -!$omp end parallel ! ! ************************ ! Finish assembling matrix @@ -288,34 +288,33 @@ subroutine AnalyticalJacobian_DiagonalBlocks(mesh,nEqn,time,Matrix) class(Matrix_t) , intent(inout) :: Matrix !-------------------------------------------- integer :: eID, fID - type(Element), pointer :: e !-------------------------------------------- ! ! Project flux Jacobian to corresponding element ! ---------------------------------------------- -!$omp do schedule(runtime) +!$omp parallel do schedule(runtime) do fID = 1, size(mesh % faces) - call mesh % faces(fID) % ProjectFluxJacobianToElements(nEqn,LEFT ,LEFT ) ! dF/dQL to the left element + call mesh % faces(fID) % ProjectFluxJacobianToElements(nEqn,LEFT ,LEFT ) ! dF/dQL to the left element if (.not. (mesh % faces(fID) % faceType == HMESH_BOUNDARY)) call mesh % faces(fID) % ProjectFluxJacobianToElements(nEqn,RIGHT,RIGHT) ! dF/dQR to the right element end do -!$omp end do +!$omp end parallel do ! ! Project flux Jacobian with respect to gradients to corresponding elements ! ------------------------------------------------------------------------- if (flowIsNavierStokes) then -!$omp do schedule(runtime) +!$omp parallel do schedule(runtime) do fID = 1, size(mesh % faces) - call mesh % faces(fID) % ProjectGradJacobianToElements(LEFT, LEFT) ! dF/dQL to the left element + call mesh % faces(fID) % ProjectGradJacobianToElements(LEFT, LEFT) ! dF/dQL to the left element if (.not. (mesh % faces(fID) % faceType == HMESH_BOUNDARY)) call mesh % faces(fID) % ProjectGradJacobianToElements(RIGHT,RIGHT) ! dF/dQR to the right element end do -!$omp end do +!$omp end parallel do end if ! ! Compute each element's diagonal block ! ------------------------------------- -!$omp do schedule(runtime) private(e) +!$omp parallel do schedule(runtime) do eID = 1, size(mesh % elements) - e => mesh % elements(eID) + associate(e => mesh % elements(eID)) call Local_SetDiagonalBlock( e, & mesh % faces( e % faceIDs(EFRONT ) ), & mesh % faces( e % faceIDs(EBACK ) ), & @@ -325,9 +324,9 @@ subroutine AnalyticalJacobian_DiagonalBlocks(mesh,nEqn,time,Matrix) mesh % faces( e % faceIDs(ELEFT ) ), & Matrix, & mesh% IBM ) + end associate end do -!$omp end do - nullify (e) +!$omp end parallel do end subroutine AnalyticalJacobian_DiagonalBlocks ! !/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -344,58 +343,56 @@ subroutine AnalyticalJacobian_OffDiagonalBlocks(mesh,Matrix) class(Matrix_t) , intent(inout) :: Matrix !-------------------------------------------- integer :: eID, fID, elSide, side - type(Element), pointer :: e_plus - type(Face) , pointer :: f !-------------------------------------------- - + ! ! Project flux Jacobian to opposed elements (RIGHT to LEFT and LEFT to RIGHT) ! --------------------------------------------------------------------------- -!$omp do schedule(runtime) +!$omp parallel do schedule(runtime) do fID = 1, size(mesh % faces) if (mesh % faces(fID) % faceType /= HMESH_BOUNDARY) then call mesh % faces(fID) % ProjectFluxJacobianToElements(NCONS, LEFT ,RIGHT) ! dF/dQR to the left element - call mesh % faces(fID) % ProjectFluxJacobianToElements(NCONS, RIGHT,LEFT ) ! dF/dQL to the right element + call mesh % faces(fID) % ProjectFluxJacobianToElements(NCONS, RIGHT,LEFT ) ! dF/dQL to the right element end if end do -!$omp end do +!$omp end parallel do ! ! Project flux Jacobian with respect to gradients to opposed elements (RIGHT to LEFT and LEFT to RIGHT) ! ----------------------------------------------------------------------------------------------------- if (flowIsNavierStokes) then -!$omp do schedule(runtime) +!$omp parallel do schedule(runtime) do fID = 1, size(mesh % faces) if (mesh % faces(fID) % faceType /= HMESH_BOUNDARY) then call mesh % faces(fID) % ProjectGradJacobianToElements(LEFT ,RIGHT) ! dF/dGradQR to the left element - call mesh % faces(fID) % ProjectGradJacobianToElements(RIGHT,LEFT ) ! dF/dGradQL to the right element + call mesh % faces(fID) % ProjectGradJacobianToElements(RIGHT,LEFT ) ! dF/dGradQL to the right element end if end do -!$omp end do +!$omp end parallel do end if - + ! ! Compute the off-diagonal blocks for each element's equations ! ------------------------------------------------------------ -!$omp do schedule(runtime) private(e_plus,elSide,fID,side,f) +!$omp parallel do schedule(runtime) private(elSide,fID,side) do eID = 1, mesh % no_of_elements - e_plus => mesh % elements(eID) + associate(e_plus => mesh % elements(eID)) ! ! One block for every neighbor element ! ------------------------------------ do elSide = 1, 6 if (e_plus % NumberOfConnections(elSide) == 0) cycle - + fID = e_plus % faceIDs(elSide) side = e_plus % faceSide(elSide) - - f => mesh % faces(fID) - + + associate(f => mesh % faces(fID)) call Local_GetOffDiagonalBlock(f,e_plus,e_plus % Connection(elSide),side,Matrix) + end associate end do + end associate end do -!$omp end do - nullify (f, e_plus) +!$omp end parallel do end subroutine AnalyticalJacobian_OffDiagonalBlocks ! !/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/basicManualAndTutorial.pdf b/doc/basicManualAndTutorial.pdf index 820d9c36f..017493a33 100644 Binary files a/doc/basicManualAndTutorial.pdf and b/doc/basicManualAndTutorial.pdf differ