From 7c783f8e2ea2d3978c253f6837339dcc4c4ab0b3 Mon Sep 17 00:00:00 2001 From: Miguel CM Date: Wed, 8 Jul 2026 10:09:15 +0200 Subject: [PATCH 1/9] Add file-based probes, horses2probes, and GPU compatibility fixes --- .../SpatialDiscretization.f90 | 314 ++++--- Solver/src/addons/probes2tecplot/Makefile | 109 +++ .../addons/probes2tecplot/Probes2tecplot.f90 | 813 ++++++++++++++++++ Solver/src/addons/probes2tecplot/main.f90 | 267 ++++++ Solver/src/libs/discretization/DGSEMClass.f90 | 16 +- .../src/libs/discretization/EllipticBR1.f90 | 36 +- .../src/libs/discretization/EllipticBR2.f90 | 44 +- Solver/src/libs/discretization/EllipticIP.f90 | 46 +- .../src/libs/discretization/SCsensorClass.f90 | 46 +- Solver/src/libs/mesh/HexElementClass.f90 | 95 +- Solver/src/libs/mesh/HexMesh.f90 | 188 ++-- Solver/src/libs/mesh/IBMClass.f90 | 34 +- Solver/src/libs/mesh/StorageClass.f90 | 2 +- Solver/src/libs/monitors/Monitors.f90 | 805 ++++++++++++++++- Solver/src/libs/monitors/Probe.f90 | 591 ++++++++++--- Solver/src/libs/monitors/SurfaceIntegrals.f90 | 125 ++- Solver/src/libs/monitors/VolumeIntegrals.f90 | 25 +- Solver/src/libs/sources/ActuatorLine.f90 | 5 +- .../timeintegrator/AnalyticalJacobian.f90 | 57 +- 19 files changed, 3154 insertions(+), 464 deletions(-) create mode 100644 Solver/src/addons/probes2tecplot/Makefile create mode 100644 Solver/src/addons/probes2tecplot/Probes2tecplot.f90 create mode 100644 Solver/src/addons/probes2tecplot/main.f90 diff --git a/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 b/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 index f9ec0cd98..f076f5acb 100644 --- a/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 +++ b/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 @@ -258,7 +258,9 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) ! Prolongation of the solution to the faces ! ----------------------------------------- ! +#ifndef _OPENACC !$omp parallel shared(mesh, time) +#endif call HexMesh_ProlongSolToFaces(mesh, NCONS) ! ---------------- ! Update MPI Faces @@ -310,7 +312,9 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) t = time) end if +#ifndef _OPENACC !$omp end parallel +#endif ! END SUBROUTINE ComputeTimeDerivative ! @@ -343,7 +347,9 @@ SUBROUTINE ComputeTimeDerivativeIsolated( mesh, particles, time, mode, HO_Elemen ! Prolongation of the solution to the faces ! ----------------------------------------- ! +#ifndef _OPENACC !$omp parallel shared(mesh, time) +#endif call mesh % ProlongSolutionToFaces(NCONS) ! ! ----------------------------------------------------- @@ -365,7 +371,9 @@ SUBROUTINE ComputeTimeDerivativeIsolated( mesh, particles, time, mode, HO_Elemen ! call TimeDerivative_ComputeQDotIsolated(mesh = mesh , & t = time ) +#ifndef _OPENACC !$omp end parallel +#endif ! END SUBROUTINE ComputeTimeDerivativeIsolated @@ -405,24 +413,33 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! *********************************************** ! if (flowIsNavierStokes) then -!$omp do schedule(runtime) private(i,j,k) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) async(1) +#else +!$omp do schedule(runtime) private(i,j,k) +#endif do eID = 1, size(mesh % elements) !$acc loop vector collapse(3) do k = 0, mesh % elements(eID) % Nxyz(3) ; do j = 0, mesh % elements(eID) % Nxyz(2) ; do i = 0, mesh % elements(eID) % Nxyz(1) call get_laminar_mu_kappa(mesh % elements(eID) % storage % Q(:,i,j,k), & - mesh % elements(eID) % storage % mu_NS(1,i,j,k),& + mesh % elements(eID) % storage % mu_NS(1,i,j,k),& mesh % elements(eID) % storage % mu_NS(2,i,j,k)) end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if if ( LESModel % active) then -!$omp do schedule(runtime) private(i,j,k,delta,mu_smag) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, LESModel) async(1) +#else +!$omp do schedule(runtime) private(i,j,k,delta,mu_smag) +#endif do eID = 1, size(mesh % elements) delta = (mesh % elements(eID) % geom % Volume / product(mesh % elements(eID) % Nxyz + 1)) ** (1.0_RP / 3.0_RP) !$acc loop vector collapse(3) @@ -440,8 +457,11 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) mesh % elements(eID) % storage % mu_turb_NS(i,j,k) * dimensionless % mu_to_kappa end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if ! ! Compute viscosity at interior and boundary faces @@ -472,23 +492,35 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! Compute Riemann solver of non-shared faces ! ****************************************** ! -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang collapse(2) present(mesh) async(1) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_interior) ; do side = 1,2 fID = mesh % faces_interior(iFace) call computeElementInterfaceFlux_viscous(mesh % faces(fID), side) - end do ; end do + end do ; end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) async(1) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_interior) fID = mesh % faces_interior(iFace) call computeElementInterfaceFlux(mesh % faces(fID)) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do nowait +#endif call computeBoundaryFlux(mesh, t) ! @@ -496,14 +528,20 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! Surface integrals and scaling of elements with non-shared faces ! *************************************************************** ! -!$omp do schedule(runtime) private(i,j,k,eID) +#ifdef _OPENACC !$acc parallel loop gang num_gangs(size(mesh % elements_sequential)) vector_length(128) present(mesh) async(1) +#else +!$omp do schedule(runtime) private(i,j,k,eID) +#endif do iEl = 1, size(mesh % elements_sequential) eID = mesh % elements_sequential(iEl) call TimeDerivative_FacesContribution(mesh % elements(eID), mesh) end do -!$acc end parallel loop +#ifdef _OPENACC +!$acc end parallel loop +#else !$omp end do +#endif ! ! **************************** ! Wait until messages are sent @@ -536,27 +574,39 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! Compute Riemann solver of shared faces ! ************************************** ! -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang num_gangs(size(mesh % faces_mpi)) present(mesh) private(fID) async(1) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_mpi) fID = mesh % faces_mpi(iFace) call computeMPIFaceFlux(mesh % faces(fID)) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif ! ! *********************************************************** ! Surface integrals and scaling of elements with shared faces ! *********************************************************** ! -!$omp do schedule(runtime) private(i,j,k,eID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) async(1) +#else +!$omp do schedule(runtime) private(i,j,k,eID) +#endif do iEl = 1, size(mesh % elements_mpi) eID = mesh % elements_mpi(iEl) call TimeDerivative_FacesContribution(mesh % elements(eID), mesh) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif ! ! Add an MPI Barrier ! ------------------ @@ -646,42 +696,55 @@ subroutine TimeDerivative_ComputeQDot( mesh , particles, t) ! *********************** ! Now add the source term ! *********************** -!$omp do schedule(runtime) private(i,j,k) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) present(mesh) async(1) +#else +!$omp do schedule(runtime) private(i,j,k) +#endif do eID = 1, mesh % no_of_elements !$acc loop vector collapse(4) do k = 0, mesh % elements(eID) % Nxyz(3) ; do j = 0, mesh % elements(eID) % Nxyz(2) ; do i = 0, mesh % elements(eID) % Nxyz(1) ; do eq = 1, NCONS mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) + mesh % elements(eID) % storage % S_NS(eq,i,j,k) end do ; end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif ! ! ********************* ! Add IBM source term ! ********************* if( mesh % IBM % active ) then if( .not. mesh % IBM % semiImplicit ) then -!$omp do schedule(runtime) private(i,j,k,Source,Q_target) +#ifdef _OPENACC ! Check if update(t) is required !$acc parallel loop gang present(mesh) copyin(t) async(1) - do eID = 1, mesh % no_of_elements +#else +!$omp do schedule(runtime) private(i,j,k,Source,Q_target) + ! Check if update(t) is required +#endif + do eID = 1, mesh % no_of_elements !$acc loop vector collapse(3) private(Source, Q_target) do k = 0, mesh % elements(eID) % Nxyz(3) ; do j = 0, mesh % elements(eID) % Nxyz(2) ; do i = 0, mesh % elements(eID) % Nxyz(1) if( mesh % elements(eID) % isInsideBody(i,j,k) ) then - if( mesh % IBM % stl(mesh % elements(eID) % STL(i,j,k)) % move ) then + if( mesh % IBM % stl(mesh % elements(eID) % STL(i,j,k)) % move ) then ! Q_target = IBM_MaskVelocity(mesh%IBM, mesh % elements(eID)% storage% Q(:,i,j,k), NCONS, mesh % elements(eID)% STL(i,j,k), mesh % elements(eID)% geom% x(:,i,j,k), t ) call IBM_MaskVelocity(mesh % IBM, mesh % elements(eID) % storage % Q(:,i,j,k), NCONS, mesh % elements(eID) % STL(i,j,k), mesh % elements(eID) % geom % x(:,i,j,k), t, Q_target ) call IBM_SourceTerm(mesh % IBM, eID = eID, Q = mesh % elements(eID) % storage % Q(:,i,j,k), Q_target = Q_target, Source = Source, wallfunction = .false. ) - else + else call IBM_SourceTerm(mesh % IBM, eID = eID, Q = mesh % elements(eID) % storage % Q(:,i,j,k), Source = Source, wallfunction = .false. ) - end if + end if mesh % elements(eID) % storage % QDot(:,i,j,k) = mesh % elements(eID) % storage % QDot(:,i,j,k) + Source end if end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do +#endif if( mesh % IBM % Wallfunction ) then !$omp single call mesh % IBM % GetBandRegionStates( mesh % elements ) @@ -750,8 +813,11 @@ subroutine TimeDerivative_ComputeQDotHO( mesh , particles, t) end if if ( LESModel % active) then - !$omp do schedule(runtime) private(i,j,k,delta,mu_smag) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, LESModel) +#else + !$omp do schedule(runtime) private(i,j,k,delta,mu_smag) +#endif do eID = 1, size(mesh % elements) delta = (mesh % elements(eID) % geom % Volume / product(mesh % elements(eID) % Nxyz + 1)) ** (1.0_RP / 3.0_RP) !$acc loop vector collapse(3) @@ -771,8 +837,11 @@ subroutine TimeDerivative_ComputeQDotHO( mesh , particles, t) mesh % elements(eID) % storage % mu_turb_NS(i,j,k) * dimensionless % mu_to_kappa end do ; end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if ! ! Compute viscosity at interior and boundary faces @@ -1043,8 +1112,11 @@ subroutine compute_viscosity_at_faces(no_of_faces, no_of_sides, face_ids, mesh) real(kind=RP) :: delta, mu_smag if (flowIsNavierStokes) then -!$omp do schedule(runtime) private(i,j) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, face_ids) async(1) +#else +!$omp do schedule(runtime) private(i,j) +#endif do iFace = 1, no_of_faces fID = face_ids(iFace) !$acc loop vector collapse(3) @@ -1056,13 +1128,19 @@ subroutine compute_viscosity_at_faces(no_of_faces, no_of_sides, face_ids, mesh) end do end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if if ( LESModel % Active ) then -!$omp do schedule(runtime) private(i,j,delta,mu_smag) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, LESModel, face_ids) async(1) +#else +!$omp do schedule(runtime) private(i,j,delta,mu_smag) +#endif do iFace = 1, no_of_faces delta = sqrt(mesh % faces(face_ids(iFace)) % geom % surface / product(mesh % faces(face_ids(iFace)) % Nf + 1)) !$acc loop vector collapse(3) @@ -1079,8 +1157,11 @@ subroutine compute_viscosity_at_faces(no_of_faces, no_of_sides, face_ids, mesh) end do end do ; end do end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if end subroutine compute_viscosity_at_faces @@ -1309,8 +1390,11 @@ subroutine TimeDerivative_VolumetricContribution(mesh) ! ! Compute inviscid - viscous contravariant flux ! --------------------------------------------- - !$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) num_gangs(9700) present(mesh) async(1) +#else + !$omp do schedule(runtime) +#endif do eID = 1 , size(mesh % elements) !$acc loop vector collapse(3) private(inviscidFlux, viscousFlux) @@ -1353,98 +1437,136 @@ subroutine TimeDerivative_VolumetricContribution(mesh) mesh % elements(eID) % storage % QDot) end do - !$acc end parallel loop +#ifdef _OPENACC + !$acc end parallel loop +#else !$omp end do +#endif end subroutine TimeDerivative_VolumetricContribution ! !/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -! + subroutine TimeDerivative_VolumetricContribution_Split_StandardDG(mesh) + use HexMeshClass + use NodalStorageClass, only: NodalStorage + use ElementClass + use DGIntegrals + use RiemannSolvers_NS, only: twopointflux_selector => StandardDG_TwoPointFlux + implicit none + type(HexMesh), intent(inout) :: mesh + integer :: i, j, k, l, eq, eID + real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc + include 'split_template.inc' + end subroutine - subroutine TimeDerivative_VolumetricContribution_Split(mesh) + subroutine TimeDerivative_VolumetricContribution_Split_Morinishi(mesh) use HexMeshClass - use NodalStorageClass, only: NodalStorage + use NodalStorageClass, only: NodalStorage use ElementClass use DGIntegrals - use RiemannSolvers_NS + use RiemannSolvers_NS, only: twopointflux_selector => Morinishi_TwoPointFlux implicit none - type(HexMesh), intent (inout) :: mesh -! -! --------------- -! Local variables -! --------------- -! - integer :: i, j, k,l,eq, eID - real(kind=RP) :: Flux(1:NCONS, 1:NDIM) + type(HexMesh), intent(inout) :: mesh + integer :: i, j, k, l, eq, eID + real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc + include 'split_template.inc' + end subroutine - !$acc parallel present(mesh) vector_length(128) num_gangs(9750) async(1) - !$acc loop gang - do eID = 1 , size(mesh % elements) - - !$acc loop vector collapse(3) private(Flux) - do k = 0, mesh % elements(eID) % Nxyz(3) - do j = 0, mesh % elements(eID) % Nxyz(2) - do i = 0, mesh % elements(eID) % Nxyz(1) - - call ViscousFlux_STATE( NCONS, NGRAD, mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % U_x(:,i,j,k), & - mesh % elements(eID) % storage % U_y(:,i,j,k) , mesh % elements(eID) % storage % U_z(:,i,j,k), & - mesh % elements(eID) % storage % mu_ns(1,i,j,k), 0.0_RP, & - mesh % elements(eID) % storage % mu_ns(2,i,j,k), Flux) - !$acc loop seq - do eq = 1, NCONS - - mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IX) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradXi(IX,i,j,k) & - - Flux(eq,IY) * mesh % elements(eID) % geom % jGradXi(IY,i,j,k) & - - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradXi(IZ,i,j,k) + subroutine TimeDerivative_VolumetricContribution_Split_Ducros(mesh) + use HexMeshClass + use NodalStorageClass, only: NodalStorage + use ElementClass + use DGIntegrals + use RiemannSolvers_NS, only: twopointflux_selector => Ducros_TwoPointFlux + implicit none + type(HexMesh), intent(inout) :: mesh + integer :: i, j, k, l, eq, eID + real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc + include 'split_template.inc' + end subroutine - mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IY) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradEta(IX,i,j,k) & - - Flux(eq,IY) * mesh % elements(eID) % geom % jGradEta(IY,i,j,k) & - - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradEta(IZ,i,j,k) - - mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IZ) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradZeta(IX,i,j,k) & - - Flux(eq,IY) * mesh % elements(eID) % geom % jGradZeta(IY,i,j,k) & - - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradZeta(IZ,i,j,k) - ! initialize to 0 to accumulate - mesh % elements(eID) % storage % Qdot(eq,i,j,k) = 0.0_RP - end do - end do ; end do ; end do + subroutine TimeDerivative_VolumetricContribution_Split_KennedyGruber(mesh) + use HexMeshClass + use NodalStorageClass, only: NodalStorage + use ElementClass + use DGIntegrals + use RiemannSolvers_NS, only: twopointflux_selector => KennedyGruber_TwoPointFlux + implicit none + type(HexMesh), intent(inout) :: mesh + integer :: i, j, k, l, eq, eID + real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc + include 'split_template.inc' + end subroutine - call ScalarWeakIntegrals_StdVolumeGreen( mesh % elements(eID) % Nxyz, NCONS, mesh % elements(eID) % storage % contravariantFlux, & - mesh % elements(eID) % storage % QDot) -! -! ************************************* -! Compute interior contravariant fluxes -! ************************************* -! -! Compute inviscid contravariant flux -! ----------------------------------- - !$acc loop vector collapse(3) private(Flux) - do k = 0, mesh % elements(eID) % Nxyz(3) - do j = 0, mesh % elements(eID) % Nxyz(2) - do i = 0, mesh % elements(eID) % Nxyz(1) - !$acc loop seq - do l = 0, mesh % elements(eID) % Nxyz(1) - call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,l,j,k), mesh % elements(eID) % geom % jGradXi(:,i,j,k), mesh % elements(eID) % geom % jGradXi(:,l,j,k), Flux(:,IX)) - call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,i,l,k), mesh % elements(eID) % geom % jGradEta(:,i,j,k), mesh % elements(eID) % geom % jGradEta(:,i,l,k), Flux(:,IY)) - call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,i,j,l), mesh % elements(eID) % geom % jGradZeta(:,i,j,k), mesh % elements(eID) % geom % jGradZeta(:,i,j,l), Flux(:,IZ)) - - !$acc loop seq - do eq = 1, NCONS - mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) & - - NodalStorage(mesh % elements(eID) % Nxyz(1)) % sharpD(i,l) * Flux(eq,IX) & - - NodalStorage(mesh % elements(eID) % Nxyz(2)) % sharpD(j,l) * Flux(eq,IY) & - - NodalStorage(mesh % elements(eID) % Nxyz(3)) % sharpD(k,l) * Flux(eq,IZ) - end do - end do + subroutine TimeDerivative_VolumetricContribution_Split_Pirozzoli(mesh) + use HexMeshClass + use NodalStorageClass, only: NodalStorage + use ElementClass + use DGIntegrals + use RiemannSolvers_NS, only: twopointflux_selector => Pirozzoli_TwoPointFlux + implicit none + type(HexMesh), intent(inout) :: mesh + integer :: i, j, k, l, eq, eID + real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc + include 'split_template.inc' + end subroutine - end do ; end do ; end do + subroutine TimeDerivative_VolumetricContribution_Split_EntropyConserving(mesh) + use HexMeshClass + use NodalStorageClass, only: NodalStorage + use ElementClass + use DGIntegrals + use RiemannSolvers_NS, only: twopointflux_selector => EntropyConserving_TwoPointFlux + implicit none + type(HexMesh), intent(inout) :: mesh + integer :: i, j, k, l, eq, eID + real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc + include 'split_template.inc' + end subroutine - enddo - !$acc end parallel loop + subroutine TimeDerivative_VolumetricContribution_Split_Chandrasekar(mesh) + use HexMeshClass + use NodalStorageClass, only: NodalStorage + use ElementClass + use DGIntegrals + use RiemannSolvers_NS, only: twopointflux_selector => Chandrasekar_TwoPointFlux + implicit none + type(HexMesh), intent(inout) :: mesh + integer :: i, j, k, l, eq, eID + real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc + include 'split_template.inc' + end subroutine + + subroutine TimeDerivative_VolumetricContribution_Split(mesh) + use HexMeshClass + use RiemannSolvers_NSKeywordsModule + use RiemannSolvers_NS, only : whichAverage + implicit none + type(HexMesh), intent(inout) :: mesh + + select case (whichAverage) + case (STANDARD_AVG) + call TimeDerivative_VolumetricContribution_Split_StandardDG(mesh) + case (MORINISHI_AVG) + call TimeDerivative_VolumetricContribution_Split_Morinishi(mesh) + case (DUCROS_AVG) + call TimeDerivative_VolumetricContribution_Split_Ducros(mesh) + case (KENNEDYGRUBER_AVG) + call TimeDerivative_VolumetricContribution_Split_KennedyGruber(mesh) + case (PIROZZOLI_AVG) + call TimeDerivative_VolumetricContribution_Split_Pirozzoli(mesh) + case (ENTROPYCONS_AVG) + call TimeDerivative_VolumetricContribution_Split_EntropyConserving(mesh) + case (CHANDRASEKAR_AVG) + call TimeDerivative_VolumetricContribution_Split_Chandrasekar(mesh) + case default + print*, "Averaging not recognized." + errorMessage(STD_OUT) + error stop + end select + end subroutine - end subroutine TimeDerivative_VolumetricContribution_Split -! !/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ! subroutine TimeDerivative_FacesContribution(e, mesh) diff --git a/Solver/src/addons/probes2tecplot/Makefile b/Solver/src/addons/probes2tecplot/Makefile new file mode 100644 index 000000000..70d430b22 --- /dev/null +++ b/Solver/src/addons/probes2tecplot/Makefile @@ -0,0 +1,109 @@ +##################################################################### +## HORSES - a High-Order Spectral Element Solver +## +## probes2tecplot Makefile +## +##################################################################### + +######################################## +# 1.- User defined parameters +######################################## +INSTALL_DIR?=./ +FFLAGS?=-cpp +AR?=ar +ARFLAGS?=cr +RANLIB?=ranlib +MOD_FLAG?=-J +PLATFORM?=LINUX +COMPILER?=gfortran +MODE?=RELEASE +COMM?=PARALLEL +MAKE=make +AR=ar +RANLIB=ranlib +PROG=./bin/probes2tecplot + +# Directory of shared source files from horses2tecplot +H2T_DIR=../horses2tecplot + +# Files compiled from the horses2tecplot directory (shared code) +H2T_OBJS= Storage \ + OutputVariables \ + sharedSpectralBasis \ + +# Files compiled from the probes2tecplot directory (new code) +LOCAL_OBJS= Probes2tecplot \ + main \ + +H2T_OBJS_BUILD=$(foreach obj,$(H2T_OBJS),./build/$(obj).o) +LOCAL_OBJS_BUILD=$(foreach obj,$(LOCAL_OBJS),./build/$(obj).o) +ALL_OBJS_BUILD=$(H2T_OBJS_BUILD) $(LOCAL_OBJS_BUILD) + + +########################################## +## Include dependencies ## +########################################## +-include ./make.deps + + +########################################################## +## COMPILATION RULES ## +########################################################## + +.DEFAULT_GOAL := $(PROG) + +# Explicit rules for local files (explicit rules always beat pattern rules) +./build/Probes2tecplot.o: ./Probes2tecplot.f90 + @echo + @echo $< + $(FC) $(FFLAGS) $(MACROS) $(EXTLIB_INC) -I./include $(INCLUDE) $(INCLUDE)_ns $(MACROS) -c $< -o $@ $(MOD_FLAG) ./include + +./build/main.o: ./main.f90 + @echo + @echo $< + $(FC) $(FFLAGS) $(MACROS) $(EXTLIB_INC) -I./include $(INCLUDE) $(INCLUDE)_ns $(MACROS) -c $< -o $@ $(MOD_FLAG) ./include + +# Pattern rule for shared files from horses2tecplot +./build/%.o: $(H2T_DIR)/%.f90 + @echo + @echo $< + $(FC) $(FFLAGS) $(MACROS) $(EXTLIB_INC) -I./include $(INCLUDE) $(INCLUDE)_ns $(MACROS) -c $< -o $@ $(MOD_FLAG) ./include + +$(PROG): header mkdirs $(ALL_OBJS_BUILD) + @echo + @echo + @echo "---------------------------------" + @echo "| Linking " $@ "|" + @echo "---------------------------------" + $(FC) $(FFLAGS) $(EXTLIB_INC) -o $(PROG) $(ALL_OBJS_BUILD) $(HORSES_LIBS_EXEC) $(LIBS) + + +install: $(PROG) + cp -p $(PROG) $(INSTALL_DIR)/bin + cp -p ./include/*.mod $(INSTALL_DIR)/include + @echo + @echo + +########################################## +## Extra auxiliary rules ## +########################################## +clean: FORCE + rm -f ./include/*.mod ./lib/*.a ./build/*.o + rm -f $(PROG) + +mkdirs: FORCE + @mkdir -p ./build + @mkdir -p ./include + @mkdir -p ./bin + @mkdir -p ./lib + +header: FORCE + @echo + @echo "================================" + @echo ">> Building probes2tecplot" + @echo "================================" + @echo + @echo + +## Dummy target to make sure a rule is executed +FORCE: \ No newline at end of file diff --git a/Solver/src/addons/probes2tecplot/Probes2tecplot.f90 b/Solver/src/addons/probes2tecplot/Probes2tecplot.f90 new file mode 100644 index 000000000..9cc973053 --- /dev/null +++ b/Solver/src/addons/probes2tecplot/Probes2tecplot.f90 @@ -0,0 +1,813 @@ +#include "Includes.h" +module Probes2tecplotModule +! +! /////////////////////////////////////////////////////////////////////////// +! +! This module implements the probes2tecplot functionality: +! - Reads a probes file with (name, x, y, z) entries +! - Finds each probe point in the mesh using Newton iteration +! accelerated with element bounding boxes +! - Caches probe locations (eID + xi) to a binary file for reuse +! - Interpolates the solution at each probe position using Lagrange basis +! - Writes output variables to a Tecplot .tec file +! +! /////////////////////////////////////////////////////////////////////////// +! + use SMConstants + use Storage + use SolutionFile + use SharedSpectralBasis + use NodalStorageClass + use OutputVariables + use PhysicsStorage + use Headers + implicit none + + private + public ProbeData_t + public ReadProbesFile + public FindAndCacheProbes + public InterpolateAndWriteProbes + + integer, parameter :: MAX_NEWTON = 100 + real(kind=RP), parameter :: NEWTON_TOL = 1.0e-10_RP + real(kind=RP), parameter :: INSIDE_TOL = 1.0e-6_RP + real(kind=RP), parameter :: BBOX_TOL = 1.0e-6_RP + integer, parameter :: MAX_WARN = 20 +#define PRECISION_FORMAT "(E18.10)" + + type ProbeData_t + character(len=LINE_LENGTH) :: name + real(kind=RP) :: x(NDIM) + real(kind=RP) :: xi(NDIM) + integer :: eID + logical :: found + end type ProbeData_t + + type ElementBBox_t + real(kind=RP) :: xmin(NDIM) + real(kind=RP) :: xmax(NDIM) + end type ElementBBox_t + + contains + + pure function getBaseName(fullPath) result(base) + character(len=*), intent(in) :: fullPath + character(len=LINE_LENGTH) :: base + integer :: slashPos, dotPos + + slashPos = index(trim(fullPath), '/', BACK=.true.) + base = fullPath(slashPos+1:) + dotPos = index(trim(base), '.', BACK=.true.) + if (dotPos > 1) base = base(1:dotPos-1) + + end function getBaseName + + pure function getDirName(fullPath) result(dir) + character(len=*), intent(in) :: fullPath + character(len=LINE_LENGTH) :: dir + integer :: slashPos + + slashPos = index(trim(fullPath), '/', BACK=.true.) + if (slashPos > 0) then + dir = fullPath(1:slashPos) + else + dir = "" + end if + + end function getDirName + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Read probe positions from file. +! Format: one probe per line -> name x y z +! Lines starting with # are comments. Name is optional. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine ReadProbesFile(fileName, probes, no_of_probes) + implicit none + character(len=*), intent(in) :: fileName + type(ProbeData_t), allocatable, intent(out) :: probes(:) + integer, intent(out) :: no_of_probes +! +! --------------- +! Local variables +! --------------- +! + integer :: fid, io, lineCount, pID, nread + character(len=512) :: line + real(kind=RP) :: x, y, z + character(len=LINE_LENGTH) :: name + + write(STD_OUT,'(/)') + call Section_Header("Reading probes file") + write(STD_OUT,'(30X,A,A)') "-> File: ", trim(fileName) + + open(newunit=fid, file=trim(fileName), action="read", status="old", iostat=io) + if (io .ne. 0) then + write(STD_OUT,'(30X,A,A,A)') "ERROR: Could not open probes file '", trim(fileName), "'" + no_of_probes = 0 + return + end if + + lineCount = 0 + do + read(fid, '(A)', iostat=io) line + if (io .ne. 0) exit + line = adjustl(line) + if (len_trim(line) .eq. 0) cycle + if (line(1:1) .eq. '#') cycle + lineCount = lineCount + 1 + end do + close(fid) + + no_of_probes = lineCount + if (no_of_probes .eq. 0) return + allocate(probes(no_of_probes)) + + open(newunit=fid, file=trim(fileName), action="read", status="old") + pID = 0 + do + read(fid, '(A)', iostat=io) line + if (io .ne. 0) exit + line = adjustl(line) + if (len_trim(line) .eq. 0) cycle + if (line(1:1) .eq. '#') cycle + pID = pID + 1 + read(line, *, iostat=nread) name, x, y, z + if (nread .ne. 0) then + read(line, *, iostat=nread) x, y, z + if (nread .ne. 0) then + write(STD_OUT,'(30X,A,I0,A)') "WARNING: Cannot parse probe line ", pID, ". Skipping." + pID = pID - 1 + cycle + end if + write(probes(pID)%name, '(A,I0)') "probe_", pID + else + probes(pID)%name = trim(name) + end if + probes(pID)%x(1) = x + probes(pID)%x(2) = y + probes(pID)%x(3) = z + probes(pID)%found = .false. + probes(pID)%eID = 0 + probes(pID)%xi = 0.0_RP + end do + no_of_probes = pID + close(fid) + write(STD_OUT,'(30X,A,I0,A)') "-> Found ", no_of_probes, " probe(s)." + + end subroutine ReadProbesFile + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Find all probes in the mesh (or load from cache if available). +! Cache file is named .pcache and stores eID + xi +! for each probe. It is invalidated if meshName or probesFileName change. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine FindAndCacheProbes(mesh, meshName, probesFileName, probes, no_of_probes) + use FileReadingUtilities, only: getFileName + implicit none + type(Mesh_t), intent(in) :: mesh + character(len=*), intent(in) :: meshName + character(len=*), intent(in) :: probesFileName + type(ProbeData_t), allocatable, intent(out) :: probes(:) + integer, intent(out) :: no_of_probes +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: cacheFileName + logical :: cacheLoaded + type(ElementBBox_t), allocatable :: bbox(:) + integer :: eID + integer(kind=8) :: t0, t1, count_rate + + cacheFileName = trim(getFileName(probesFileName)) // ".pcache" + + write(STD_OUT,'(/)') + call Section_Header("Probe location lookup") + write(STD_OUT,'(30X,A,A)') "-> Cache file: ", trim(cacheFileName) + + call system_clock(t0, count_rate) + call ReadProbeCache(cacheFileName, meshName, probesFileName, probes, no_of_probes, cacheLoaded) + + if (cacheLoaded) then + call system_clock(t1) + write(STD_OUT,'(30X,A,I0,A)') "-> Loaded ", no_of_probes, " probe locations from cache." + write(STD_OUT,'(30X,A,F8.3,A)') "-> Cache load time: ", & + real(t1-t0,RP)/real(count_rate,RP), " s" + return + end if + + write(STD_OUT,'(30X,A)') "-> Cache not found or stale. Searching mesh..." +! +! Read probe positions from file +! -------------------------------- + call ReadProbesFile(probesFileName, probes, no_of_probes) + if (no_of_probes .eq. 0) return +! +! Initialize spectral basis for mesh geometry orders +! --------------------------------------------------- + do eID = 1, mesh % no_of_elements + associate(e => mesh % elements(eID)) + call addNewSpectralBasis(spA, e % Nmesh, mesh % nodeType) + end associate + end do +! +! Build element bounding boxes +! ---------------------------- + call BuildBoundingBoxes(mesh, bbox) +! +! Find each probe in the mesh +! ---------------------------- + call Section_Header("Finding probes in mesh") + call system_clock(t0, count_rate) + call FindAllProbesInMesh(mesh, bbox, probes, no_of_probes) + call system_clock(t1) + write(STD_OUT,'(30X,A,F8.3,A)') "-> Mesh search time: ", & + real(t1-t0,RP)/real(count_rate,RP), " s" + + deallocate(bbox) +! +! Save cache for future runs +! -------------------------- + call SaveProbeCache(cacheFileName, meshName, probesFileName, probes, no_of_probes) + + end subroutine FindAndCacheProbes + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Read probe location cache from binary file. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine ReadProbeCache(cacheFileName, meshName, probesFileName, probes, no_of_probes, success) + implicit none + character(len=*), intent(in) :: cacheFileName + character(len=*), intent(in) :: meshName + character(len=*), intent(in) :: probesFileName + type(ProbeData_t), allocatable, intent(out) :: probes(:) + integer, intent(out) :: no_of_probes + logical, intent(out) :: success +! +! --------------- +! Local variables +! --------------- +! + integer :: fid, io, pID + character(len=LINE_LENGTH) :: stored_mesh, stored_probes + + success = .false. + no_of_probes = 0 + + open(newunit=fid, file=trim(cacheFileName), action="read", form="unformatted", & + access="stream", status="old", iostat=io) + if (io .ne. 0) return + + read(fid, iostat=io) stored_mesh ; if (io .ne. 0) then ; close(fid) ; return ; end if + read(fid, iostat=io) stored_probes; if (io .ne. 0) then ; close(fid) ; return ; end if + + if (trim(stored_mesh) .ne. trim(meshName) .or. & + trim(stored_probes) .ne. trim(probesFileName)) then + write(STD_OUT,'(30X,A)') "-> Cache is stale (mesh or probes file changed). Recomputing." + close(fid) + return + end if + + read(fid, iostat=io) no_of_probes ; if (io .ne. 0) then ; close(fid) ; return ; end if + + allocate(probes(no_of_probes)) + do pID = 1, no_of_probes + read(fid, iostat=io) probes(pID) % name, probes(pID) % x, & + probes(pID) % eID, probes(pID) % xi, probes(pID) % found + if (io .ne. 0) then + write(STD_OUT,'(30X,A)') "WARNING: Cache read error. Recomputing." + deallocate(probes) + no_of_probes = 0 + close(fid) + return + end if + end do + + close(fid) + success = .true. + + end subroutine ReadProbeCache + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Save probe location cache to binary file. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine SaveProbeCache(cacheFileName, meshName, probesFileName, probes, no_of_probes) + implicit none + character(len=*), intent(in) :: cacheFileName + character(len=*), intent(in) :: meshName + character(len=*), intent(in) :: probesFileName + type(ProbeData_t), intent(in) :: probes(:) + integer, intent(in) :: no_of_probes +! +! --------------- +! Local variables +! --------------- +! + integer :: fid, io, pID + character(len=LINE_LENGTH) :: stored_mesh, stored_probes + + stored_mesh = trim(meshName) + stored_probes = trim(probesFileName) + + open(newunit=fid, file=trim(cacheFileName), action="write", form="unformatted", & + access="stream", status="replace", iostat=io) + if (io .ne. 0) then + write(STD_OUT,'(30X,A,A)') "WARNING: Could not write cache file: ", trim(cacheFileName) + return + end if + + write(fid) stored_mesh + write(fid) stored_probes + write(fid) no_of_probes + do pID = 1, no_of_probes + write(fid) probes(pID) % name, probes(pID) % x, & + probes(pID) % eID, probes(pID) % xi, probes(pID) % found + end do + close(fid) + + write(STD_OUT,'(30X,A,A)') "-> Cache saved: ", trim(cacheFileName) + + end subroutine SaveProbeCache + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Build axis-aligned bounding boxes for all elements. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine BuildBoundingBoxes(mesh, bbox) + implicit none + type(Mesh_t), intent(in) :: mesh + type(ElementBBox_t), allocatable, intent(out) :: bbox(:) +! +! --------------- +! Local variables +! --------------- +! + integer :: e, i, j, k + + allocate(bbox(mesh % no_of_elements)) + + do e = 1, mesh % no_of_elements + associate(elem => mesh % elements(e)) + bbox(e) % xmin = huge(1.0_RP) + bbox(e) % xmax = -huge(1.0_RP) + do k = 0, elem % Nmesh(3) + do j = 0, elem % Nmesh(2) + do i = 0, elem % Nmesh(1) + bbox(e) % xmin = min(bbox(e) % xmin, elem % x(:,i,j,k)) + bbox(e) % xmax = max(bbox(e) % xmax, elem % x(:,i,j,k)) + end do + end do + end do + end associate + end do + + end subroutine BuildBoundingBoxes + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Find all probes in the mesh using Newton iteration with bounding box filter. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine FindAllProbesInMesh(mesh, bbox, probes, no_of_probes) + implicit none + type(Mesh_t), intent(in) :: mesh + type(ElementBBox_t), intent(in) :: bbox(:) + type(ProbeData_t), intent(inout) :: probes(:) + integer, intent(in) :: no_of_probes +! +! --------------- +! Local variables +! --------------- +! + integer :: pID, found_count, warn_count, processed_count, new_count, milestone + integer(kind=8) :: t0, t1, count_rate + + found_count = 0 + warn_count = 0 + processed_count = 0 + milestone = max(1, no_of_probes / 20) ! 5% steps + + write(STD_OUT,'(30X,A,I0,A)') "-> Searching ", no_of_probes, " probes..." + + call system_clock(t0, count_rate) + + !$OMP PARALLEL DO SCHEDULE(DYNAMIC,100) REDUCTION(+:found_count) DEFAULT(SHARED) PRIVATE(new_count) + do pID = 1, no_of_probes + call FindProbeInMesh(mesh, bbox, probes(pID) % x, & + probes(pID) % eID, probes(pID) % xi, probes(pID) % found) + if (probes(pID) % found) found_count = found_count + 1 + !$OMP ATOMIC CAPTURE + processed_count = processed_count + 1 + new_count = processed_count + !$OMP END ATOMIC + if (mod(new_count, milestone) .eq. 0) then + !$OMP CRITICAL (progress_bar) + write(STD_OUT,'(30X,A,I0,A,I0,A,F5.1,A)') & + " [", new_count, "/", no_of_probes, & + "] ", 100.0_RP * new_count / no_of_probes, "%" + !$OMP END CRITICAL (progress_bar) + end if + end do + !$OMP END PARALLEL DO + + call system_clock(t1) + write(STD_OUT,'(30X,A)') " [done]" + write(STD_OUT,'(30X,A,F8.3,A)') "-> Parallel search time: ", & + real(t1-t0,RP)/real(count_rate,RP), " s" + + ! Print warnings sequentially after the parallel search + do pID = 1, no_of_probes + if (.not. probes(pID) % found) then + warn_count = warn_count + 1 + if (warn_count .le. MAX_WARN) then + write(STD_OUT,'(30X,A,A,A,3(ES12.4,1X),A)') "-> WARNING: Probe '", & + trim(probes(pID) % name), "' at (", probes(pID) % x, ") not found in mesh." + end if + end if + end do + + if (warn_count .gt. MAX_WARN) then + write(STD_OUT,'(30X,A,I0,A)') " (", warn_count - MAX_WARN, " additional not-found probes suppressed)" + end if + write(STD_OUT,'(30X,A,I0,A,I0,A)') "-> ", found_count, " of ", no_of_probes, " probes found in mesh." + + end subroutine FindAllProbesInMesh + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Search all elements for x_probe, using bounding box pre-filter. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine FindProbeInMesh(mesh, bbox, x_probe, eID, xi, found) + implicit none + type(Mesh_t), intent(in) :: mesh + type(ElementBBox_t), intent(in) :: bbox(:) + real(kind=RP), intent(in) :: x_probe(NDIM) + integer, intent(out) :: eID + real(kind=RP), intent(out) :: xi(NDIM) + logical, intent(out) :: found +! +! --------------- +! Local variables +! --------------- +! + integer :: e + + found = .false. + eID = 0 + + do e = 1, mesh % no_of_elements + if (any(x_probe < bbox(e) % xmin - BBOX_TOL)) cycle + if (any(x_probe > bbox(e) % xmax + BBOX_TOL)) cycle + call FindPointInElement(mesh % elements(e), mesh % is2D, x_probe, xi, found) + if (found) then + eID = e + return + end if + end do + + end subroutine FindProbeInMesh + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Newton iteration to find reference coordinates (xi,eta,zeta) of x_probe +! inside element e. Returns found=.true. if the point is inside the element. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine FindPointInElement(e, is2D, x_probe, xi, found) + implicit none + type(Element_t), intent(in) :: e + logical, intent(in) :: is2D + real(kind=RP), intent(in) :: x_probe(NDIM) + real(kind=RP), intent(out) :: xi(NDIM) + logical, intent(out) :: found +! +! --------------- +! Local variables +! --------------- +! + real(kind=RP) :: Lxi (0:e % Nmesh(1)) + real(kind=RP) :: Leta (0:e % Nmesh(2)) + real(kind=RP) :: Lzeta (0:e % Nmesh(3)) + real(kind=RP) :: dLxi (0:e % Nmesh(1)) + real(kind=RP) :: dLeta (0:e % Nmesh(2)) + real(kind=RP) :: dLzeta(0:e % Nmesh(3)) + real(kind=RP) :: x_calc(NDIM), F(NDIM), Jac(NDIM,NDIM), dxi_step(NDIM) + integer :: iter, i, j, k + + xi = 0.0_RP + found = .false. + + do iter = 1, MAX_NEWTON + + Lxi = spA(e % Nmesh(1)) % lj(xi(1)) + Leta = spA(e % Nmesh(2)) % lj(xi(2)) + dLxi = spA(e % Nmesh(1)) % dlj(xi(1)) + dLeta = spA(e % Nmesh(2)) % dlj(xi(2)) + + if (is2D) then + Lzeta(0) = 1.0_RP + dLzeta(0) = 0.0_RP + else + Lzeta = spA(e % Nmesh(3)) % lj(xi(3)) + dLzeta = spA(e % Nmesh(3)) % dlj(xi(3)) + end if + + x_calc = 0.0_RP + Jac = 0.0_RP + do k = 0, e % Nmesh(3) + do j = 0, e % Nmesh(2) + do i = 0, e % Nmesh(1) + x_calc = x_calc + e % x(:,i,j,k) * Lxi(i) * Leta(j) * Lzeta(k) + Jac(:,1) = Jac(:,1) + e % x(:,i,j,k) * dLxi(i)* Leta(j) * Lzeta(k) + Jac(:,2) = Jac(:,2) + e % x(:,i,j,k) * Lxi(i) * dLeta(j) * Lzeta(k) + Jac(:,3) = Jac(:,3) + e % x(:,i,j,k) * Lxi(i) * Leta(j) * dLzeta(k) + end do + end do + end do + + F = x_calc - x_probe + if (norm2(F) < NEWTON_TOL) exit + + dxi_step = Solve3x3(Jac, F) + xi = xi - dxi_step + + xi(1) = max(min(xi(1), 2.0_RP), -2.0_RP) + xi(2) = max(min(xi(2), 2.0_RP), -2.0_RP) + if (.not. is2D) xi(3) = max(min(xi(3), 2.0_RP), -2.0_RP) + + end do + + if (is2D) then + found = (abs(xi(1)) <= 1.0_RP + INSIDE_TOL) .and. & + (abs(xi(2)) <= 1.0_RP + INSIDE_TOL) + else + found = all(abs(xi) <= 1.0_RP + INSIDE_TOL) + end if + + end subroutine FindPointInElement + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Interpolate conservative variables Q at reference coordinates xi. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + function InterpolateQAtProbe(e, xi, nodeType) result(Q_probe) + implicit none + type(Element_t), intent(in) :: e + real(kind=RP), intent(in) :: xi(NDIM) + integer, intent(in) :: nodeType + real(kind=RP) :: Q_probe(NVARS) +! +! --------------- +! Local variables +! --------------- +! + real(kind=RP) :: Lxi (0:e % Nsol(1)) + real(kind=RP) :: Leta (0:e % Nsol(2)) + real(kind=RP) :: Lzeta(0:e % Nsol(3)) + integer :: i, j, k + + Lxi = spA(e % Nsol(1)) % lj(xi(1)) + Leta = spA(e % Nsol(2)) % lj(xi(2)) + + if (e % Nsol(3) .eq. 0) then + Lzeta(0) = 1.0_RP + else + Lzeta = spA(e % Nsol(3)) % lj(xi(3)) + end if + + Q_probe = 0.0_RP + do k = 0, e % Nsol(3) + do j = 0, e % Nsol(2) + do i = 0, e % Nsol(1) + Q_probe = Q_probe + e % Q(:,i,j,k) * Lxi(i) * Leta(j) * Lzeta(k) + end do + end do + end do + + end function InterpolateQAtProbe + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! For each solution: interpolate Q at all probe locations, compute output +! variables and write a Tecplot .tec file. The mock element is allocated +! once and reused across probes. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + subroutine InterpolateAndWriteProbes(mesh, probes, no_of_probes, solutionName, probesFileName) + use FileReadingUtilities, only: getFileName + implicit none + type(Mesh_t), intent(inout) :: mesh + type(ProbeData_t), intent(in) :: probes(:) + integer, intent(in) :: no_of_probes + character(len=*), intent(in) :: solutionName + character(len=*), intent(in) :: probesFileName +! +! --------------- +! Local variables +! --------------- +! + type(Element_t) :: mock_e + real(kind=RP), allocatable :: output(:,:,:,:) + real(kind=RP), allocatable :: outVars(:,:) + real(kind=RP), allocatable :: Q_all(:,:) + integer :: pID, fid, eID, found_count + integer(kind=8) :: t_total, t_end, count_rate + character(len=LINE_LENGTH) :: outputFile, formatout + character(len=1024) :: title + + write(STD_OUT,'(/)') + call Section_Header("Probes extraction") + call system_clock(t_total, count_rate) + found_count = count(probes(1:no_of_probes) % found) + write(STD_OUT,'(30X,A,I0,A,I0,A)') "-> Extracting ", found_count, & + " of ", no_of_probes, " probes..." +! +! Initialize spectral basis for solution orders +! ----------------------------------------------- + do eID = 1, mesh % no_of_elements + associate(e => mesh % elements(eID)) + call addNewSpectralBasis(spA, e % Nsol, mesh % nodeType) + end associate + end do +! +! Set up output variable metadata +! --------------------------------- + call getOutputVariables() + allocate(output (no_of_outputVariables, 0:0, 0:0, 0:0)) + allocate(outVars(no_of_outputVariables, no_of_probes)) + outVars = 0.0_RP +! +! Allocate mock element once — reused for every probe +! ----------------------------------------------------- + mock_e % Nout = 0 + allocate(mock_e % Qout (NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % xOut (NDIM, 0:0, 0:0, 0:0)) + allocate(mock_e % QDot_out(NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % U_xout (NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % U_yout (NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % U_zout (NVARS, 0:0, 0:0, 0:0)) + allocate(mock_e % statsout (1, 0:0, 0:0, 0:0)) + allocate(mock_e % mu_NSout (1, 0:0, 0:0, 0:0)) + allocate(mock_e % ut_NSout (1, 0:0, 0:0, 0:0)) + allocate(mock_e % wallYout (1, 0:0, 0:0, 0:0)) + allocate(mock_e % mu_sgsout(1, 0:0, 0:0, 0:0)) + allocate(mock_e % wallY (1, 0:0, 0:0, 0:0)) + allocate(mock_e % ut_NS (1, 0:0, 0:0, 0:0)) + + mock_e % QDot_out = 0.0_RP + mock_e % U_xout = 0.0_RP + mock_e % U_yout = 0.0_RP + mock_e % U_zout = 0.0_RP + mock_e % wallY = 0.0_RP + mock_e % ut_NS = 0.0_RP + mock_e % statsout = 0.0_RP + mock_e % mu_NSout = 0.0_RP + mock_e % ut_NSout = 0.0_RP + mock_e % wallYout = 0.0_RP + mock_e % mu_sgsout = 0.0_RP + mock_e % sensor = 0.0_RP +! +! Phase 1: interpolate Q at all probe locations (parallel) +! ---------------------------------------------------------- + allocate(Q_all(NVARS, no_of_probes)) + Q_all = 0.0_RP + + !$OMP PARALLEL DO SCHEDULE(DYNAMIC,256) DEFAULT(SHARED) + do pID = 1, no_of_probes + if (.not. probes(pID) % found) cycle + Q_all(:,pID) = InterpolateQAtProbe(mesh % elements(probes(pID) % eID), & + probes(pID) % xi, mesh % nodeType) + end do + !$OMP END PARALLEL DO +! +! Phase 2: compute output variables (sequential — shared mock element) +! --------------------------------------------------------------------- + do pID = 1, no_of_probes + if (.not. probes(pID) % found) cycle + + mock_e % Qout(:,0,0,0) = Q_all(:,pID) + mock_e % xOut(:,0,0,0) = probes(pID) % x + + call ComputeOutputVariables(no_of_outputVariables, outputVariableNames, & + mock_e % Nout, mock_e, output, mesh % refs, & + .false., .false., .false.) + outVars(:,pID) = output(:,0,0,0) + end do + + deallocate(Q_all) +! +! Deallocate mock element +! ------------------------ + deallocate(mock_e % Qout, mock_e % xOut, mock_e % QDot_out) + deallocate(mock_e % U_xout, mock_e % U_yout, mock_e % U_zout) + deallocate(mock_e % statsout, mock_e % mu_NSout, mock_e % ut_NSout) + deallocate(mock_e % wallYout, mock_e % mu_sgsout) + deallocate(mock_e % wallY, mock_e % ut_NS) + deallocate(output) +! +! Write output .tec file +! ----------------------- + outputFile = trim(getDirName(solutionName)) // trim(getBaseName(solutionName)) // & + "." // trim(getBaseName(probesFileName)) // ".tec" + write(STD_OUT,'(30X,A,A)') "-> Output file: ", trim(outputFile) + + open(newunit=fid, file=trim(outputFile), action="write", status="unknown") + + write(title,'(A,A,A)') '"Probes extracted from ', trim(solutionName), '"' + write(fid,'(A,A)') "TITLE = ", trim(title) + write(fid,'(A,A)') 'VARIABLES = "x","y","z"', trim(getOutputVariablesLabel()) + write(fid,'(A,I0,A)') 'ZONE T="Probes", I=', no_of_probes, ', J=1, K=1, F=POINT' + + write(formatout,'(A,I0,A,A)') "(", 3 + no_of_outputVariables, PRECISION_FORMAT, ")" + + do pID = 1, no_of_probes + write(fid, trim(formatout)) probes(pID) % x, outVars(:,pID) + end do + + close(fid) + call system_clock(t_end) + write(STD_OUT,'(30X,A,I0,A,I0,A,F8.3,A)') "-> Written ", found_count, & + " of ", no_of_probes, " probes in ", & + real(t_end-t_total,8)/real(count_rate,8), " s" + + deallocate(outVars) + + end subroutine InterpolateAndWriteProbes + +! +!///////////////////////////////////////////////////////////////////////////////////// +! +! Solve a 3x3 linear system A*x = b using Cramer's rule. +! +!///////////////////////////////////////////////////////////////////////////////////// +! + pure function Solve3x3(A, b) result(x) + implicit none + real(kind=RP), intent(in) :: A(3,3), b(3) + real(kind=RP) :: x(3) +! +! --------------- +! Local variables +! --------------- +! + real(kind=RP) :: det, inv_det + + det = A(1,1)*(A(2,2)*A(3,3) - A(2,3)*A(3,2)) & + - A(1,2)*(A(2,1)*A(3,3) - A(2,3)*A(3,1)) & + + A(1,3)*(A(2,1)*A(3,2) - A(2,2)*A(3,1)) + + if (abs(det) < 1.0e-30_RP) then + x = 0.0_RP + return + end if + + inv_det = 1.0_RP / det + + x(1) = inv_det * ( b(1)*(A(2,2)*A(3,3) - A(2,3)*A(3,2)) & + - A(1,2)*(b(2)*A(3,3) - A(2,3)*b(3)) & + + A(1,3)*(b(2)*A(3,2) - A(2,2)*b(3)) ) + + x(2) = inv_det * ( A(1,1)*(b(2)*A(3,3) - A(2,3)*b(3)) & + - b(1)*(A(2,1)*A(3,3) - A(2,3)*A(3,1)) & + + A(1,3)*(A(2,1)*b(3) - b(2)*A(3,1)) ) + + x(3) = inv_det * ( A(1,1)*(A(2,2)*b(3) - b(2)*A(3,2)) & + - A(1,2)*(A(2,1)*b(3) - b(2)*A(3,1)) & + + b(1)*(A(2,1)*A(3,2) - A(2,2)*A(3,1)) ) + + end function Solve3x3 + +end module Probes2tecplotModule diff --git a/Solver/src/addons/probes2tecplot/main.f90 b/Solver/src/addons/probes2tecplot/main.f90 new file mode 100644 index 000000000..742c1a310 --- /dev/null +++ b/Solver/src/addons/probes2tecplot/main.f90 @@ -0,0 +1,267 @@ +! +!/////////////////////////////////////////////////////////////////////////////////////// +! +! probes2tecplot - Extract solution variables at probe points from HORSES3D solution files +! +! Usage (control file): +! probes2tecplot control_file.convert +! +! Usage (command line): +! probes2tecplot mesh.hmesh solution.hsol probes.dat [--output-variables=p,u,v,w] +! +! Control file format: +! hmesh file= Mesh.hmesh +! hsol file= Solution.hsol +! probes file= probes.dat +! output variables= p, u, v, w +! flow equations= ns +! boundary file= optional +! partition file= optional +! +! Probes file format (one probe per line, lines starting with # are comments): +! probe_name x y z +! +! Probe location cache: +! On the first run, probe locations (element ID + reference coordinates) are +! saved to .pcache. Subsequent runs with the same mesh and probes +! file load directly from cache, skipping the mesh search entirely. +! +!/////////////////////////////////////////////////////////////////////////////////////// +! +program probes2tecplot + use SMConstants + use Storage + use SolutionFile + use SharedSpectralBasis + use Headers + use MPI_Process_Info + use OutputVariables, only: outScale, hasVariablesFlag, askedVariables, Lreference, getNoOfCommas + use Probes2tecplotModule + implicit none + + integer :: no_of_solutions + character(len=LINE_LENGTH) :: meshName + character(len=LINE_LENGTH) :: probesFileName + character(len=LINE_LENGTH), allocatable :: solutionNames(:) + integer :: iSol + type(Mesh_t) :: mesh + type(ProbeData_t), allocatable :: probes(:) + integer :: no_of_probes + integer(kind=8) :: t_sol, t_end, sol_rate + + call MPI_Process % Init + call Main_Header("HORSES Probes to TecPlot extraction utility", __DATE__, __TIME__) +! +! Parse the control file or command line arguments +! ------------------------------------------------- + call getProbes2tecplotArgs(meshName, probesFileName, no_of_solutions, solutionNames) +! +! Construct spectral basis +! ------------------------ + call ConstructSpectralBasis +! +! Read mesh +! --------- + call mesh % ReadMesh(meshName) +! +! Find probes in mesh (or load from cache) — done only once +! ---------------------------------------------------------- + call FindAndCacheProbes(mesh, trim(meshName), trim(probesFileName), probes, no_of_probes) +! +! Process each solution file +! -------------------------- + if (no_of_probes .gt. 0) then + do iSol = 1, no_of_solutions + write(STD_OUT,'(/,/)') + call Section_Header("Processing solution") + call system_clock(t_sol, sol_rate) + call mesh % ReadSolution(solutionNames(iSol)) + call system_clock(t_end) + write(STD_OUT,'(30X,A,A)') "-> Solution: ", trim(solutionNames(iSol)) + write(STD_OUT,'(30X,A,F8.3,A)') "-> Read time: ", & + real(t_end-t_sol,8)/real(sol_rate,8), " s" + call InterpolateAndWriteProbes(mesh, probes, no_of_probes, solutionNames(iSol), trim(probesFileName)) + call system_clock(t_end) + write(STD_OUT,'(30X,A,F8.3,A)') "-> Total time: ", & + real(t_end-t_sol,8)/real(sol_rate,8), " s" + end do + end if +! +! Cleanup +! ------- + if (allocated(probes)) deallocate(probes) + call DestructSpectralBasis + call MPI_Process % Close + + contains + + subroutine getProbes2tecplotArgs(meshName, probesFileName, no_of_solutions, solutionNames) + use FTValueDictionaryClass, only: FTValueDictionary + use FileReaders , only: ReadControlFile + use FileReadingUtilities , only: getCharArrayFromString + use Utilities , only: toLower + implicit none + character(len=*), intent(out) :: meshName + character(len=*), intent(out) :: probesFileName + integer, intent(out) :: no_of_solutions + character(len=LINE_LENGTH), allocatable, intent(out) :: solutionNames(:) +! +! --------------- +! Local variables +! --------------- +! + integer :: no_of_arguments, i, pos, pos2, fID, io + character(len=LINE_LENGTH) :: arg, inputResultName + type(FTValueDictionary) :: controlVariables + logical :: useControlFile + + no_of_arguments = command_argument_count() + no_of_solutions = 0 +! +! Determine input mode: control file (1 arg) or command line (>1 args) +! --------------------------------------------------------------------- + useControlFile = (no_of_arguments .eq. 1) + + if (no_of_arguments .eq. 0) then + write(STD_OUT,'(A)') "Usage: probes2tecplot control_file.convert" + write(STD_OUT,'(A)') " probes2tecplot mesh.hmesh solution.hsol probes.dat [--output-variables=p,u,v,w]" + call exit(1) + end if +! +! ---- Control file mode ---- +! --------------------------- + if (useControlFile) then + call controlVariables % initWithSize(16) + call ReadControlFile(controlVariables) + + if (.not. controlVariables % containsKey("hmesh file")) then + write(STD_OUT,'(A)') "ERROR: 'hmesh file' not specified in control file." + call exit(1) + end if + meshName = controlVariables % stringValueForKey("hmesh file", LINE_LENGTH) + + if (.not. controlVariables % containsKey("probes file")) then + write(STD_OUT,'(A)') "ERROR: 'probes file' not specified in control file." + call exit(1) + end if + probesFileName = controlVariables % stringValueForKey("probes file", LINE_LENGTH) + + if (.not. controlVariables % containsKey("hsol file")) then + write(STD_OUT,'(A)') "ERROR: 'hsol file' not specified in control file." + call exit(1) + end if + + inputResultName = controlVariables % stringValueForKey("hsol file", LINE_LENGTH) + no_of_solutions = getNoOfCommas(trim(inputResultName)) + 1 + allocate(solutionNames(no_of_solutions)) + + if (no_of_solutions .eq. 1) then + solutionNames(1) = adjustl(trim(inputResultName)) + else + pos = 0 + do i = 1, no_of_solutions - 1 + pos2 = index(trim(inputResultName(pos+1:)), ",") + pos + solutionNames(i) = trim(adjustl(inputResultName(pos+1:pos2-1))) + pos = pos2 + end do + pos = index(trim(inputResultName), ",", BACK=.true.) + solutionNames(no_of_solutions) = adjustl(trim(inputResultName(pos+1:))) + end if + + hasVariablesFlag = controlVariables % containsKey("output variables") + if (hasVariablesFlag) askedVariables = controlVariables % stringValueForKey("output variables", LINE_LENGTH) + outScale = .not. controlVariables % logicalValueForKey("dimensionless") + Lreference = controlVariables % getValueOrDefault("reference length (m)", 1.0_RP) + + hasBoundaries = controlVariables % containsKey("boundary file") + if (hasBoundaries) boundaryFileName = controlVariables % stringValueForKey("boundary file", LINE_LENGTH) + hasMPIranks = controlVariables % containsKey("partition file") + if (hasMPIranks) partitionFileName = controlVariables % stringValueForKey("partition file", LINE_LENGTH) + + if (controlVariables % containsKey("flow equations")) then + flowEq = controlVariables % stringValueForKey("flow equations", LINE_LENGTH) + call toLower(flowEq) + else + flowEq = "ns" + end if +! +! ---- Command line mode ---- +! --------------------------- + else + outScale = .true. + hasVariablesFlag = .false. + hasBoundaries = .false. + hasMPIranks = .false. + flowEq = "ns" + Lreference = 1.0_RP + probesFileName = "" + + do i = 1, no_of_arguments + call get_command_argument(i, arg) + open(newunit=fID, file=trim(arg), action="read", form="unformatted", & + access="stream", iostat=io) + close(fID) + if (io .ne. 0) cycle + if (getSolutionFileType(arg) .eq. MESH_FILE .or. & + getSolutionFileType(arg) .eq. ZONE_MESH_FILE) then + meshName = trim(arg) + exit + end if + end do + + no_of_solutions = 0 + do i = 1, no_of_arguments + call get_command_argument(i, arg) + open(newunit=fID, file=trim(arg), action="read", form="unformatted", & + access="stream", iostat=io) + close(fID) + if (io .ne. 0) cycle + if (getSolutionFileType(arg) .ne. MESH_FILE .and. & + getSolutionFileType(arg) .ne. ZONE_MESH_FILE) then + no_of_solutions = no_of_solutions + 1 + end if + end do + + allocate(solutionNames(max(1, no_of_solutions))) + pos = 0 + do i = 1, no_of_arguments + call get_command_argument(i, arg) + if (arg(1:2) .eq. "--") then + if (index(arg, "--output-variables=") .ne. 0) then + hasVariablesFlag = .true. + askedVariables = arg(len("--output-variables=")+1:) + else if (index(arg, "--dimensionless") .ne. 0) then + outScale = .false. + end if + cycle + end if + + open(newunit=fID, file=trim(arg), action="read", form="unformatted", & + access="stream", iostat=io) + close(fID) + + if (io .eq. 0) then + if (getSolutionFileType(arg) .ne. MESH_FILE .and. & + getSolutionFileType(arg) .ne. ZONE_MESH_FILE) then + pos = pos + 1 + solutionNames(pos) = trim(arg) + end if + else + open(newunit=fID, file=trim(arg), action="read", status="old", iostat=io) + close(fID) + if (io .eq. 0 .and. len_trim(probesFileName) .eq. 0) then + probesFileName = trim(arg) + end if + end if + end do + + if (len_trim(probesFileName) .eq. 0) then + write(STD_OUT,'(A)') "ERROR: No probes file found in arguments." + write(STD_OUT,'(A)') "Usage: probes2tecplot mesh.hmesh solution.hsol probes.dat" + call exit(1) + end if + end if + + end subroutine getProbes2tecplotArgs + +end program probes2tecplot \ No newline at end of file diff --git a/Solver/src/libs/discretization/DGSEMClass.f90 b/Solver/src/libs/discretization/DGSEMClass.f90 index 00851690e..01e09bfc8 100644 --- a/Solver/src/libs/discretization/DGSEMClass.f90 +++ b/Solver/src/libs/discretization/DGSEMClass.f90 @@ -643,9 +643,12 @@ FUNCTION ComputeMaxResiduals(mesh) RESULT(maxResidual) R6 = 0.0_RP c = 0.0_RP +#ifdef _OPENACC !$acc parallel loop gang present(mesh) reduction(max:R1,R2,R3,R4,R5,c) +#else !$omp parallel shared(maxResidual, R1, R2, R3, R4, R5, R6, c, mesh) default(private) !$omp do reduction(max:R1,R2,R3,R4,R5,R6,c) schedule(runtime) +#endif DO id = 1, SIZE( mesh % elements ) N = mesh % elements(id) % Nxyz @@ -700,9 +703,12 @@ FUNCTION ComputeMaxResiduals(mesh) RESULT(maxResidual) END DO +#ifdef _OPENACC +!$acc end parallel loop +#else !$omp end do !$omp end parallel -!$acc end parallel loop +#endif #if defined FLOW && (!(SPALARTALMARAS)) @@ -778,9 +784,12 @@ subroutine MaxTimeStep( self, cfl, dcfl, MaxDt , MaxDtVec) TimeStep_Conv = huge(1._RP) TimeStep_Visc = huge(1._RP) if (present(MaxDtVec)) MaxDtVec = huge(1._RP) +#ifdef _OPENACC !$acc parallel loop gang present(self) copyin(cfl, dcfl) reduction(min:TimeStep_Conv,TimeStep_Visc) +#else !$omp parallel shared(self,TimeStep_Conv,TimeStep_Visc,NodalStorage,cfl,dcfl,flowIsNavierStokes,MaxDtVec) default(private) !$omp do reduction(min:TimeStep_Conv,TimeStep_Visc) schedule(runtime) +#endif do eID = 1, SIZE(self % mesh % elements) N = self % mesh % elements(eID) % Nxyz @@ -876,9 +885,12 @@ subroutine MaxTimeStep( self, cfl, dcfl, MaxDt , MaxDtVec) end do ; end do ; end do end do +#ifdef _OPENACC +!$acc end parallel loop +#else !$omp end do !$omp end parallel -!$acc end parallel loop +#endif !!$acc update host(TimeStep_Conv, TimeStep_Visc) diff --git a/Solver/src/libs/discretization/EllipticBR1.f90 b/Solver/src/libs/discretization/EllipticBR1.f90 index 20850b5c0..6c9a74bd1 100644 --- a/Solver/src/libs/discretization/EllipticBR1.f90 +++ b/Solver/src/libs/discretization/EllipticBR1.f90 @@ -170,14 +170,20 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) ! Compute Riemann solvers of non-shared faces ! ******************************************* ! -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, self) async(1) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_interior) fID = mesh % faces_interior(iFace) call BR1_ComputeElementInterfaceAverage(self, mesh % faces(fID), nEqn, nGradEqn) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do nowait +#endif nZones = size(mesh % zones) !$omp do schedule(runtime) private(zoneID) @@ -186,8 +192,11 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) enddo !$omp end do +#ifdef _OPENACC + !$acc parallel loop gang present(mesh) async(1) +#else !$omp do schedule(runtime) private(eID) - !$acc parallel loop gang present(mesh) async(1) +#endif do iEl = 1, size(mesh % elements_sequential) eID = mesh % elements_sequential(iEl) ! @@ -195,8 +204,11 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) ! ------------------------- call BR1_GradientFaceLoop(nGradEqn, mesh % elements(eID), mesh) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif call HexMesh_ProlongGradientsToFaces(mesh, size(mesh % elements_sequential), mesh % elements_sequential, nGradEqn) @@ -207,18 +219,27 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) end if !$omp end single +#ifdef _OPENACC + !$acc parallel loop gang present(mesh, self) async(1) +#else !$omp do schedule(runtime) private(fID) - !$acc parallel loop gang present(mesh, self) async(1) +#endif do iFace = 1, size(mesh % faces_mpi) fID = mesh % faces_mpi(iFace) call BR1_ComputeMPIFaceAverage(self, mesh % faces(fID), nEqn, nGradEqn) end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do +#endif ! -!$omp do schedule(runtime) private(eID) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) present(mesh, self) async(1) +#else +!$omp do schedule(runtime) private(eID) +#endif do iEl = 1, size(mesh % elements_mpi) eID = mesh % elements_mpi(iEl) ! @@ -226,8 +247,11 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) ! ------------------------- call BR1_GradientFaceLoop(nGradEqn, mesh % elements(eID), mesh) end do -!$omp end do +#ifdef _OPENACC !$acc end parallel loop +#else +!$omp end do +#endif ! ! Prolong gradients ! ----------------- diff --git a/Solver/src/libs/discretization/EllipticBR2.f90 b/Solver/src/libs/discretization/EllipticBR2.f90 index 740767dff..de25c78dd 100644 --- a/Solver/src/libs/discretization/EllipticBR2.f90 +++ b/Solver/src/libs/discretization/EllipticBR2.f90 @@ -199,14 +199,20 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient end do !$omp end do nowait else -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) private(fID) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_interior) fID = mesh % faces_interior(iFace) call BR2_GradientInterfaceSolution(self, mesh % faces(fID), nEqn, nGradEqn) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do nowait +#endif end if !$acc wait print*, "I am in BR2 line 232" @@ -219,14 +225,20 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient end do !$omp end do else -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_boundary) fID = mesh % faces_boundary(iFace) call BR2_GradientInterfaceSolutionBoundary(mesh % faces(fID), nEqn, nGradEqn, time) end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do +#endif end if !$acc wait print*, "I am in BR2 line 254" @@ -244,14 +256,20 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient end do !$omp end do else -!$omp do schedule(runtime) private(eID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, self) copyin(self) +#else +!$omp do schedule(runtime) private(eID) +#endif do iEl = 1, size(mesh % elements_sequential) eID = mesh % elements_sequential(iEl) call BR2_ComputeGradientFaceIntegrals(self, nGradEqn, mesh % elements(eID), mesh) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if !$acc wait print*, "I am in BR2 line 278" @@ -272,14 +290,20 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient ! Compute MPI interface solutions ! ******************************* ! -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_mpi) fID = mesh % faces_mpi(iFace) call BR2_GradientInterfaceSolutionMPI(self, mesh % faces(fID), nEqn, nGradEqn) end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do +#endif ! ! ************************************************** ! Compute face integrals for elements with MPI faces @@ -293,14 +317,20 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient end do !$omp end do else -!$omp do schedule(runtime) private(eID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, self) copyin(self) +#else +!$omp do schedule(runtime) private(eID) +#endif do iEl = 1, size(mesh % elements_mpi) eID = mesh % elements_mpi(iEl) call BR2_ComputeGradientFaceIntegrals(self, nGradEqn, mesh % elements(eID), mesh) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if #endif diff --git a/Solver/src/libs/discretization/EllipticIP.f90 b/Solver/src/libs/discretization/EllipticIP.f90 index d71bf7931..0ab056795 100644 --- a/Solver/src/libs/discretization/EllipticIP.f90 +++ b/Solver/src/libs/discretization/EllipticIP.f90 @@ -265,14 +265,20 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO end do !$omp end do else -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_interior) fID = mesh % faces_interior(iFace) call IP_GradientInterfaceSolution(self, mesh % faces(fID), nEqn, nGradEqn) end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do +#endif end if if (HOElements) then @@ -284,15 +290,21 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO end do !$omp end do else -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_boundary) fID = mesh % faces_boundary(iFace) zoneBCName = mesh % zones (mesh % faces(fID) % zone) % zoneBCName call IP_GradientInterfaceSolutionBoundary(self, mesh % faces(fID), zoneBCName, nEqn, nGradEqn, time) end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do +#endif end if ! ! ********************** @@ -307,14 +319,20 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO end do !$omp end do else -!$omp do schedule(runtime) private(eID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, self) copyin(self) +#else +!$omp do schedule(runtime) private(eID) +#endif do iEl = 1, size(mesh % elements_sequential) eID = mesh % elements_sequential(iEl) call IP_ComputeGradientFaceIntegrals(self,nGradEqn, mesh % elements(eID), mesh) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if ! ! ****************** @@ -332,14 +350,20 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO ! Compute MPI interface solutions ! ******************************* ! -!$omp do schedule(runtime) private(fID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) +#else +!$omp do schedule(runtime) private(fID) +#endif do iFace = 1, size(mesh % faces_mpi) fID = mesh % faces_mpi(iFace) call IP_GradientInterfaceSolutionMPI(self, mesh % faces(fID), nEqn, nGradEqn) end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do +#endif ! ! ************************************************** ! Compute face integrals for elements with MPI faces @@ -353,14 +377,20 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO end do !$omp end do else -!$omp do schedule(runtime) private(eID) +#ifdef _OPENACC !$acc parallel loop gang present(mesh, self) copyin(self) +#else +!$omp do schedule(runtime) private(eID) +#endif do iEl = 1, size(mesh % elements_mpi) eID = mesh % elements_mpi(iEl) call IP_ComputeGradientFaceIntegrals(self,nGradEqn, mesh % elements(eID), mesh) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end if #endif diff --git a/Solver/src/libs/discretization/SCsensorClass.f90 b/Solver/src/libs/discretization/SCsensorClass.f90 index 06e7ef0af..551e2772f 100755 --- a/Solver/src/libs/discretization/SCsensorClass.f90 +++ b/Solver/src/libs/discretization/SCsensorClass.f90 @@ -457,7 +457,6 @@ subroutine Compute_SCsensor(sensor, sem, t) ! Local variables ! --------------- ! - type(Element), pointer :: e integer :: eID real(RP) :: s @@ -469,28 +468,27 @@ subroutine Compute_SCsensor(sensor, sem, t) ! Add 'inertia' to the scaled value ! --------------------------------- if (sensor % min_steps > 1) then ! Enter the loop only if necessary -!$omp parallel do default(private) shared(sem) +!$omp parallel do default(private) shared(sem, sensor) do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) - s = e % storage % sensor + associate(st => sem % mesh % elements(eID) % storage) + s = st % sensor if (s > 0.0_RP) then - if (e % storage % prev_sensor <= 0.0_RP) then - e % storage % first_sensed = 0 - e % storage % prev_sensor = s + if (st % prev_sensor <= 0.0_RP) then + st % first_sensed = 0 + st % prev_sensor = s else - e % storage % first_sensed = e % storage % first_sensed + 1 - e % storage % prev_sensor = s + st % first_sensed = st % first_sensed + 1 + st % prev_sensor = s end if - elseif (e % storage % first_sensed < sensor % min_steps) then - e % storage % first_sensed = e % storage % first_sensed + 1 - e % storage % sensor = e % storage % prev_sensor + elseif (st % first_sensed < sensor % min_steps) then + st % first_sensed = st % first_sensed + 1 + st % sensor = st % prev_sensor end if + end associate end do !$omp end parallel do end if - nullify(e) - end subroutine Compute_SCsensor ! !/////////////////////////////////////////////////////////////////////////////// @@ -821,8 +819,6 @@ subroutine Sensor_truncation(sensor, sem, t) real(RP) :: mTE procedure(UserDefinedSourceTermNS_f) :: UserDefinedSourceTermNS - type(Element), pointer :: e - type(Element), pointer :: ce type(DGSem), pointer :: csem @@ -832,13 +828,10 @@ subroutine Sensor_truncation(sensor, sem, t) ! --------------- !$omp parallel do default(private) shared(sem, csem) do eID = 1, sem % mesh % no_of_elements - - e => sem % mesh % elements(eID) - ce => csem % mesh % elements(eID) - + associate(e => sem % mesh % elements(eID), ce => csem % mesh % elements(eID)) call Interp3DArrays(NCONS, e % Nxyz, e % storage % Q, ce % Nxyz, ce % storage % Q) ce % storage % sensor = e % storage % sensor - + end associate end do !$omp end parallel do @@ -846,12 +839,9 @@ subroutine Sensor_truncation(sensor, sem, t) ! ! Maximum TE computation ! ---------------------- -!$omp parallel do default(private) shared(sem, csem) +!$omp parallel do default(private) shared(sem, csem, sensor) do eID = 1, sem % mesh % no_of_elements - - e => sem % mesh % elements(eID) - ce => csem % mesh % elements(eID) - + associate(e => sem % mesh % elements(eID), ce => csem % mesh % elements(eID)) ! Use G_NS as temporary storage for Qdot call Interp3DArrays(NCONS, e % Nxyz, e % storage % QDot, ce % Nxyz, ce % storage % G_NS) @@ -875,12 +865,10 @@ subroutine Sensor_truncation(sensor, sem, t) else e % storage % sensor = SinRamp(sensor, log10(mTE)) end if - + end associate end do !$omp end parallel do - nullify(e) - nullify(ce) nullify(csem) end subroutine Sensor_truncation diff --git a/Solver/src/libs/mesh/HexElementClass.f90 b/Solver/src/libs/mesh/HexElementClass.f90 index 86576a189..3c03413e2 100644 --- a/Solver/src/libs/mesh/HexElementClass.f90 +++ b/Solver/src/libs/mesh/HexElementClass.f90 @@ -35,7 +35,12 @@ Module ElementClass private public Element - public HexElement_ComputeLocalGradient, HexElement_ProlongSolToFaces, HexElement_ProlongGradientsToFaces, HexElement_ProlongGradientsToFaces_GL, HexElement_ProlongSolToFaces_GL + public HexElement_ComputeLocalGradient, HexElement_ProlongSolToFaces, & + HexElement_ProlongGradientsToFaces, HexElement_ProlongGradientsToFaces_GL, & + HexElement_ProlongSolToFaces_GL +#ifdef NAVIERSTOKES + public HexElement_NSGradientVariables +#endif public PrintElement, SetElementBoundaryNames, SurfInfo_t ! @@ -341,50 +346,50 @@ subroutine HexElement_ProlongSolToFaces(self, nEqn, f_side, side) case(1) !$acc loop vector collapse(2) private(Qlocal) - do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Qlocal(1:nEqn)= self % storage % Q(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(1)) % v(0,FRONT) + do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) + Qlocal(1:nEqn)= self % storage % Q(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(2)) % v(0,FRONT) !$acc loop seq - do j = 1, self % Nxyz(3) + do j = 1, self % Nxyz(2) !$acc loop seq do eq = 1, nEqn - Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(j,FRONT) + Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(2)) % v(j,FRONT) enddo end do f_side % storage(self % faceSide(EFRONT)) % Q_aux(1:nEqn,i,k) = Qlocal(1:nEqn) end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT)) case(2) !$acc loop vector collapse(2) private(Qlocal) - do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Qlocal(1:nEqn) = self % storage % Q(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(1)) % v(0,BACK) + do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) + Qlocal(1:nEqn) = self % storage % Q(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(2)) % v(0,BACK) !$acc loop seq - do j = 1, self % Nxyz(3) + do j = 1, self % Nxyz(2) !$acc loop seq do eq = 1, nEqn - Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(j,BACK) + Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(2)) % v(j,BACK) enddo end do f_side % storage(self % faceSide(EBACK)) % Q_aux(1:nEqn,i,k) = Qlocal(1:nEqn) end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK)) case(3) !$acc loop vector collapse(2) private(Qlocal) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Qlocal(1:nEqn)= self % storage % Q(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(1)) % v(0,BOTTOM) + Qlocal(1:nEqn)= self % storage % Q(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(3)) % v(0,BOTTOM) !$acc loop seq do k = 1, self % Nxyz(3) !$acc loop seq do eq = 1, nEqn - Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(k,BOTTOM) + Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(3)) % v(k,BOTTOM) enddo end do f_side % storage(self % faceSide(EBOTTOM)) % Q_aux(1:nEqn,i,j) = Qlocal(1:nEqn) end do ; end do - call Face_AdaptSolToFace(f_side,nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM)) + call Face_AdaptSolToFace(f_side,nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM)) case(4) @@ -405,17 +410,17 @@ subroutine HexElement_ProlongSolToFaces(self, nEqn, f_side, side) case(5) !$acc loop vector collapse(2) private(Qlocal) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Qlocal(1:nEqn) = self % storage % Q(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(1)) % v(0,TOP) + Qlocal(1:nEqn) = self % storage % Q(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(3)) % v(0,TOP) !$acc loop seq do k = 1, self % Nxyz(3) !$acc loop seq do eq = 1, nEqn - Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(k,TOP) + Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(3)) % v(k,TOP) enddo end do f_side % storage(self % faceSide(ETOP)) % Q_aux(1:nEqn,i,j) = Qlocal(1:nEqn) end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP)) case(6) @@ -459,20 +464,20 @@ subroutine HexElement_ProlongSolToFaces_GL(self, nEqn, f_side, side) select case(side) case(1) !$acc loop vector collapse(3) - do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) + do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) do eq = 1, nEqn f_side % storage(self % faceSide(EFRONT)) % Q_aux(eq,i,k) = self % storage % Q(eq,i,0,k) enddo end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT)) case(2) !$acc loop vector collapse(3) - do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) + do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) do eq = 1, nEqn f_side % storage(self % faceSide(EBACK)) % Q_aux(eq,i,k) = self % storage % Q(eq,i,self % Nxyz(2),k) enddo end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK)) case(3) !$acc loop vector collapse(3) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) @@ -480,7 +485,7 @@ subroutine HexElement_ProlongSolToFaces_GL(self, nEqn, f_side, side) f_side % storage(self % faceSide(EBOTTOM)) % Q_aux(eq,i,j) = self % storage % Q(eq,i,j,0) enddo end do ; end do - call Face_AdaptSolToFace(f_side,nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM)) + call Face_AdaptSolToFace(f_side,nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM)) case(4) !$acc loop vector collapse(3) @@ -498,7 +503,7 @@ subroutine HexElement_ProlongSolToFaces_GL(self, nEqn, f_side, side) f_side % storage(self % faceSide(ETOP)) % Q_aux(eq,i,j) = self % storage % Q(eq,i,j,self % Nxyz(3)) enddo end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP)) case(6) !$acc loop vector collapse(3) @@ -537,12 +542,12 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d select case(side) case(1) !$acc loop vector collapse(2) private(Q_grad) - do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Q_grad(1:nEqn)= U_xyz(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(1)) % v(0,FRONT) + do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) + Q_grad(1:nEqn)= U_xyz(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(2)) % v(0,FRONT) !$acc loop seq - do j = 1, self % Nxyz(3) + do j = 1, self % Nxyz(2) do eq = 1, nEqn - Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(j,FRONT) + Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(2)) % v(j,FRONT) enddo end do !$acc loop seq @@ -550,16 +555,16 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d f_side % storage(self % faceSide(EFRONT)) % Q_aux(eq,i,k) = Q_grad(eq) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT), dir) case(2) !$acc loop vector collapse(2) private(Q_grad) - do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Q_grad(1:nEqn) = U_xyz(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(1)) % v(0,BACK) + do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) + Q_grad(1:nEqn) = U_xyz(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(2)) % v(0,BACK) !$acc loop seq - do j = 1, self % Nxyz(3) + do j = 1, self % Nxyz(2) do eq = 1, nEqn - Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(j,BACK) + Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(2)) % v(j,BACK) enddo end do !$acc loop seq @@ -567,16 +572,16 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d f_side % storage(self % faceSide(EBACK)) % Q_aux(eq,i,k) = Q_grad(eq) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK), dir) case(3) !$acc loop vector collapse(2) private(Q_grad) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Q_grad(1:nEqn)= U_xyz(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(1)) % v(0,BOTTOM) + Q_grad(1:nEqn)= U_xyz(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(3)) % v(0,BOTTOM) !$acc loop seq do k = 1, self % Nxyz(3) do eq = 1, nEqn - Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(k,BOTTOM) + Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(3)) % v(k,BOTTOM) enddo end do !$acc loop seq @@ -584,7 +589,7 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d f_side % storage(self % faceSide(EBOTTOM)) % Q_aux(eq,i,j) = Q_grad(eq) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side,nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM), dir) + call Face_AdaptGradientsToFace(f_side,nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM), dir) case(4) !$acc loop vector collapse(2) private(Q_grad) @@ -606,11 +611,11 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d case(5) !$acc loop vector collapse(2) private(Q_grad) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Q_grad(1:nEqn) = U_xyz(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(1)) % v(0,TOP) + Q_grad(1:nEqn) = U_xyz(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(3)) % v(0,TOP) !$acc loop seq do k = 1, self % Nxyz(3) do eq = 1, nEqn - Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(k,TOP) + Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(3)) % v(k,TOP) enddo end do !$acc loop seq @@ -618,7 +623,7 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d f_side % storage(self % faceSide(ETOP)) % Q_aux(eq,i,j) = Q_grad(eq) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP), dir) case(6) !$acc loop vector collapse(2) private(Q_grad) @@ -664,20 +669,20 @@ subroutine HexElement_ProlongGradientsToFaces_GL(self, nEqn, f_side, U_xyz, side select case(side) case(1) !$acc loop vector collapse(3) - do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) + do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) do eq = 1, nEqn f_side % storage(self % faceSide(EFRONT)) % Q_aux(eq,i,k) = U_xyz(eq,i,0,k) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT), dir) case(2) !$acc loop vector collapse(3) - do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) + do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) do eq = 1, nEqn f_side % storage(self % faceSide(EBACK)) % Q_aux(eq,i,k) = U_xyz(eq,i,self % Nxyz(2),k) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK), dir) case(3) !$acc loop vector collapse(3) @@ -686,7 +691,7 @@ subroutine HexElement_ProlongGradientsToFaces_GL(self, nEqn, f_side, U_xyz, side f_side % storage(self % faceSide(EBOTTOM)) % Q_aux(eq,i,j) = U_xyz(eq,i,j,0) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side,nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM), dir) + call Face_AdaptGradientsToFace(f_side,nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM), dir) case(4) !$acc loop vector collapse(3) @@ -704,7 +709,7 @@ subroutine HexElement_ProlongGradientsToFaces_GL(self, nEqn, f_side, U_xyz, side f_side % storage(self % faceSide(ETOP)) % Q_aux(eq,i,j) = U_xyz(eq,i,j,self % Nxyz(3)) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP), dir) case(6) !$acc loop vector collapse(3) diff --git a/Solver/src/libs/mesh/HexMesh.f90 b/Solver/src/libs/mesh/HexMesh.f90 index 2e8a628cd..a5eef59e6 100644 --- a/Solver/src/libs/mesh/HexMesh.f90 +++ b/Solver/src/libs/mesh/HexMesh.f90 @@ -982,25 +982,37 @@ subroutine HexMesh_ProlongSolToFaces(self, nEqn) select case ( self %nodeType ) case(1) !Gauss -!$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang collapse(2) present(self) num_gangs(size(self % elements)) vector_length(32) async(1) +#else +!$omp do schedule(runtime) +#endif do eID = 1, size(self % elements) do fID = 1, 6 - call HexElement_ProlongSolToFaces(self % elements(eID), nEqn, self % faces(self % elements(eID) % faceIDs(fID)), fID) + call HexElement_ProlongSolToFaces(self % elements(eID), nEqn, self % faces(self % elements(eID) % faceIDs(fID)), fID) end do ; end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif case(2) !Gauss-Lobatto +#ifdef _OPENACC +!$acc parallel loop gang collapse(2) present(self) num_gangs(size(self % elements)) vector_length(32) async(1) +#else !$omp do schedule(runtime) -!$acc parallel loop gang collapse(2) present(self) num_gangs(size(self % elements)) vector_length(32) async(1) +#endif do eID = 1, size(self % elements) do fID = 1, 6 - call HexElement_ProlongSolToFaces_GL(self % elements(eID), nEqn, self % faces(self % elements(eID) % faceIDs(fID)), fID) + call HexElement_ProlongSolToFaces_GL(self % elements(eID), nEqn, self % faces(self % elements(eID) % faceIDs(fID)), fID) end do ; end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end select @@ -1024,8 +1036,11 @@ subroutine HexMesh_ProlongGradientsToFaces(self, size_element_list, element_list select case ( self %nodeType ) case(1) !Gauss +#ifdef _OPENACC !$acc parallel loop gang num_gangs(size_element_list) collapse(2) present(self, element_list) private(fIDs) async(1) +#else !$omp do schedule(runtime) private(eID) +#endif do iEl = 1, size_element_list do fid = 1,6 eID = element_list(iEl) @@ -1041,47 +1056,68 @@ subroutine HexMesh_ProlongGradientsToFaces(self, size_element_list, element_list call HexElement_ProlongGradientsToFaces(self % elements(eID), nGradEqn, & self % faces(fIDs(fid)), & self % elements(eID) % storage % U_z,fid, 3) - end do ; enddo + end do ; enddo +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif case(2) !Gauss-Lobatto -!$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) +#else +!$omp do schedule(runtime) +#endif do iEl = 1, size_element_list do fID = 1, 6 eID = element_list(iEl) call HexElement_ProlongGradientsToFaces_GL(self % elements(eID), nGradEqn, & self % faces(self % elements(eID) % faceIDs(fID)), & - self % elements(eID) % storage % U_x, fID,1) + self % elements(eID) % storage % U_x, fID,1) end do ; end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif +#ifdef _OPENACC +!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) +#else !$omp do schedule(runtime) -!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) +#endif do iEl = 1, size_element_list do fID = 1, 6 eID = element_list(iEl) call HexElement_ProlongGradientsToFaces_GL(self % elements(eID), nGradEqn, & self % faces(self % elements(eID) % faceIDs(fID)), & - self % elements(eID) % storage % U_y, fID,2) + self % elements(eID) % storage % U_y, fID,2) end do ; end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do - +#endif + +#ifdef _OPENACC +!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) +#else !$omp do schedule(runtime) -!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) +#endif do iEl = 1, size_element_list do fID = 1, 6 eID = element_list(iEl) call HexElement_ProlongGradientsToFaces_GL(self % elements(eID), nGradEqn, & self % faces(self % elements(eID) % faceIDs(fID)), & - self % elements(eID) % storage % U_z, fID,3) + self % elements(eID) % storage % U_z, fID,3) end do ; end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end select end subroutine HexMesh_ProlongGradientsToFaces @@ -1962,6 +1998,7 @@ subroutine HexMesh_CheckIfMeshIs2D(self) xNodesF1(:,nID) = self % nodes(face1Nodes(nID)) % x xNodesF2(:,nID) = self % nodes(face2Nodes(nID)) % x + end do ! ! Compute the delta x vectors @@ -2960,8 +2997,7 @@ subroutine CommunicateMPIFaceMinimumDistance(self) end do call mpi_waitall(MPI_Process % nProcs, sendReq, sendSt, ierr) - - deallocate(hsend, hrecv) + deallocate(hsend, hrecv) #endif end subroutine CommunicateMPIFaceMinimumDistance @@ -3960,7 +3996,7 @@ subroutine GetDiscretizationError(mesh,controlVariables) e => mesh % elements(eID) ) e_aux % globID = e % globID e_aux % Nxyz = e % Nxyz - NDOF = NDOF + product(e % Nxyz + 1) + NDOF = NDOF + product(e % Nxyz + 1) end associate end do @@ -4135,6 +4171,8 @@ subroutine HexMesh_ComputeWallDistances(self,facesList,elementList) else num_of_elems = self % no_of_elements end if + + !$acc data copyin(self, Xwall) if(no_of_wallDOFS .gt. 0) do ii = 1, num_of_elems if ( present(elementList) ) then eID = elementList (ii) @@ -4149,18 +4187,34 @@ subroutine HexMesh_ComputeWallDistances(self,facesList,elementList) endif if( .not. self% IBM% active ) then - do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) - xP = e % geom % x(:,i,j,k) - - minimumDistance = HUGE(1.0_RP) - do fID = 1, no_of_wallDOFS - currentDistance = sum(POW2(xP - Xwall(:,fID))) - minimumDistance = min(minimumDistance, currentDistance) + if (no_of_wallDOFS .ne. 0) then + !$acc data copyin(self % elements(eID), self % elements(eID) % geom, self % elements(eID) % geom % x) & + !$acc& copyout(self % elements(eID) % geom % dWall) + !$acc parallel loop gang collapse(3) present(self, Xwall) & + !$acc& private(xP, currentDistance, minimumDistance) vector_length(128) + do k = 0, self % elements(eID) % Nxyz(3) + do j = 0, self % elements(eID) % Nxyz(2) + do i = 0, self % elements(eID) % Nxyz(1) + xP = self % elements(eID) % geom % x(:,i,j,k) + + minimumDistance = HUGE(1.0_RP) + !$acc loop vector reduction(min:minimumDistance) private(currentDistance) + do fID = 1, no_of_wallDOFS + currentDistance = POW2(xP(1) - Xwall(1,fID)) + & + POW2(xP(2) - Xwall(2,fID)) + & + POW2(xP(3) - Xwall(3,fID)) + minimumDistance = min(minimumDistance, currentDistance) + end do + + self % elements(eID) % geom % dWall(i,j,k) = sqrt(minimumDistance) + end do + end do end do - - e % geom % dWall(i,j,k) = sqrt(minimumDistance) - - end do ; end do ; end do + !$acc end parallel loop + !$acc end data + else + e % geom % dWall = huge(1.0_RP) + end if end if end associate @@ -4187,22 +4241,37 @@ subroutine HexMesh_ComputeWallDistances(self,facesList,elementList) endif if( .not. self% IBM% active ) then - do j = 0, fe % Nf(2) ; do i = 0, fe % Nf(1) - xP = fe % geom % x(:,i,j) - - minimumDistance = HUGE(1.0_RP) - do fID = 1, no_of_wallDOFS - currentDistance = sum(POW2(xP - Xwall(:,fID))) - minimumDistance = min(minimumDistance, currentDistance) + if (no_of_wallDOFS .ne. 0) then + !$acc data copyin(self % faces(eID), self % faces(eID) % geom, self % faces(eID) % geom % x) & + !$acc& copyout(self % faces(eID) % geom % dWall) + !$acc parallel loop gang collapse(2) present(self, Xwall) & + !$acc& private(xP, currentDistance, minimumDistance) vector_length(128) + do j = 0, self % faces(eID) % Nf(2) + do i = 0, self % faces(eID) % Nf(1) + xP = self % faces(eID) % geom % x(:,i,j) + + minimumDistance = HUGE(1.0_RP) + !$acc loop vector reduction(min:minimumDistance) private(currentDistance) + do fID = 1, no_of_wallDOFS + currentDistance = POW2(xP(1) - Xwall(1,fID)) + & + POW2(xP(2) - Xwall(2,fID)) + & + POW2(xP(3) - Xwall(3,fID)) + minimumDistance = min(minimumDistance, currentDistance) + end do + + self % faces(eID) % geom % dWall(i,j) = sqrt(minimumDistance) + end do end do - - fe % geom % dWall(i,j) = sqrt(minimumDistance) - - end do ; end do - end if + !$acc end parallel loop + !$acc end data + else + fe % geom % dWall = huge(1.0_RP) + end if + endif end associate end do + !$acc end data deallocate(Xwall) @@ -5666,13 +5735,19 @@ subroutine HexMesh_ComputeLocalGradientNS(self, set_mu) integer :: eID !-------------------------------------------------- -!$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) present(self) async(1) +#else +!$omp do schedule(runtime) +#endif do eID = 1 , size(self % elements) call HexElement_ComputeLocalGradient(self % elements(eID), NCONS, NGRAD, self % elements(eID) % storage % Q) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do nowait +#endif end subroutine HexMesh_ComputeLocalGradientNS @@ -5686,19 +5761,25 @@ subroutine HexMesh_ComputeLocalGradientiNS(self) integer :: eID, i, j, k !-------------------------------------------------- -!$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) present(self) async(1) +#else +!$omp do schedule(runtime) +#endif do eID = 1 , size(self % elements) - !$acc loop vector collapse(3) + !$acc loop vector collapse(3) do k = 0, self % elements(eID) % Nxyz(3) ; do j = 0, self % elements(eID) % Nxyz(2) ; do i = 0, self % elements(eID) % Nxyz(1) call iNSGradientVariables(NCONS, NGRAD, self % elements(eID) % storage % Q(:,i,j,k), self % elements(eID) % storage % Q_grad_iNS(:,i,j,k)) end do ; end do ; end do call HexElement_ComputeLocalGradient(self % elements(eID), NCONS, NGRAD, self % elements(eID) % storage % Q_grad_iNS) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do nowait +#endif end subroutine HexMesh_ComputeLocalGradientiNS #endif @@ -5713,20 +5794,25 @@ subroutine HexMesh_ComputeLocalGradientCH(self, set_mu) integer :: eID, i, j, k !-------------------------------------------------- -!$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) present(self) copyin(set_mu) async(1) +#else +!$omp do schedule(runtime) +#endif do eID = 1 , size(self % elements) - !$acc loop vector collapse(3) + !$acc loop vector collapse(3) do k = 0, self % elements(eID) % Nxyz(3) ; do j = 0, self % elements(eID) % Nxyz(2) ; do i = 0, self % elements(eID) % Nxyz(1) call chGradientVariables(NCOMP, NCOMP, self % elements(eID) % storage % Q(1:IMC,i,j,k), self % elements(eID) % storage % Q_grad_CH(1:IMC,i,j,k)) - !if ( set_mu ) self % elements(eID) % storage % Q_grad_CH(IGMU,i,j,k) = self % elements(eID) % storage % mu(1,i,j,k) end do ; end do ; end do call HexElement_ComputeLocalGradient(self % elements(eID), NCOMP, NCOMP, self % elements(eID) % storage % Q_grad_CH) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do nowait +#endif end subroutine HexMesh_ComputeLocalGradientCH @@ -5740,23 +5826,27 @@ subroutine HexMesh_ComputeLocalGradientMU(self, set_mu) integer :: eID, i, j, k !-------------------------------------------------- -!$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang vector_length(128) present(self) copyin(set_mu) async(1) +#else +!$omp do schedule(runtime) +#endif do eID = 1 , size(self % elements) - !$acc loop vector collapse(3) + !$acc loop vector collapse(3) do k = 0, self % elements(eID) % Nxyz(3) ; do j = 0, self % elements(eID) % Nxyz(2) ; do i = 0, self % elements(eID) % Nxyz(1) call mGradientVariables(NCONS, NGRAD, self % elements(eID) % storage % Q(:,i,j,k), self % elements(eID) % storage % Q_grad_mu(:,i,j,k), self % elements(eID) % storage % rho(i,j,k)) - !if ( set_mu == .true.) then ! This is not working - weird - above it works self % elements(eID) % storage % Q_grad_mu(IGMU,i,j,k) = self % elements(eID) % storage % mu(1,i,j,k) - !end if end do ; end do ; end do call HexElement_ComputeLocalGradient(self % elements(eID), NCONS, NGRAD, self % elements(eID) % storage % Q_grad_mu) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do nowait +#endif end subroutine HexMesh_ComputeLocalGradientMU #endif -END MODULE HexMeshClass +END MODULE HexMeshClass \ No newline at end of file diff --git a/Solver/src/libs/mesh/IBMClass.f90 b/Solver/src/libs/mesh/IBMClass.f90 index 03338be5f..e21dc8020 100644 --- a/Solver/src/libs/mesh/IBMClass.f90 +++ b/Solver/src/libs/mesh/IBMClass.f90 @@ -1072,16 +1072,19 @@ subroutine IBM_BandPoint_state( this, elements, STLNum, gradients ) end if !$acc wait +#ifdef _OPENACC + !$acc parallel loop present(this, elements) +#else !$omp parallel !$omp do schedule(runtime) private(i,j,k,eID) - !$acc parallel loop present(this, elements) +#endif do n = 1, this% BandRegion(STLNum)% NumOfObjs if( this% BandRegion(STLNum)% x(n)% partition .eq. MPI_Process% rank ) then i = this% BandRegion(STLNum)% x(n)% local_Position(1) j = this% BandRegion(STLNum)% x(n)% local_Position(2) k = this% BandRegion(STLNum)% x(n)% local_Position(3) eID = this% BandRegion(STLNum)% x(n)% element_index - this% BandRegion(STLNum)% Q(:,n) = elements(eID)% storage% Q(:,i,j,k) + this% BandRegion(STLNum)% Q(:,n) = elements(eID)% storage% Q(:,i,j,k) if( gradients ) then this% BandRegion(STLNum)% U_x(:,n) = elements(eID)% storage% U_x(:,i,j,k) this% BandRegion(STLNum)% U_y(:,n) = elements(eID)% storage% U_y(:,i,j,k) @@ -1089,9 +1092,12 @@ subroutine IBM_BandPoint_state( this, elements, STLNum, gradients ) end if end if end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do !$omp end parallel +#endif !$acc wait !Update CPU variables @@ -2092,19 +2098,24 @@ subroutine IBM_SemiImplicitCorrection( this, elements, dt ) !$acc wait if( this % Wallfunction ) call this % GetBandRegionStates( elements ) -!$omp parallel - !$omp do schedule(runtime) private(i,j,k) +#ifdef _OPENACC !$acc parallel loop gang present(this, elements) +#else +!$omp parallel +!$omp do schedule(runtime) private(i,j,k) +#endif do eID = 1, SIZE( elements ) - !$acc loop vector collapse(3) + !$acc loop vector collapse(3) do i = 0, elements(eID) % Nxyz(1); do j = 0, elements(eID) % Nxyz(2); do k = 0, elements(eID) % Nxyz(3) - if( elements(eID) % isInsideBody(i,j,k) ) then - call IBM_GetSemiImplicitStep(this, eID, 0.5_RP*dt, elements(eID) % storage % Q(:,i,j,k) ) - end if + if( elements(eID) % isInsideBody(i,j,k) ) then + call IBM_GetSemiImplicitStep(this, eID, 0.5_RP*dt, elements(eID) % storage % Q(:,i,j,k) ) + end if end do; end do; end do end do +#ifdef _OPENACC !$acc end parallel loop -!$omp end do +#else +!$omp end do if( this% Wallfunction ) then !$omp do schedule(runtime) private(i,j,k) @@ -2119,7 +2130,8 @@ subroutine IBM_SemiImplicitCorrection( this, elements, dt ) end do !$omp end do end if -!$omp end parallel +!$omp end parallel +#endif end subroutine IBM_SemiImplicitCorrection subroutine IBM_GetBandRegionStates( this, elements ) @@ -3652,4 +3664,4 @@ subroutine IBM_CreateDeviceData(self) end subroutine IBM_CreateDeviceData -end module IBMClass +end module IBMClass \ No newline at end of file diff --git a/Solver/src/libs/mesh/StorageClass.f90 b/Solver/src/libs/mesh/StorageClass.f90 index 182541918..161e54e00 100644 --- a/Solver/src/libs/mesh/StorageClass.f90 +++ b/Solver/src/libs/mesh/StorageClass.f90 @@ -1293,7 +1293,7 @@ end subroutine ElementStorage_SetStorageToCH_mu ! Interpolate solution to another element storage ! this % Q -> other % Q ! ----------------------------------------------- - impure elemental subroutine ElementStorage_InterpolateSolution(this,other,nodes,with_gradients) + subroutine ElementStorage_InterpolateSolution(this,other,nodes,with_gradients) implicit none !-arguments---------------------------------------------- class(ElementStorage_t), intent(in) :: this diff --git a/Solver/src/libs/monitors/Monitors.f90 b/Solver/src/libs/monitors/Monitors.f90 index 641be3051..eaa4f1cff 100644 --- a/Solver/src/libs/monitors/Monitors.f90 +++ b/Solver/src/libs/monitors/Monitors.f90 @@ -27,10 +27,17 @@ module MonitorsClass ! type Monitor_t character(len=LINE_LENGTH) :: solution_file + character(len=LINE_LENGTH) :: probes_solution_file = "" integer :: no_of_probes integer :: no_of_surfaceMonitors integer :: no_of_volumeMonitors integer :: no_of_loadBalancingMonitors + integer :: no_of_fileProbes = 0 + character(len=LINE_LENGTH) :: probesFileName = "" + character(len=STR_LEN_MONITORS), allocatable :: probesVariables(:) + real(kind=RP) :: probeFileSaveTimestep = 0.0_RP + character(len=8) :: probeFileOutputFormat = "ASCII" + real(kind=RP) :: fp_lastSavedTime = -huge(0.0_RP) integer :: bufferLine integer , allocatable :: iter(:) integer :: dt_restriction @@ -55,6 +62,7 @@ module MonitorsClass procedure :: WriteValues => Monitor_WriteValues procedure :: UpdateValues => Monitor_UpdateValues procedure :: WriteToFile => Monitor_WriteToFile + procedure :: WriteProbesFileSummary => Monitor_WriteProbesFileSummary procedure :: destruct => Monitor_Destruct procedure :: copy => Monitor_Assign generic :: assignment(=) => copy @@ -66,9 +74,10 @@ module MonitorsClass ! !/////////////////////////////////////////////////////////////////////////////////////// ! - subroutine Monitors_Construct( Monitors, mesh, controlVariables ) + subroutine Monitors_Construct( Monitors, mesh, controlVariables ) use FTValueDictionaryClass use mainKeywordsModule + use MPI_Process_Info implicit none class(Monitor_t) :: Monitors class(HexMesh), intent(in) :: mesh @@ -82,9 +91,19 @@ subroutine Monitors_Construct( Monitors, mesh, controlVariables ) integer :: fID , io integer :: i character(len=STR_LEN_MONITORS) :: line - character(len=STR_LEN_MONITORS) :: solution_file + character(len=STR_LEN_MONITORS) :: solution_file logical, save :: FirstCall = .TRUE. logical :: saveGradients + character(len=LINE_LENGTH) :: probesFileName + character(len=LINE_LENGTH) :: probesVariablesLine + character(len=STR_LEN_MONITORS), allocatable :: probesVariables(:) + integer :: no_of_fileProbes + integer :: no_of_probesVariables + real(kind=RP) :: probeFileSaveTimestep + character(len=8) :: probeFileOutputFormat + character(len=LINE_LENGTH) :: probes_solution_file + character(len=LINE_LENGTH) :: probes_dir + integer :: last_slash ! ! Setup the buffer ! ---------------- @@ -106,8 +125,24 @@ subroutine Monitors_Construct( Monitors, mesh, controlVariables ) solution_file = trim(getFileName(solution_file)) Monitors % solution_file = trim(solution_file) ! +! Build the probes output subdirectory: /probes/ +! ---------------------------------------------------------- + last_slash = index(trim(solution_file), '/', back=.true.) + if (last_slash .gt. 0) then + probes_dir = trim(solution_file(1:last_slash)) // "probes" + probes_solution_file = trim(solution_file(1:last_slash)) // "probes/" // trim(solution_file(last_slash+1:)) + else + probes_dir = "probes" + probes_solution_file = "probes/" // trim(solution_file) + end if + if (MPI_Process % isRoot) then + call execute_command_line('mkdir -p "' // trim(probes_dir) // '"', wait=.true.) + end if + Monitors % probes_solution_file = trim(probes_solution_file) +! ! Search in case file for probes, surface monitors, and volume monitors ! --------------------------------------------------------------------- + no_of_fileProbes = 0 if (mesh % child) then ! Return doing nothing if this is a child mesh Monitors % no_of_probes = 0 Monitors % no_of_surfaceMonitors = 0 @@ -115,6 +150,25 @@ subroutine Monitors_Construct( Monitors, mesh, controlVariables ) Monitors % no_of_loadBalancingMonitors = 0 else call getNoOfMonitors( Monitors % no_of_probes, Monitors % no_of_surfaceMonitors, Monitors % no_of_volumeMonitors, Monitors % no_of_loadBalancingMonitors ) +! +! Check for an additional probes definition file (one probe per +! line: "x y z variable1 [variable2 ...]"), allowing several +! variables to be sampled and saved for the same probe location +! --------------------------------------------------------------- +#ifdef FLOW + call readProbesFileBlock( probesFileName, probesVariablesLine, no_of_probesVariables, probeFileSaveTimestep, probeFileOutputFormat ) + + if ( len_trim(probesFileName) .gt. 0 ) then + call countProbesInFile( trim(probesFileName), no_of_fileProbes ) + + if ( len_trim(probesVariablesLine) .gt. 0 ) then + call splitIntoTokens( trim(probesVariablesLine), probesVariables, no_of_probesVariables ) + else + write(STD_OUT,*) "Error: 'variables' must be specified inside the '#define probe file' block." + stop + end if + end if +#endif end if ! ! Initialize the Monitors class in the GPU @@ -142,11 +196,31 @@ subroutine Monitors_Construct( Monitors, mesh, controlVariables ) end do #ifdef FLOW - allocate ( Monitors % probes ( Monitors % no_of_probes ) ) + allocate ( Monitors % probes ( Monitors % no_of_probes + no_of_fileProbes ) ) !$acc update device(Monitors) do i = 1 , Monitors % no_of_probes - call Monitors % probes(i) % Initialization ( mesh , i, solution_file , FirstCall ) + call Monitors % probes(i) % Initialization ( mesh , i, probes_solution_file , FirstCall ) end do + + if ( no_of_fileProbes .gt. 0 ) then + call InitializeProbesFromFile( trim(probesFileName), Monitors % probes, Monitors % no_of_probes, & + mesh, probes_solution_file, FirstCall, probesVariables, probeFileSaveTimestep, & + probeFileOutputFormat ) + Monitors % probesFileName = trim(probesFileName) + Monitors % probeFileSaveTimestep = probeFileSaveTimestep + Monitors % probeFileOutputFormat = probeFileOutputFormat + Monitors % fp_lastSavedTime = -huge(0.0_RP) + allocate( Monitors % probesVariables(size(probesVariables)) ) + Monitors % probesVariables = probesVariables +#ifdef HAS_HDF5 + if ( trim(probeFileOutputFormat) .eq. "HDF5" ) then + call Monitor_InitFileProbesHDF5( Monitors, no_of_fileProbes ) + end if +#endif + end if + + Monitors % no_of_fileProbes = no_of_fileProbes + Monitors % no_of_probes = Monitors % no_of_probes + no_of_fileProbes #endif #if defined(NAVIERSTOKES) || defined(INCNS) @@ -210,9 +284,9 @@ subroutine Monitor_WriteLabel ( self ) #ifdef FLOW ! -! Write probes labels -! ------------------- - do i = 1 , self % no_of_probes +! Write probes labels (file-based probes excluded for readability) +! --------------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes call self % probes(i) % WriteLabel end do #endif @@ -282,9 +356,9 @@ subroutine Monitor_WriteUnderlines( self ) #ifdef FLOW ! -! Print dashes for probes -! ----------------------- - do i = 1 , self % no_of_probes +! Print dashes for probes (file-based probes excluded for readability) +! ---------------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes if ( self % probes(i) % active ) then write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % probes(i) % monitorName ) + 2 ) ) end if @@ -349,9 +423,9 @@ subroutine Monitor_WriteValues ( self ) #ifdef FLOW ! -! Print probes -! ------------ - do i = 1 , self % no_of_probes +! Print probes (file-based probes excluded for readability) +! ---------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes call self % probes(i) % WriteValues ( self % bufferLine ) end do #endif @@ -433,11 +507,20 @@ subroutine Monitor_UpdateValues ( self, mesh, t , iter, maxResiduals, Autosave, #ifdef FLOW ! -! Update probes -! ------------- - do i = 1 , self % no_of_probes +! Update standard probes (GPU path with per-probe MPI) +! ----------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes call self % probes(i) % Update( mesh , self % bufferLine ) end do +! +! Update file probes: compute into slot 1, then flush immediately. +! File probes use a size-1 values buffer to avoid large memory +! allocations when the number of probes is O(1e6). +! ---------------------------------------------------------- + if ( self % no_of_fileProbes .gt. 0 ) then + call Monitor_UpdateFileProbes( self, mesh, 1 ) + call Monitor_FlushFileProbesNow( self, t, iter ) + end if #endif #if defined(NAVIERSTOKES) || defined(INCNS) @@ -500,12 +583,12 @@ subroutine Monitor_WriteToFile ( self , mesh, force) end do #ifdef FLOW - do i = 1 , self % no_of_probes + do i = 1 , self % no_of_probes - self % no_of_fileProbes call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) end do #endif - -#if defined(NAVIERSTOKES) || defined(INCNS) + +#if defined(NAVIERSTOKES) || defined(INCNS) do i = 1 , self % no_of_surfaceMonitors call self % surfaceMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) end do @@ -541,8 +624,8 @@ subroutine Monitor_WriteToFile ( self , mesh, force) end do #ifdef FLOW - do i = 1 , self % no_of_probes - call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + do i = 1 , self % no_of_probes - self % no_of_fileProbes + call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) end do #endif @@ -560,7 +643,47 @@ subroutine Monitor_WriteToFile ( self , mesh, force) end if end subroutine Monitor_WriteToFile - + + subroutine Monitor_WriteProbesFileSummary ( self ) +! +! ******************************************************************** +! Prints a startup-log summary of the bulk probes-file monitor, +! shown instead of including the file-based probes in the +! per-iteration screen log. +! ******************************************************************** +! + use Headers + use MPI_Process_Info + implicit none + class(Monitor_t) :: self +! +! --------------- +! Local variables +! --------------- +! + integer :: j + + if ( .not. MPI_Process % isRoot ) return + if ( self % no_of_fileProbes .le. 0 ) return + + write(STD_OUT,'(/)') + call Section_Header("Probes file monitor") + write(STD_OUT,'(/)') + + write(STD_OUT,'(30X,A,A28,A)') "->" , "File: " , trim(self % probesFileName) + write(STD_OUT,'(30X,A,A28,I10)') "->" , "Number of probes: " , self % no_of_fileProbes + write(STD_OUT,'(30X,A,A28)',advance="no") "->" , "Variables: " + do j = 1 , size(self % probesVariables) + write(STD_OUT,'(A)',advance="no") trim(self % probesVariables(j)) // " " + end do + write(STD_OUT,*) + if ( self % probeFileSaveTimestep .gt. 0.0_RP ) then + write(STD_OUT,'(30X,A,A28,ES14.6)') "->" , "Save timestep: " , self % probeFileSaveTimestep + end if + write(STD_OUT,'(30X,A,A28,A)') "->" , "Output format: " , trim(self % probeFileOutputFormat) + + end subroutine Monitor_WriteProbesFileSummary + subroutine Monitor_Destruct (self) implicit none class(Monitor_t) :: self @@ -600,10 +723,21 @@ impure elemental subroutine Monitor_Assign ( to, from ) !------------------------------------------------ to % solution_file = from % solution_file + to % probes_solution_file = from % probes_solution_file to % no_of_probes = from % no_of_probes to % no_of_surfaceMonitors = from % no_of_surfaceMonitors to % no_of_volumeMonitors = from % no_of_volumeMonitors to % no_of_loadBalancingMonitors = from % no_of_loadBalancingMonitors + to % no_of_fileProbes = from % no_of_fileProbes + to % probesFileName = from % probesFileName + to % probeFileSaveTimestep = from % probeFileSaveTimestep + to % probeFileOutputFormat = from % probeFileOutputFormat + to % fp_lastSavedTime = from % fp_lastSavedTime + if ( allocated(from % probesVariables) ) then + safedeallocate ( to % probesVariables ) + allocate ( to % probesVariables ( size(from % probesVariables) ) ) + to % probesVariables = from % probesVariables + end if to % bufferLine = from % bufferLine safedeallocate ( to % iter ) @@ -716,7 +850,12 @@ subroutine getNoOfMonitors(no_of_probes, no_of_surfaceMonitors, no_of_volumeMoni ! --------- line = getSquashedLine( line ) - if ( index ( line , '#defineprobe' ) .gt. 0 ) then + if ( index ( line , '#defineprobefile' ) .gt. 0 ) then +! +! The probe-file block is not an individual probe definition +! ----------------------------------------------------------- + + elseif ( index ( line , '#defineprobe' ) .gt. 0 ) then no_of_probes = no_of_probes + 1 elseif ( index ( line , '#definesurfacemonitor' ) .gt. 0 ) then @@ -740,7 +879,625 @@ subroutine getNoOfMonitors(no_of_probes, no_of_surfaceMonitors, no_of_volumeMoni end subroutine getNoOfMonitors +! +!/////////////////////////////////////////////////////////////////////////////////// +! +! Probes-from-file auxiliary routines +! +! The probes file is a plain text file. Blank lines and lines starting +! with "#" are ignored. Every remaining line holds the coordinates of +! one probe: +! +! x y z +! +! The list of variables to sample (shared by every probe in the file) +! is given through the "probes file variables" control-file keyword. +! +! A separate output file ".probe_.probe" is created for +! each probe, with one column per variable, and one row written per +! saved time step. +! +!/////////////////////////////////////////////////////////////////////////////////// +! + subroutine readProbesFileBlock(fileName, variablesLine, no_of_probesVariables, saveTimestep, outputFormat) +! +! ****************************************************************** +! Reads the "#define probe file ... #end" block from the case +! file, if present: +! +! #define probe file +! file = Probe.dat +! variables = u +! probe save timestep = 1.0E-3 +! output format = HDF5 +! #end +! +! Note: this is a hand-rolled parser (rather than readValueInRegion) +! because readValueInRegion lower-cases every line it reads, which +! would corrupt a case-sensitive file path. +! ****************************************************************** +! + use ParamfileRegions, only: getSquashedLine + implicit none + character(len=LINE_LENGTH), intent(out) :: fileName + character(len=LINE_LENGTH), intent(out) :: variablesLine + integer, intent(out) :: no_of_probesVariables + real(kind=RP), intent(out) :: saveTimestep + character(len=8), intent(out) :: outputFormat +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: paramFile + character(len=LINE_LENGTH) :: line, squashed, valStr + integer :: fID, io, position + + logical :: inside + + fileName = "" + variablesLine = "" + no_of_probesVariables = 0 + saveTimestep = 0.0_RP + outputFormat = "ASCII" + inside = .false. + + call get_command_argument(1, paramFile) + + open ( newunit = fID , file = trim(paramFile) , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + squashed = getSquashedLine(line) + + if ( squashed .eq. getSquashedLine("#define probe file") ) then + inside = .true. + cycle + elseif ( squashed .eq. getSquashedLine("#end") ) then + inside = .false. + cycle + end if + + if ( .not. inside ) cycle +! +! Strip a trailing comment, keeping the value's original case +! ------------------------------------------------------------- + position = index(line , '!') + if ( position .gt. 0 ) line = line(1:position-1) + + position = max( index(line,'='), index(line,':') ) + if ( position .eq. 0 ) cycle + + if ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("file") ) then + fileName = adjustl( removeQuotes( line(position+1:) ) ) + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("variables") ) then + variablesLine = adjustl( removeQuotes( line(position+1:) ) ) + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("probe save timestep") ) then + valStr = adjustl( removeQuotes( line(position+1:) ) ) + read( valStr , * ) saveTimestep + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("output format") ) then + valStr = adjustl( removeQuotes( line(position+1:) ) ) + valStr = adjustl( getSquashedLine(valStr) ) + if ( index(valStr, "HDF5") .gt. 0 ) then + outputFormat = "HDF5" + else + outputFormat = "ASCII" + end if + end if + end do + + close(fID) + + end subroutine readProbesFileBlock + + function removeQuotes(str) result(res) + implicit none + character(len=*), intent(in) :: str + character(len=LINE_LENGTH) :: res + character(len=LINE_LENGTH) :: auxstr + integer :: i, j + + auxstr = trim(adjustl(str)) + res = "" + j = 0 + + do i = 1 , len_trim(auxstr) + if ( auxstr(i:i) .eq. '"' .or. auxstr(i:i) .eq. "'" ) cycle + j = j + 1 + res(j:j) = auxstr(i:i) + end do + + end function removeQuotes + + subroutine countProbesInFile(fileName, n) + implicit none + character(len=*), intent(in) :: fileName + integer, intent(out) :: n +! +! --------------- +! Local variables +! --------------- +! + integer :: fID, io + character(len=LINE_LENGTH) :: line + + n = 0 + open ( newunit = fID , file = fileName , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + line = adjustl(line) + if ( len_trim(line) .eq. 0 ) cycle + if ( line(1:1) .eq. '#' ) cycle + + n = n + 1 + end do + + close(fID) + + end subroutine countProbesInFile + + subroutine splitIntoTokens(line, tokens, n) + implicit none + character(len=*), intent(in) :: line + character(len=STR_LEN_MONITORS), allocatable, intent(out) :: tokens(:) + integer, intent(out) :: n +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: auxline + character(len=STR_LEN_MONITORS) :: buffer(64) + integer :: pos + + auxline = adjustl(line) + n = 0 + + do while ( len_trim(auxline) .gt. 0 ) + pos = index(trim(auxline), " ") + n = n + 1 + + if ( pos .eq. 0 ) then + buffer(n) = trim(auxline) + auxline = "" + else + buffer(n) = auxline(1:pos-1) + auxline = adjustl(auxline(pos+1:)) + end if + end do + + allocate( tokens(n) ) + tokens(1:n) = buffer(1:n) + + end subroutine splitIntoTokens + +#ifdef FLOW + subroutine Monitor_FlushFileProbesNow(self, t_now, iter_now) +! +! Writes the current (single-slot) file-probe values to disk, +! applying the probeFileSaveTimestep filter. Called every timestep +! from Monitor_UpdateValues after Monitor_UpdateFileProbes. +! + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + real(kind=RP), intent(in) :: t_now + integer, intent(in) :: iter_now +! +! --------------- +! Local variables +! --------------- +! + integer :: i + integer :: iter_arr(1) + real(kind=RP) :: t_arr(1) + + iter_arr(1) = iter_now + t_arr(1) = t_now + +#ifdef HAS_HDF5 + if ( trim(self % probeFileOutputFormat) .eq. "HDF5" ) then + call Monitor_WriteFileProbesHDF5( self, iter_arr, t_arr, 1 ) + else +#endif + !$omp parallel do schedule(dynamic,16) default(shared) + do i = self % no_of_probes - self % no_of_fileProbes + 1, self % no_of_probes + call self % probes(i) % WriteToFile( iter_arr, t_arr, 1 ) + end do + !$omp end parallel do +#ifdef HAS_HDF5 + end if +#endif + + end subroutine Monitor_FlushFileProbesNow + + subroutine Monitor_UpdateFileProbes(self, mesh, bufferPos) +! +! Evaluates all file-probes using OpenMP threads (CPU path, no OpenACC), +! then performs a single MPI_Allreduce(SUM) to share results across ranks. +! This replaces N_fileProbes individual MPI_Bcast calls with one collective. +! + use MPI_Process_Info +#ifdef _HAS_MPI_ + use mpi +#endif + implicit none + class(Monitor_t), intent(inout) :: self + class(HexMesh), intent(in) :: mesh + integer, intent(in) :: bufferPos +! +! --------------- +! Local variables +! --------------- +! + integer :: i, v, j, nfp, nv, fp_offset, ierr + real(kind=RP), allocatable :: buf(:) + + nfp = self % no_of_fileProbes + nv = size(self % probesVariables) + fp_offset = self % no_of_probes - nfp + + ! Parallel CPU computation — each probe writes to its own values(:,bufferPos). + ! Non-owning ranks store 0 so MPI_Allreduce(SUM) gives the correct result. + !$omp parallel do schedule(dynamic,16) default(shared) + do i = fp_offset + 1, self % no_of_probes + call self % probes(i) % ComputeLocal(mesh, bufferPos) + end do + !$omp end parallel do + +#ifdef _HAS_MPI_ + if ( MPI_Process % doMPIAction ) then + ! Pack all file-probe values into a flat buffer + allocate( buf(nfp * nv) ) + j = 0 + do i = fp_offset + 1, self % no_of_probes + do v = 1, nv + j = j + 1 + buf(j) = self % probes(i) % values(v, bufferPos) + end do + end do + + ! Single collective instead of nfp individual Bcasts + call MPI_Allreduce(MPI_IN_PLACE, buf, nfp * nv, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_WORLD, ierr) + + ! Unpack + j = 0 + do i = fp_offset + 1, self % no_of_probes + do v = 1, nv + j = j + 1 + self % probes(i) % values(v, bufferPos) = buf(j) + end do + end do + deallocate(buf) + end if +#endif + + end subroutine Monitor_UpdateFileProbes + + subroutine InitializeProbesFromFile(fileName, probes, offset, mesh, solution_file, FirstCall, variables, saveTimestep, outputFormat) + implicit none + character(len=*), intent(in) :: fileName + class(Probe_t), intent(inout) :: probes(:) + integer, intent(in) :: offset + class(HexMesh), intent(in) :: mesh + character(len=*), intent(in) :: solution_file + logical, intent(in) :: FirstCall + character(len=*), intent(in) :: variables(:) + real(kind=RP), intent(in) :: saveTimestep + character(len=*), intent(in) :: outputFormat +! +! --------------- +! Local variables +! --------------- +! + integer :: fID, io, idx, nTok + character(len=LINE_LENGTH) :: line + real(kind=RP) :: x(NDIM) + character(len=STR_LEN_MONITORS), allocatable :: tokens(:) + character(len=STR_LEN_MONITORS) :: pname + + idx = offset + open ( newunit = fID , file = fileName , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + line = adjustl(line) + if ( len_trim(line) .eq. 0 ) cycle + if ( line(1:1) .eq. '#' ) cycle + + call splitIntoTokens(line, tokens, nTok) + + if ( nTok .lt. 3 ) then + deallocate(tokens) + cycle + end if + + read(tokens(1),*) x(1) + read(tokens(2),*) x(2) + read(tokens(3),*) x(3) + + idx = idx + 1 + write(pname,'(A,I0)') "probe_", idx + + call probes(idx) % Initialization( mesh, idx, solution_file, FirstCall, & + x_in = x, variables_in = variables, name_in = trim(pname), & + isFileProbe_in = .true., outputFormat_in = trim(outputFormat) ) + probes(idx) % saveTimestep = saveTimestep + + deallocate(tokens) + end do + + close(fID) + + end subroutine InitializeProbesFromFile + +! ============================================================ +! HDF5 collective I/O for file-based probes +! Compiled only when HAS_HDF5 is defined. +! ============================================================ + +#ifdef HAS_HDF5 + subroutine Monitor_InitFileProbesHDF5(self, no_of_fileProbes) +! +! Creates the HDF5 output file for bulk file-probes with +! the following structure: +! +! /coordinates (3, nProbes) — fixed, written once +! /time (extendible) — one value per saved step +! /iteration (extendible) — one value per saved step +! / (nProbes, ext) — one row per saved step +! + use HDF5 + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + integer, intent(in) :: no_of_fileProbes +! +! --------------- +! Local variables +! --------------- +! + integer(HID_T) :: file_id, dset_id, dspace_id, dcpl_id + integer(HSIZE_T) :: dims2(2), maxdims2(2), chunk2(2) + integer(HSIZE_T) :: dims1(1), maxdims1(1), chunk1(1) + integer :: iError, j, v, fp_offset, nv + character(len=LINE_LENGTH) :: fname + real(kind=RP), allocatable :: coords(:,:) + + if ( .not. MPI_Process % isRoot ) return + + nv = size(self % probesVariables) + fp_offset = self % no_of_probes ! probes are allocated 1..no_of_probes+no_of_fileProbes + ! but at this point no_of_probes has not yet been bumped + + write(fname,'(A,A)') trim(self % probes_solution_file), ".probes.h5" + + call h5open_f(iError) + call h5fcreate_f(trim(fname), H5F_ACC_TRUNC_F, file_id, iError) + + ! /coordinates (3, nProbes) — fixed at creation + allocate( coords(3, no_of_fileProbes) ) + do j = 1, no_of_fileProbes + coords(:, j) = self % probes(fp_offset + j) % x + end do + dims2 = [ int(3, HSIZE_T), int(no_of_fileProbes, HSIZE_T) ] + call h5screate_simple_f(2, dims2, dspace_id, iError) + call h5dcreate_f(file_id, "coordinates", H5T_NATIVE_DOUBLE, dspace_id, dset_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, coords, dims2, iError) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + deallocate(coords) + + ! /time (extendible 1-D) + dims1(1) = 0 + maxdims1(1) = H5S_UNLIMITED_F + chunk1(1) = int(max(BUFFER_SIZE, 1), HSIZE_T) + call h5screate_simple_f(1, dims1, dspace_id, iError, maxdims1) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 1, chunk1, iError) + call h5dcreate_f(file_id, "time", H5T_NATIVE_DOUBLE, dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + + ! /iteration (extendible 1-D) + call h5screate_simple_f(1, dims1, dspace_id, iError, maxdims1) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 1, chunk1, iError) + call h5dcreate_f(file_id, "iteration", H5T_NATIVE_INTEGER, dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + + ! / (extendible × nProbes) — h5ls shows {nProbes, Inf} + ! chunk2(1)=1: one time step per chunk avoids pre-allocation waste when + ! the save-timestep filter makes n_write << BUFFER_SIZE per flush. + dims2(1) = 0 + dims2(2) = int(no_of_fileProbes, HSIZE_T) + maxdims2(1) = H5S_UNLIMITED_F + maxdims2(2) = int(no_of_fileProbes, HSIZE_T) + chunk2(1) = int(1, HSIZE_T) + chunk2(2) = int(no_of_fileProbes, HSIZE_T) + + do v = 1, nv + call h5screate_simple_f(2, dims2, dspace_id, iError, maxdims2) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 2, chunk2, iError) + call h5dcreate_f(file_id, trim(self % probesVariables(v)), H5T_NATIVE_DOUBLE, & + dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + end do + + call h5fclose_f(file_id, iError) + call h5close_f(iError) + + end subroutine Monitor_InitFileProbesHDF5 + + subroutine Monitor_WriteFileProbesHDF5(self, iter, t, no_of_lines) +! +! Appends one buffer of file-probe data to the HDF5 output file. +! Only lines that satisfy the saveTimestep filter are written. +! Called from Monitor_WriteToFile in place of the per-probe ASCII loop. +! + use HDF5 + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + integer, intent(in) :: iter(:) + real(kind=RP), intent(in) :: t(:) + integer, intent(in) :: no_of_lines +! +! --------------- +! Local variables +! --------------- +! + integer(HID_T) :: file_id, dset_id, dspace_id, mspace_id + integer(HSIZE_T) :: cur1(1), max1(1), off1(1), cnt1(1) + integer(HSIZE_T) :: new1(1) + integer(HSIZE_T) :: cur2(2), max2(2), off2(2), cnt2(2) + integer(HSIZE_T) :: new2(2) + integer :: iError, i, j, k, v, nfp, nv, fp_offset, n_write + character(len=LINE_LENGTH) :: fname + logical, allocatable :: wmask(:) + integer, allocatable :: ibuf(:) + real(kind=RP), allocatable :: rbuf(:), vbuf(:) + + if ( .not. MPI_Process % isRoot ) return + + nfp = self % no_of_fileProbes + nv = size(self % probesVariables) + fp_offset = self % no_of_probes - nfp + + ! Build write mask (apply saveTimestep filter) + allocate( wmask(no_of_lines) ) + wmask = .false. + do i = 1, no_of_lines + if ( self % probeFileSaveTimestep .gt. 0.0_RP ) then + if ( t(i) .lt. self % fp_lastSavedTime + self % probeFileSaveTimestep ) cycle + end if + wmask(i) = .true. + self % fp_lastSavedTime = t(i) + end do + n_write = count(wmask) + if ( n_write .eq. 0 ) then + deallocate(wmask) + return + end if + + write(fname,'(A,A)') trim(self % probes_solution_file), ".probes.h5" + + call h5open_f(iError) + call h5fopen_f(trim(fname), H5F_ACC_RDWR_F, file_id, iError) + + ! Query current extent of /time to get append offset + call h5dopen_f(file_id, "time", dset_id, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sget_simple_extent_dims_f(dspace_id, cur1, max1, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + ! cur1(1) = number of time steps already written + + cnt1(1) = int(n_write, HSIZE_T) + off1(1) = cur1(1) + new1(1) = cur1(1) + cnt1(1) + + ! Pack filtered iteration values + allocate( ibuf(n_write) ) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + ibuf(j) = iter(i) + end do + + call h5dopen_f(file_id, "iteration", dset_id, iError) + call h5dextend_f(dset_id, new1, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off1, cnt1, iError) + call h5screate_simple_f(1, cnt1, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_INTEGER, ibuf, cnt1, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + deallocate(ibuf) + + ! Pack filtered time values + allocate( rbuf(n_write) ) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + rbuf(j) = t(i) + end do + + call h5dopen_f(file_id, "time", dset_id, iError) + call h5dextend_f(dset_id, new1, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off1, cnt1, iError) + call h5screate_simple_f(1, cnt1, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, rbuf, cnt1, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + deallocate(rbuf) + + ! Write each variable: one extend + one write per flush (avoids chunk waste). + ! vbuf2(nfp, n_write) packs all filtered timesteps into a contiguous 2D block. + off2(1) = cur1(1) ! append after existing time steps + off2(2) = int(0, HSIZE_T) + cnt2(1) = int(n_write, HSIZE_T) + cnt2(2) = int(nfp, HSIZE_T) + new2(1) = cur1(1) + int(n_write, HSIZE_T) + new2(2) = int(nfp, HSIZE_T) + + allocate( vbuf(nfp * n_write) ) + + do v = 1, nv + call h5dopen_f(file_id, trim(self % probesVariables(v)), dset_id, iError) + + ! Pack into column-major 2D layout vbuf(n_write, nfp): + ! time index j varies fastest (Fortran column-major with cnt2=[n_write,nfp]) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + do k = 1, nfp + vbuf( j + (k-1)*n_write ) = self % probes(fp_offset + k) % values(v, i) + end do + end do + + call h5dextend_f(dset_id, new2, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off2, cnt2, iError) + call h5screate_simple_f(2, cnt2, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, vbuf, cnt2, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + + call h5dclose_f(dset_id, iError) + end do + + deallocate(vbuf) + deallocate(wmask) + + call h5fclose_f(file_id, iError) + call h5close_f(iError) + + end subroutine Monitor_WriteFileProbesHDF5 +#endif ! HAS_HDF5 + +#endif ! FLOW + end module MonitorsClass ! !/////////////////////////////////////////////////////////////////////////////////// -! \ No newline at end of file +! diff --git a/Solver/src/libs/monitors/Probe.f90 b/Solver/src/libs/monitors/Probe.f90 index 5e193c457..e91279dbc 100644 --- a/Solver/src/libs/monitors/Probe.f90 +++ b/Solver/src/libs/monitors/Probe.f90 @@ -24,20 +24,25 @@ module ProbeClass ! type Probe_t logical :: active + logical :: isFileProbe integer :: rank integer :: ID integer :: eID + integer :: nVars real(kind=RP) :: x(NDIM) real(kind=RP) :: xi(NDIM) - real(kind=RP), allocatable :: values(:) + real(kind=RP), allocatable :: values(:,:) real(kind=RP), allocatable :: lxi(:) , leta(:), lzeta(:) real(kind=RP), allocatable :: var(:,:,:) + real(kind=RP) :: saveTimestep + real(kind=RP) :: lastSavedTime character(len=STR_LEN_MONITORS) :: fileName character(len=STR_LEN_MONITORS) :: monitorName - character(len=STR_LEN_MONITORS) :: variable + character(len=STR_LEN_MONITORS), allocatable :: variableNames(:) contains procedure :: Initialization => Probe_Initialization procedure :: Update => Probe_Update + procedure :: ComputeLocal => Probe_ComputeLocal procedure :: WriteLabel => Probe_WriteLabel procedure :: WriteValues => Probe_WriteValue procedure :: WriteToFile => Probe_WriteToFile @@ -49,7 +54,7 @@ module ProbeClass contains - subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) + subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall, x_in, variables_in, name_in, isFileProbe_in, outputFormat_in) use ParamfileRegions use MPI_Process_Info use Utilities, only: toLower @@ -59,45 +64,82 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) integer :: ID character(len=*) :: solution_file logical, intent(in) :: FirstCall + real(kind=RP), intent(in), optional :: x_in(NDIM) + character(len=*), intent(in), optional :: variables_in(:) + character(len=*), intent(in), optional :: name_in + logical, intent(in), optional :: isFileProbe_in + character(len=*), intent(in), optional :: outputFormat_in ! ! --------------- ! Local variables ! --------------- ! - integer :: i, j, k, fid + integer :: i, j, k, v, fid character(len=STR_LEN_MONITORS) :: in_label character(len=STR_LEN_MONITORS) :: fileName character(len=STR_LEN_MONITORS) :: paramFile character(len=STR_LEN_MONITORS) :: coordinates - + character(len=STR_LEN_MONITORS) :: variable + character(len=STR_LEN_MONITORS) :: outputFormat + + self % isFileProbe = .false. + if ( present(isFileProbe_in) ) self % isFileProbe = isFileProbe_in + + outputFormat = "ASCII" + if ( present(outputFormat_in) ) outputFormat = trim(outputFormat_in) + if (FirstCall) then ! -! Allocate memory -! --------------- - allocate ( self % values(BUFFER_SIZE) ) -! ! Get monitor ID ! -------------- self % ID = ID ! -! Search for the parameters in the case file -! ------------------------------------------ - write(in_label , '(A,I0)') "#define probe " , self % ID - - call get_command_argument(1, paramFile) - call readValueInRegion(trim(paramFile), "name" , self % monitorName, in_label, "# end" ) - call readValueInRegion(trim(paramFile), "variable", self % variable , in_label, "# end" ) - call readValueInRegion(trim(paramFile), "position", coordinates , in_label, "# end" ) +! Get the probe definition, either from a probes file (x_in/variables_in +! provided) or from a "#define probe" block in the case file +! ------------------------------------------------------------------------ + if ( present(x_in) ) then + self % monitorName = name_in + self % x = x_in + self % nVars = size(variables_in) + allocate( self % variableNames(self % nVars) ) + self % variableNames = variables_in + + else +! +! Search for the parameters in the case file +! ------------------------------------------ + write(in_label , '(A,I0)') "#define probe " , self % ID + + call get_command_argument(1, paramFile) + call readValueInRegion(trim(paramFile), "name" , self % monitorName, in_label, "# end" ) + call readValueInRegion(trim(paramFile), "variable", variable , in_label, "# end" ) + call readValueInRegion(trim(paramFile), "position", coordinates , in_label, "# end" ) ! -! Get the coordinates -! ------------------- - self % x = getRealArrayFromString(coordinates) +! Get the coordinates +! ------------------- + self % x = getRealArrayFromString(coordinates) + + self % nVars = 1 + allocate( self % variableNames(1) ) + self % variableNames(1) = variable + end if +! +! Allocate memory +! --------------- + if (self % isFileProbe) then + allocate ( self % values(self % nVars, 1) ) + else + allocate ( self % values(self % nVars, BUFFER_SIZE) ) + end if + self % saveTimestep = 0.0_RP + self % lastSavedTime = -huge(self % lastSavedTime) ! -! Check the variable -! ------------------ - call tolower(self % variable) +! Check the variables +! -------------------- + do v = 1, self % nVars + call tolower(self % variableNames(v)) - select case ( trim(self % variable) ) + select case ( trim(self % variableNames(v)) ) #ifdef NAVIERSTOKES case ("pressure") case ("velocity") @@ -106,8 +148,9 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) case ("w") case ("mach") case ("k") + case ("rho") case default - print*, 'Probe variable "',trim(self % variable),'" not implemented.' + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' print*, "Options available are:" print*, " * pressure" print*, " * velocity" @@ -116,6 +159,7 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) print*, " * w" print*, " * Mach" print*, " * K" + print*, " * rho" end select #endif #ifdef INCNS @@ -124,21 +168,23 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) case ("u") case ("v") case ("w") + case ("rho") case default - print*, 'Probe variable "',trim(self % variable),'" not implemented.' + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' print*, "Options available are:" print*, " * pressure" print*, " * velocity" print*, " * u" print*, " * v" print*, " * w" + print*, " * rho" end select #endif #ifdef MULTIPHASE case ("static-pressure") case default - print*, 'Probe variable "',trim(self % variable),'" not implemented.' + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' print*, "Options available are:" print*, " * static-pressure" @@ -151,7 +197,7 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) case ("v") case ("w") case default - print*, 'Probe variable "',trim(self % variable),'" not implemented.' + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' print*, "Options available are:" print*, " * pressure" print*, " * velocity" @@ -160,7 +206,8 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) print*, " * w" end select #endif - + end do + ! ! Find the requested point in the mesh ! ------------------------------------ @@ -218,33 +265,36 @@ subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall) safedeallocate(self % var ) ; allocate( self % var(0 : e % Nxyz(1),0 : e % Nxyz(2),0 : e % Nxyz(3)) ) self % var = 0.0_RP - !$acc enter data copyin(self) - !$acc enter data copyin(self % eiD) - !$acc enter data copyin(self % id) - !$acc enter data copyin(self % var) - !$acc enter data copyin(self % lxi) - !$acc enter data copyin(self % leta) - !$acc enter data copyin(self % lzeta) + if ( .not. self % isFileProbe ) then + !$acc enter data copyin(self) + !$acc enter data copyin(self % eiD) + !$acc enter data copyin(self % id) + !$acc enter data copyin(self % var) + !$acc enter data copyin(self % lxi) + !$acc enter data copyin(self % leta) + !$acc enter data copyin(self % lzeta) + end if ! ! **************** ! Prepare the file ! **************** ! -! Create file -! ----------- - if (FirstCall) then - open ( newunit = fID , file = trim(self % fileName) , status = "unknown" , action = "write" ) +! Create file (skip for file-probes using HDF5 output) +! ----------------------------------------------------- + if (FirstCall .and. .not. (self % isFileProbe .and. trim(outputFormat) .eq. "HDF5")) then + open ( newunit = fID , file = trim(self % fileName) , status = "unknown" , action = "write" ) ! ! Write the file headers ! ---------------------- write( fID , '(A20,A )') "Monitor name: ", trim(self % monitorName) - write( fID , '(A20,A )') "Selected variable: " , trim(self % variable) - write( fID , '(A20,ES24.10)') "x coordinate: ", self % x(1) - write( fID , '(A20,ES24.10)') "y coordinate: ", self % x(2) - write( fID , '(A20,ES24.10)') "z coordinate: ", self % x(3) + write( fID , '(A25,ES24.10,2(4X,ES24.10))') "x, y, z coordinates: ", self % x(1), self % x(2), self % x(3) write( fID , * ) - write( fID , '(A10,2X,A24,2X,A24)' ) "Iteration" , "Time" , trim(self % variable) + write( fID , '(A10,2X,A24)' , advance = "no") "Iteration" , "Time" + do v = 1 , self % nVars + write( fID , '(2X,A24)' , advance = "no") trim(self % variableNames(v)) + end do + write( fID , * ) close ( fID ) end if @@ -264,59 +314,176 @@ subroutine Probe_Update(self, mesh, bufferPosition) ! Local variables ! --------------- ! - integer :: i, j, k, ierr + integer :: i, j, k, v, ierr real(kind=RP) :: value - if ( .not. self % active ) return + if ( .not. self % active ) return if ( MPI_Process % rank .eq. self % rank ) then ! ! Update the probe ! ---------------- - - select case (trim(self % variable)) + do v = 1, self % nVars + + if ( self % isFileProbe ) then +! +! CPU path for file-probes (no OpenACC to avoid runtime table overflow) +! ---------------------------------------------------------------------- + select case (trim(self % variableNames(v))) +#ifdef NAVIERSTOKES + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("mach") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) + self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& + (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) + end do ; end do ; end do + case("k") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef INCNS + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef MULTIPHASE + case("static-pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & + - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & + - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) + end do ; end do ; end do +#endif +#ifdef ACOUSTIC + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAP,i,j,k) + end do ; end do ; end do + case("density") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAARHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAU,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAV,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAW,i,j,k) + end do ; end do ; end do +#endif + end select + + value = 0.0_RP + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) + end do ; end do ; end do + + else +! +! GPU path for standard probes (OpenACC) +! --------------------------------------- + select case (trim(self % variableNames(v))) #ifdef NAVIERSTOKES case("pressure") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) end do ; end do ; end do !$acc end parallel loop - + case("velocity") !$acc parallel loop collapse(3) present(mesh, self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("u") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop case("v") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("w") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("mach") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) ! Vabs**2 @@ -324,93 +491,107 @@ subroutine Probe_Update(self, mesh, bufferPosition) (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) end do ; end do ; end do !$acc end parallel loop - + case("k") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop + + case("rho") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop #endif #ifdef INCNS case("pressure") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("velocity") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & POW2( mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("u") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("v") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop - + case("w") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) end do ; end do ; end do !$acc end parallel loop + + case("rho") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop #endif #ifdef MULTIPHASE case("static-pressure") !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) end do ; end do ; end do !$acc end parallel loop -#endif +#endif #ifdef ACOUSTIC case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAAP,i,j,k) end do ; end do ; end do case("density") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAARHO,i,j,k) end do ; end do ; end do - + case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAAU,i,j,k) end do ; end do ; end do - + case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAAV,i,j,k) end do ; end do ; end do - + case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) self % var(i,j,k) = Q(ICAAW,i,j,k) end do ; end do ; end do #endif end select - + value = 0.0_RP !$acc parallel loop collapse(3) present(mesh, self) reduction(+:value) async(self % ID) do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) @@ -420,14 +601,18 @@ subroutine Probe_Update(self, mesh, bufferPosition) !$acc wait - self % values(bufferPosition) = value - -#ifdef _HAS_MPI_ + end if ! isFileProbe + + self % values(v, bufferPosition) = value + + end do +! +#ifdef _HAS_MPI_ if ( MPI_Process % doMPIAction ) then ! ! Share the result with the rest of the processes -! ----------------------------------------------- - call mpi_bcast(value, 1, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) +! ----------------------------------------------- + call mpi_bcast(self % values(:,bufferPosition), self % nVars, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) end if #endif @@ -437,12 +622,160 @@ subroutine Probe_Update(self, mesh, bufferPosition) ! -------------------------------------------------------- #ifdef _HAS_MPI_ if ( MPI_Process % doMPIAction ) then - call mpi_bcast(self % values(bufferPosition), 1, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) + call mpi_bcast(self % values(:,bufferPosition), self % nVars, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) end if #endif end if end subroutine Probe_Update + subroutine Probe_ComputeLocal(self, mesh, bufferPosition) +! +! ************************************************************* +! CPU-only computation for file-probes, no MPI. +! Non-owning ranks set values to 0 so a caller can +! accumulate results with a single MPI_Allreduce(SUM). +! ************************************************************* +! + use Physics + use MPI_Process_Info + implicit none + class(Probe_t) :: self + type(HexMesh) :: mesh + integer :: bufferPosition +! +! --------------- +! Local variables +! --------------- +! + integer :: i, j, k, v + real(kind=RP) :: value + + if ( .not. self % active ) then + self % values(:, bufferPosition) = 0.0_RP + return + end if + + if ( MPI_Process % rank .ne. self % rank ) then + self % values(:, bufferPosition) = 0.0_RP + return + end if + + do v = 1, self % nVars + + select case (trim(self % variableNames(v))) +#ifdef NAVIERSTOKES + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("mach") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) + self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& + (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) + end do ; end do ; end do + case("k") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef INCNS + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef MULTIPHASE + case("static-pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & + - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & + - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) + end do ; end do ; end do +#endif +#ifdef ACOUSTIC + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAP,i,j,k) + end do ; end do ; end do + case("density") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAARHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAU,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAV,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAW,i,j,k) + end do ; end do ; end do +#endif + end select + + value = 0.0_RP + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) + end do ; end do ; end do + + self % values(v, bufferPosition) = value + + end do + + end subroutine Probe_ComputeLocal + subroutine Probe_WriteLabel ( self ) ! ! ************************************************************* @@ -469,9 +802,9 @@ subroutine Probe_WriteValue ( self , bufferLine ) class(Probe_t) :: self integer :: bufferLine - write(STD_OUT , '(1X,A,1X,ES10.3)' , advance = "no") "|" , self % values ( bufferLine ) + write(STD_OUT , '(1X,A,1X,ES10.3)' , advance = "no") "|" , self % values ( 1 , bufferLine ) - end subroutine Probe_WriteValue + end subroutine Probe_WriteValue subroutine Probe_WriteToFile ( self , iter , t , no_of_lines) ! @@ -489,22 +822,34 @@ subroutine Probe_WriteToFile ( self , iter , t , no_of_lines) ! Local variables ! --------------- ! - integer :: i + integer :: i, v integer :: fID + if ( .not. self % active ) then + if ( no_of_lines .ne. 0 ) self % values(:,1) = self % values(:,no_of_lines) + return + end if + if ( MPI_Process % isRoot ) then open( newunit = fID , file = trim ( self % fileName ) , action = "write" , access = "append" , status = "old" ) - - do i = 1 , no_of_lines - write( fID , '(I10,2X,ES24.16,2X,ES24.16)' ) iter(i) , t(i) , self % values(i) + do i = 1 , no_of_lines + if ( self % saveTimestep > 0.0_RP ) then + if ( t(i) < self % lastSavedTime + self % saveTimestep ) cycle + end if + write( fID , '(I10,2X,ES24.16)' , advance = "no" ) iter(i) , t(i) + do v = 1 , self % nVars + write( fID , '(2X,ES24.16)' , advance = "no" ) self % values(v,i) + end do + write( fID , * ) + self % lastSavedTime = t(i) end do - + close ( fID ) end if - - if ( no_of_lines .ne. 0 ) self % values(1) = self % values(no_of_lines) + + if ( no_of_lines .ne. 0 ) self % values(:,1) = self % values(:,no_of_lines) end subroutine Probe_WriteToFile @@ -566,6 +911,7 @@ elemental subroutine Probe_Destruct (self) safedeallocate (self % lxi) safedeallocate (self % leta) safedeallocate (self % lzeta) + safedeallocate (self % variableNames) end subroutine Probe_Destruct elemental subroutine Probe_Assign (to, from) @@ -574,32 +920,47 @@ elemental subroutine Probe_Assign (to, from) type(Probe_t) , intent(in) :: from to % active = from % active + to % isFileProbe = from % isFileProbe to % rank = from % rank to % ID = from % ID - to % eID = from % eID + to % eID = from % eID + to % nVars = from % nVars to % x = from % x to % xi = from % xi + + if ( allocated(from % values) ) then + safedeallocate ( to % values ) + allocate ( to % values ( size(from % values, 1), size(from % values, 2) ) ) + to % values = from % values + end if + + if ( allocated(from % lxi) ) then + safedeallocate ( to % lxi ) + allocate ( to % lxi ( size(from % lxi) ) ) + to % lxi = from % lxi + end if + + if ( allocated(from % leta) ) then + safedeallocate ( to % leta ) + allocate ( to % leta ( size(from % leta) ) ) + to % leta = from % leta + end if + + if ( allocated(from % lzeta) ) then + safedeallocate ( to % lzeta ) + allocate ( to % lzeta ( size(from % lzeta) ) ) + to % lzeta = from % lzeta + end if - safedeallocate ( to % values ) - allocate ( to % values ( size(from % values) ) ) - to % values = from % values - - safedeallocate ( to % lxi ) - allocate ( to % lxi ( size(from % lxi) ) ) - to % lxi = from % lxi - - safedeallocate ( to % leta ) - allocate ( to % leta ( size(from % leta) ) ) - to % leta = from % leta - - safedeallocate ( to % lzeta ) - allocate ( to % lzeta ( size(from % lzeta) ) ) - to % lzeta = from % lzeta - + to % saveTimestep = from % saveTimestep + to % lastSavedTime = from % lastSavedTime to % fileName = from % fileName to % monitorName = from % monitorName - to % variable = from % variable - + + safedeallocate ( to % variableNames ) + allocate ( to % variableNames ( size(from % variableNames) ) ) + to % variableNames = from % variableNames + end subroutine Probe_Assign end module ProbeClass diff --git a/Solver/src/libs/monitors/SurfaceIntegrals.f90 b/Solver/src/libs/monitors/SurfaceIntegrals.f90 index d2b04c234..6807203c8 100644 --- a/Solver/src/libs/monitors/SurfaceIntegrals.f90 +++ b/Solver/src/libs/monitors/SurfaceIntegrals.f90 @@ -70,8 +70,11 @@ function ScalarSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) ! ! Loop the zone to get faces and elements ! --------------------------------------- +#ifdef _OPENACC !$acc parallel loop gang reduction(+:val) present(mesh) +#else !$omp parallel do private(fID) reduction(+:val) schedule(runtime) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID @@ -83,8 +86,11 @@ function ScalarSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) val = val + ScalarSurfaceIntegral_Face(mesh % faces(fID), integralType) end do -!$omp end parallel do +#ifdef _OPENACC !$acc end parallel loop +#else +!$omp end parallel do +#endif #ifdef _HAS_MPI_ localval = val @@ -274,14 +280,17 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) select case ( integralType ) case ( SURFACE ) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#else !$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID ! -------------- fID = mesh % zones(zoneID) % faces(zonefID) - + localx = 0.0_RP localy = 0.0_RP localz = 0.0_RP @@ -295,31 +304,37 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) val = NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) & * mesh % faces(fid) % geom % jacobian(i,j) * mesh % faces(fid) % geom % normal(:,i,j) - + localx = localx + val(1) localy = localy + val(2) localz = localz + val(3) - + end do ; end do - + valx = valx + localx valy = valy + localy valz = valz + localz - + end do -!$omp end parallel do +#ifdef _OPENACC !$acc end parallel loop +#else +!$omp end parallel do +#endif case ( TOTAL_FORCE ) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#else !$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID ! -------------- fID = mesh % zones(zoneID) % faces(zonefID) - + localx = 0.0_RP localy = 0.0_RP localz = 0.0_RP @@ -337,34 +352,40 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) p = Pressure(mesh % faces(fid) % storage(1) % Q(:,i,j)) call getStressTensor(mesh % faces(fid) % storage(1) % Q(:,i,j), mesh % faces(fid) % storage(1) % U_x(:,i,j),& mesh % faces(fid) % storage(1) % U_y(:,i,j),mesh % faces(fid) % storage(1) % U_z(:,i,j), tau) - + localval = ( p * mesh % faces(fid) % geom % normal(:,i,j) - matmul(tau,mesh % faces(fid) % geom % normal(:,i,j)) ) & * mesh % faces(fid) % geom % jacobian(i,j) & * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - + localx = localx + localval(1) localy = localy + localval(2) localz = localz + localval(3) end do ; end do - + valx = valx + localx valy = valy + localy valz = valz + localz - + end do -!$omp end parallel do +#ifdef _OPENACC !$acc end parallel loop +#else +!$omp end parallel do +#endif case ( PRESSURE_FORCE ) -!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#ifdef _OPENACC +!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#else !$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID ! -------------- fID = mesh % zones(zoneID) % faces(zonefID) - + localx = 0.0_RP localy = 0.0_RP localz = 0.0_RP @@ -373,41 +394,47 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) ! Computes the pressure forces experienced by the zone ! F = \int p \vec{n}ds ! **************************************************** - ! + ! !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(val) do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) ! ! Compute the integral ! -------------------- p = Pressure(mesh % faces(fid) % storage(1) % Q(:,i,j)) - + val = ( p * mesh % faces(fid) % geom % normal(:,i,j) ) * mesh % faces(fid) % geom % jacobian(i,j) & * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) & * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - + localx = localx + val(1) localy = localy + val(2) localz = localz + val(3) end do ; end do - + valx = valx + localx valy = valy + localy valz = valz + localz - + end do -!$omp end parallel do +#ifdef _OPENACC !$acc end parallel loop +#else +!$omp end parallel do +#endif case ( VISCOUS_FORCE ) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#else !$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID ! -------------- fID = mesh % zones(zoneID) % faces(zonefID) - + localx = 0.0_RP localy = 0.0_RP localz = 0.0_RP @@ -424,23 +451,26 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) ! -------------------- call getStressTensor(mesh % faces(fid) % storage(1) % Q(:,i,j), mesh % faces(fid) % storage(1) % U_x(:,i,j),& mesh % faces(fid) % storage(1) % U_y(:,i,j),mesh % faces(fid) % storage(1) % U_z(:,i,j), tau) - + val = - matmul(tau,mesh % faces(fid) % geom % normal(:,i,j)) * mesh % faces(fid) % geom % jacobian(i,j) & * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) & * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - + localx = localx + val(1) localy = localy + val(2) localz = localz + val(3) end do ; end do - + valx = valx + localx valy = valy + localy valz = valz + localz - + end do -!$omp end parallel do +#ifdef _OPENACC !$acc end parallel loop +#else +!$omp end parallel do +#endif end select @@ -1143,8 +1173,11 @@ subroutine ProlongToFaces(mesh, zoneID) select case ( mesh %nodeType ) case(1) !Gauss +#ifdef _OPENACC + !$acc parallel loop gang present(mesh) +#else !$omp do schedule(runtime) private(eID) - !$acc parallel loop gang present(mesh) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces fID = mesh % zones(zoneID) % faces(zonefID) eID = mesh % faces(fID) % elementIDs(1) @@ -1160,46 +1193,67 @@ subroutine ProlongToFaces(mesh, zoneID) mesh % faces(fID), & mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1), 3) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif case(2) !Gauss-Lobatto - !$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) private(fID) +#else + !$omp do schedule(runtime) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces fID = mesh % zones(zoneID) % faces(zonefID) eID = mesh % faces(fID) % elementIDs(1) call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & mesh % faces(fID), & - mesh % elements(eID) % storage % U_x, mesh % faces(fID) % elementSide(1),1) + mesh % elements(eID) % storage % U_x, mesh % faces(fID) % elementSide(1),1) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif - !$omp do schedule(runtime) +#ifdef _OPENACC !$acc parallel loop gang present(mesh) private(fID) +#else + !$omp do schedule(runtime) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces fID = mesh % zones(zoneID) % faces(zonefID) eID = mesh % faces(fID) % elementIDs(1) call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & mesh % faces(fID), & - mesh % elements(eID) % storage % U_y, mesh % faces(fID) % elementSide(1),2) + mesh % elements(eID) % storage % U_y, mesh % faces(fID) % elementSide(1),2) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do - - !$omp do schedule(runtime) +#endif + +#ifdef _OPENACC !$acc parallel loop gang present(mesh) private(fID) +#else + !$omp do schedule(runtime) +#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces fID = mesh % zones(zoneID) % faces(zonefID) eID = mesh % faces(fID) % elementIDs(1) call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & mesh % faces(fID), & - mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1),3) + mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1),3) end do +#ifdef _OPENACC !$acc end parallel loop +#else !$omp end do +#endif end select end if ! @@ -1212,3 +1266,4 @@ end subroutine ProlongToFaces end module SurfaceIntegrals #endif + diff --git a/Solver/src/libs/monitors/VolumeIntegrals.f90 b/Solver/src/libs/monitors/VolumeIntegrals.f90 index 353d9b78f..55d60f112 100644 --- a/Solver/src/libs/monitors/VolumeIntegrals.f90 +++ b/Solver/src/libs/monitors/VolumeIntegrals.f90 @@ -607,8 +607,11 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) case ( VELOCITY ) ! num_of_vars is equal to NDIM +#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(9700) reduction(+:val1,val2,val3) async(1) +#else !$omp parallel do reduction(+:val1,val2,val3) private(val) schedule(guided) +#endif do eID = 1, mesh % no_of_elements ! local1 = 0.0_RP @@ -645,14 +648,19 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) val3 = val3 + local3 end do -!$omp end parallel do +#ifdef _OPENACC !$acc end parallel loop - +#else +!$omp end parallel do +#endif case ( MOMENTUM ) ! num_of_vars is equal to NDIM +#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(9700) reduction(+:val1,val2,val3) +#else !$omp parallel do reduction(+:val1,val2,val3) private(val) schedule(guided) +#endif do eID = 1, mesh % no_of_elements ! local1 = 0.0_RP @@ -684,8 +692,11 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) val3 = val3 + local3 end do -!$omp end parallel do +#ifdef _OPENACC !$acc end parallel loop +#else +!$omp end parallel do +#endif ! case ( PSOURCE ) ! ! num_of_vars is equal to NCONS @@ -730,8 +741,11 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) case ( SOURCE ) ! num_of_vars is equal to NCONS +#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(9700) reduction(+:val1,val2,val3,val4,val5) +#else !$omp parallel do reduction(+:val1,val2,val3,val4,val5) private(val) schedule(guided) +#endif do eID = 1, mesh % no_of_elements ! local1 = 0.0_RP @@ -777,8 +791,11 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) val5 = val5 + local5 end do -!$omp end parallel do +#ifdef _OPENACC !$acc end parallel loop +#else +!$omp end parallel do +#endif ! #endif ! diff --git a/Solver/src/libs/sources/ActuatorLine.f90 b/Solver/src/libs/sources/ActuatorLine.f90 index ed549004c..3c062aeb1 100644 --- a/Solver/src/libs/sources/ActuatorLine.f90 +++ b/Solver/src/libs/sources/ActuatorLine.f90 @@ -711,7 +711,7 @@ subroutine UpdateFarm(self,time, mesh) ! pointsToFind = 0 newPointToFind = 0 -!$omp do schedule(runtime)private(ii,jj,kk,eID,Q,Qtemp,delta_temp,xi,found,allfound) +!$omp do schedule(runtime)private(ii,jj,kk,eID,delta_temp,xi,found,allfound) do kk = 1, self%num_turbines do jj = 1, self%turbine_t(kk)%num_blades @@ -751,6 +751,7 @@ subroutine UpdateFarm(self,time, mesh) end do enddo enddo +!$omp end do ! update MPI partitions and look for points that have changed ! ----------------------------------------------------------- @@ -832,7 +833,7 @@ subroutine UpdateFarm(self,time, mesh) enddo enddo -!$omp end do + ! send local forces arrays and angle to device do kk=1, self % num_turbines do jj=1, self % turbine_t(kk) % num_blades diff --git a/Solver/src/libs/timeintegrator/AnalyticalJacobian.f90 b/Solver/src/libs/timeintegrator/AnalyticalJacobian.f90 index 2a27395ce..6ba5cbdb0 100644 --- a/Solver/src/libs/timeintegrator/AnalyticalJacobian.f90 +++ b/Solver/src/libs/timeintegrator/AnalyticalJacobian.f90 @@ -230,6 +230,7 @@ subroutine AnJacobian_Compute(this, sem, nEqn, time, Matrix, TimeDerivative, Tim ! ************************************************************************ ! call ComputeNumericalFluxJacobian(sem % mesh,nEqn,time) +!$omp end parallel ! ! *************** ! Diagonal blocks @@ -242,7 +243,6 @@ subroutine AnJacobian_Compute(this, sem, nEqn, time, Matrix, TimeDerivative, Tim ! ******************* ! if (.not. BlockDiagonal) call AnalyticalJacobian_OffDiagonalBlocks(sem % mesh,Matrix) -!$omp end parallel ! ! ************************ ! Finish assembling matrix @@ -288,34 +288,33 @@ subroutine AnalyticalJacobian_DiagonalBlocks(mesh,nEqn,time,Matrix) class(Matrix_t) , intent(inout) :: Matrix !-------------------------------------------- integer :: eID, fID - type(Element), pointer :: e !-------------------------------------------- ! ! Project flux Jacobian to corresponding element ! ---------------------------------------------- -!$omp do schedule(runtime) +!$omp parallel do schedule(runtime) do fID = 1, size(mesh % faces) - call mesh % faces(fID) % ProjectFluxJacobianToElements(nEqn,LEFT ,LEFT ) ! dF/dQL to the left element + call mesh % faces(fID) % ProjectFluxJacobianToElements(nEqn,LEFT ,LEFT ) ! dF/dQL to the left element if (.not. (mesh % faces(fID) % faceType == HMESH_BOUNDARY)) call mesh % faces(fID) % ProjectFluxJacobianToElements(nEqn,RIGHT,RIGHT) ! dF/dQR to the right element end do -!$omp end do +!$omp end parallel do ! ! Project flux Jacobian with respect to gradients to corresponding elements ! ------------------------------------------------------------------------- if (flowIsNavierStokes) then -!$omp do schedule(runtime) +!$omp parallel do schedule(runtime) do fID = 1, size(mesh % faces) - call mesh % faces(fID) % ProjectGradJacobianToElements(LEFT, LEFT) ! dF/dQL to the left element + call mesh % faces(fID) % ProjectGradJacobianToElements(LEFT, LEFT) ! dF/dQL to the left element if (.not. (mesh % faces(fID) % faceType == HMESH_BOUNDARY)) call mesh % faces(fID) % ProjectGradJacobianToElements(RIGHT,RIGHT) ! dF/dQR to the right element end do -!$omp end do +!$omp end parallel do end if ! ! Compute each element's diagonal block ! ------------------------------------- -!$omp do schedule(runtime) private(e) +!$omp parallel do schedule(runtime) do eID = 1, size(mesh % elements) - e => mesh % elements(eID) + associate(e => mesh % elements(eID)) call Local_SetDiagonalBlock( e, & mesh % faces( e % faceIDs(EFRONT ) ), & mesh % faces( e % faceIDs(EBACK ) ), & @@ -325,9 +324,9 @@ subroutine AnalyticalJacobian_DiagonalBlocks(mesh,nEqn,time,Matrix) mesh % faces( e % faceIDs(ELEFT ) ), & Matrix, & mesh% IBM ) + end associate end do -!$omp end do - nullify (e) +!$omp end parallel do end subroutine AnalyticalJacobian_DiagonalBlocks ! !/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -344,58 +343,56 @@ subroutine AnalyticalJacobian_OffDiagonalBlocks(mesh,Matrix) class(Matrix_t) , intent(inout) :: Matrix !-------------------------------------------- integer :: eID, fID, elSide, side - type(Element), pointer :: e_plus - type(Face) , pointer :: f !-------------------------------------------- - + ! ! Project flux Jacobian to opposed elements (RIGHT to LEFT and LEFT to RIGHT) ! --------------------------------------------------------------------------- -!$omp do schedule(runtime) +!$omp parallel do schedule(runtime) do fID = 1, size(mesh % faces) if (mesh % faces(fID) % faceType /= HMESH_BOUNDARY) then call mesh % faces(fID) % ProjectFluxJacobianToElements(NCONS, LEFT ,RIGHT) ! dF/dQR to the left element - call mesh % faces(fID) % ProjectFluxJacobianToElements(NCONS, RIGHT,LEFT ) ! dF/dQL to the right element + call mesh % faces(fID) % ProjectFluxJacobianToElements(NCONS, RIGHT,LEFT ) ! dF/dQL to the right element end if end do -!$omp end do +!$omp end parallel do ! ! Project flux Jacobian with respect to gradients to opposed elements (RIGHT to LEFT and LEFT to RIGHT) ! ----------------------------------------------------------------------------------------------------- if (flowIsNavierStokes) then -!$omp do schedule(runtime) +!$omp parallel do schedule(runtime) do fID = 1, size(mesh % faces) if (mesh % faces(fID) % faceType /= HMESH_BOUNDARY) then call mesh % faces(fID) % ProjectGradJacobianToElements(LEFT ,RIGHT) ! dF/dGradQR to the left element - call mesh % faces(fID) % ProjectGradJacobianToElements(RIGHT,LEFT ) ! dF/dGradQL to the right element + call mesh % faces(fID) % ProjectGradJacobianToElements(RIGHT,LEFT ) ! dF/dGradQL to the right element end if end do -!$omp end do +!$omp end parallel do end if - + ! ! Compute the off-diagonal blocks for each element's equations ! ------------------------------------------------------------ -!$omp do schedule(runtime) private(e_plus,elSide,fID,side,f) +!$omp parallel do schedule(runtime) private(elSide,fID,side) do eID = 1, mesh % no_of_elements - e_plus => mesh % elements(eID) + associate(e_plus => mesh % elements(eID)) ! ! One block for every neighbor element ! ------------------------------------ do elSide = 1, 6 if (e_plus % NumberOfConnections(elSide) == 0) cycle - + fID = e_plus % faceIDs(elSide) side = e_plus % faceSide(elSide) - - f => mesh % faces(fID) - + + associate(f => mesh % faces(fID)) call Local_GetOffDiagonalBlock(f,e_plus,e_plus % Connection(elSide),side,Matrix) + end associate end do + end associate end do -!$omp end do - nullify (f, e_plus) +!$omp end parallel do end subroutine AnalyticalJacobian_OffDiagonalBlocks ! !/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From 8937ce07e2528f527f2f9144a2deefb62d32faf5 Mon Sep 17 00:00:00 2001 From: Gonzalo Rubio Date: Wed, 8 Jul 2026 10:56:58 +0200 Subject: [PATCH 2/9] Activate openmp threads in CI (CPU+GPU) --- .github/workflows/CI_parallel_GPU_OPENMP.yml | 1477 ++++++++++++++++++ .github/workflows/CI_parallel_MU.yml | 2 +- .github/workflows/CI_parallel_NS.yml | 2 +- .github/workflows/CI_parallel_iNS.yml | 2 +- .github/workflows/CI_serial_GPU_OPENMP.yml | 1309 ++++++++++++++++ .github/workflows/CI_serial_MU.yml | 2 +- 6 files changed, 2790 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/CI_parallel_GPU_OPENMP.yml create mode 100644 .github/workflows/CI_serial_GPU_OPENMP.yml diff --git a/.github/workflows/CI_parallel_GPU_OPENMP.yml b/.github/workflows/CI_parallel_GPU_OPENMP.yml new file mode 100644 index 000000000..47d945b68 --- /dev/null +++ b/.github/workflows/CI_parallel_GPU_OPENMP.yml @@ -0,0 +1,1477 @@ +name: CI PARALLEL (MPI) GPU + +######################################################################## +on: + # Manual trigger only + workflow_dispatch: + +jobs: + wait-guard: + name: Runner availability guard + runs-on: ubuntu-latest + timeout-minutes: 2880 # 48 hours + steps: + # This job serves as a guard to prevent the workflow from pending indefinitely if the self-hosted GPU runner is not available. + - run: echo "GPU runner did not appear within 48 hours" + + parallel-gpu-tests: + # Uses self-hosted runner on nibbler cluster with GPU access + name: Parallel GPU Tests (NVFORTRAN) + runs-on: [self-hosted, nibbler, gpu] + # Prevent runaway jobs from consuming cluster resources indefinitely + timeout-minutes: 120 + + env: + METIS_HOME: ${{ github.workspace }}/metis-5.1.0/build/Linux-x86_64 + + steps: + - uses: actions/checkout@v4 + + # Fix ownership so the SLURM job user can read all checked-out files. + # Root (or the runner service account) owns files after checkout; a.ballout + # needs read access to mesh files, binaries, and control files at runtime. + - name: Fix workspace ownership + run: sudo chown -R a.ballout:a.ballout $GITHUB_WORKSPACE + + # Install METIS library (required for parallel partitioning) + - name: Install METIS + run: | + module purge + module load cmake + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + if [ ! -d "metis-5.1.0" ]; then + wget "https://src.fedoraproject.org/lookaside/pkgs/metis/metis-5.1.0.tar.gz/5465e67079419a69e0116de24fce58fe/metis-5.1.0.tar.gz" + tar -xvf metis-5.1.0.tar.gz + cd metis-5.1.0 + # Update CMake minimum required version to 3.5 + sed -i 's/cmake_minimum_required(VERSION [0-9.]\+)/cmake_minimum_required(VERSION 3.5)/' CMakeLists.txt + make config cc=gcc + make + mkdir -p build/Linux-x86_64/lib build/Linux-x86_64/bin + cp -r build/Linux-x86_64/libmetis/* build/Linux-x86_64/lib/ 2>/dev/null || true + cp -r build/Linux-x86_64/programs/* build/Linux-x86_64/bin/ 2>/dev/null || true + fi + + - name: Configure Horses3D + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cd Solver + ./configure + + - name: Compile horses3d + working-directory: ./Solver + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make allclean COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + make ns ins mu COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + +######################################################################## +####### NAVIER-STOKES TESTS ######## +######################################################################## + +# +# 1) CYLINDER +# ----------- + + - name: Build NSCylinder + working-directory: ./Solver/test/NavierStokes/Cylinder/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Fix test directory ownership + run: sudo chown -R a.ballout:a.ballout $GITHUB_WORKSPACE + if: '!cancelled()' + + - name: Run NSCylinder + working-directory: ./Solver/test/NavierStokes/Cylinder + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinder + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns Cylinder.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 5) CYLINDER SMAGORINSKY +# ----------------------- + + # - name: Build NSCylinderSmagorinsky + # working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + # make clean || true + # make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + # if: '!cancelled()' + + # - name: Run NSCylinderSmagorinsky + # working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=NSCylinderSmagorinsky + # #SBATCH --ntasks=2 + # #SBATCH --partition=gpu + # #SBATCH --mem=32G + # #SBATCH --gres=gpu:2 # number of gpus max 4 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + # mpirun -np 2 --bind-to none ./horses3d.ns CylinderSmagorinsky.control + # EOF + + # job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + +# +# 6) CYLINDER WALE +# ------------------ + + - name: Build NSCylinderWALE + working-directory: ./Solver/test/NavierStokes/CylinderWALE/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run NSCylinderWALE + working-directory: ./Solver/test/NavierStokes/CylinderWALE + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderWALE + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderWALE.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 7) CYLINDER VREMAN +# ------------------ + + - name: Build NSCylinderVreman + working-directory: ./Solver/test/NavierStokes/CylinderVreman/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run NSCylinderVreman + working-directory: ./Solver/test/NavierStokes/CylinderVreman + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderVreman + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderVreman.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 8) CYLINDER ChandrasekarRoe +# ------------------ + + - name: Build NSCylinderChandrasekarRoe + working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run NSCylinderChandrasekarRoe + working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderChandrasekarRoe + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderChandrasekarRoe.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# # +# # 9) TAYLOR GREEN VORTEX +# # ---------------------- + +# - name: Build TaylorGreen +# working-directory: ./Solver/test/NavierStokes/TaylorGreen/SETUP +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi +# make clean || true +# make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES +# if: '!cancelled()' + +# - name: Run TaylorGreen +# working-directory: ./Solver/test/NavierStokes/TaylorGreen +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + +# cat > run_test.sh << 'EOF' +# #!/bin/bash +# #SBATCH --job-name=TaylorGreen +# #SBATCH --ntasks=2 +# #SBATCH --partition=gpu +# #SBATCH --mem=32G +# #SBATCH --gres=gpu:2 # number of gpus max 4 +# #SBATCH --time=00:15:00 + +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi +# mpirun -np 2 --bind-to none ./horses3d.ns TaylorGreen.control +# EOF + +# job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) +# while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + +# cat slurm-${job_id}.out + +# # Check exit code (ExitCode:Signal, e.g., "0:0" means success) +# exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) +# if [ "$exit_code" != "0" ]; then +# echo "Test failed with exit code: $exit_code" +# exit 1 +# fi +# if: '!cancelled()' + +# # +# # 18) NACA0012 Unsteady Dual Time Stepping +# # ---------------------------- + +# - name: Build DualTimeStepping +# working-directory: ./Solver/test/NavierStokes/DualTimeStepping/SETUP +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi +# make clean || true +# make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES +# if: '!cancelled()' + +# - name: Run DualTimeStepping +# working-directory: ./Solver/test/NavierStokes/DualTimeStepping +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + +# cat > run_test.sh << 'EOF' +# #!/bin/bash +# #SBATCH --job-name=DualTimeStepping +# #SBATCH --ntasks=2 +# #SBATCH --partition=gpu +# #SBATCH --mem=32G +# #SBATCH --gres=gpu:2 # number of gpus max 4 +# #SBATCH --time=00:15:00 + +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 +# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi +# mpirun -np 2 --bind-to none ./horses3d.ns DualTimeStepping.control +# EOF + +# job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) +# while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + +# cat slurm-${job_id}.out + +# exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) +# if [ "$exit_code" != "0" ]; then +# echo "Test failed with exit code: $exit_code" +# exit 1 +# fi +# if: '!cancelled()' + +# +# 26) IBM CYLINDER +# ------------------- + + - name: Build IBM_Cylinder + working-directory: ./Solver/test/NavierStokes/IBM_Cylinder/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run IBM_Cylinder + working-directory: ./Solver/test/NavierStokes/IBM_Cylinder + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=IBM_Cylinder + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns IBMCylinder.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 40) CYLINDER EntropyConservingCentral +# ------------------------------------- + - name: Build NSCylinderEntropyConservingCentral + working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderEntropyConservingCentral + working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderEntropyConservingCentral + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderEntropyConservingCentral.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 41) CYLINDER RoePikePirozzoli +# ---------------------------- + - name: Build NSCylinderRoePikePirozzoli + working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderRoePikePirozzoli + working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderRoePikePirozzoli + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderRoePikePirozzoli.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 42) CYLINDER KennedyGruberLaxFriedrichs +# --------------------------------------- + - name: Build NSCylinderKennedyGruberLaxFriedrichs + working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderKennedyGruberLaxFriedrichs + working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderKennedyGruberLaxFriedrichs + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderKennedyGruberLaxFriedrichs.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 43) CYLINDER MorinishiLowDissipationRoe +# ---------------------------------------- + - name: Build NSCylinderMorinishiLowDissipationRoe + working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderMorinishiLowDissipationRoe + working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderMorinishiLowDissipationRoe + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderMorinishiLowDissipationRoe.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 44) CYLINDER DucrosMatrixDissipation +# ------------------------------------- + - name: Build NSCylinderDucrosMatrixDissipation + working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderDucrosMatrixDissipation + working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderDucrosMatrixDissipation + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderDucrosMatrixDissipation.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 45) CYLINDER ViscousStandard +# ------------------------- + - name: Build NSCylinderViscousStandard + working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderViscousStandard + working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderViscousStandard + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderViscousStandard.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 46) CYLINDER RusanovStandard +# ------------------------- + - name: Build NSCylinderRusanovStandard + working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderRusanovStandard + working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderRusanovStandard + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderRusanovStandard.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 47) CYLINDER UdissStandard +# ------------------------- + - name: Build NSCylinderUdissStandard + working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderUdissStandard + working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderUdissStandard + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns CylinderUdissStandard.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 48) CYLINDER sprk33 +# ------------------- + - name: Build NSCylinderssprk33 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk33/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderssprk33 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk33 + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderssprk33 + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns Cylinderssprk33.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 49) CYLINDER sprk43 +# ------------------- + - name: Build NSCylinderssprk43 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk43/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSCylinderssprk43 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk43 + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderssprk43 + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns Cylinderssprk43.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 50) Limiter Test +# ------------------- + - name: Build NSLimiterTest + working-directory: ./Solver/test/NavierStokes/LimiterTest/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + - name: Run NSLimiterTest + working-directory: ./Solver/test/NavierStokes/LimiterTest + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSLimiterTest + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.ns LimiterTest.control + EOF + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +######################################################################## +####### INCOMPRESSIBLE_NS TESTS ######## +######################################################################## + +### Sometimes this test passes and sometimes it failscc +# +# 1) NACA0012 +# ---------------------------- + + # - name: Build iNS-NACA0012 + # working-directory: ./Solver/test/IncompressibleNS/NACA0012/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + # make clean || true + # make ins COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + # if: '!cancelled()' + + # - name: Run iNS-NACA0012 + # working-directory: ./Solver/test/IncompressibleNS/NACA0012 + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=NACA0012 + # #SBATCH --ntasks=2 + # #SBATCH --partition=gpu + # #SBATCH --cpus-per-task=1 + # #SBATCH --mem=32G + # #SBATCH --gres=gpu:2 # number of gpus max 4 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + # export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + # mpirun -np 2 --bind-to none ./horses3d.ins NACA0012.control + # EOF + + # job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + + +######################################################################## +####### MULTIPHASE TESTS ######## +######################################################################## + +# +# 4) Entropy conserving test +# ---------------------------- + + - name: Build MultiphaseEntropyConservingTest + working-directory: ./Solver/test/Multiphase/EntropyConservingTest/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + make clean || true + make mu COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=YES WITH_METIS=YES + if: '!cancelled()' + + - name: Run MultiphaseEntropyConservingTest + working-directory: ./Solver/test/Multiphase/EntropyConservingTest + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=EntropyConservingTest + #SBATCH --ntasks=2 + #SBATCH --partition=gpu + #SBATCH --mem=32G + #SBATCH --gres=gpu:2 # number of gpus max 4 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6 + export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi + mpirun -np 2 --bind-to none ./horses3d.mu EntropyConservingTest.control + EOF + + job_id=$(sbatch --uid=a.ballout --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' \ No newline at end of file diff --git a/.github/workflows/CI_parallel_MU.yml b/.github/workflows/CI_parallel_MU.yml index 05bd52861..1161b0811 100644 --- a/.github/workflows/CI_parallel_MU.yml +++ b/.github/workflows/CI_parallel_MU.yml @@ -34,7 +34,7 @@ jobs: compiler: ['gfortran'] mode: ['DEBUG'] comm: ['PARALLEL'] - enable_threads: ['NO'] + enable_threads: ['NO','YES'] name: MU - ${{ matrix.compiler }} - ${{ matrix.mode }} - ${{ matrix.comm }} - ${{ matrix.enable_threads }} diff --git a/.github/workflows/CI_parallel_NS.yml b/.github/workflows/CI_parallel_NS.yml index 1cff15029..9fe72f8fc 100644 --- a/.github/workflows/CI_parallel_NS.yml +++ b/.github/workflows/CI_parallel_NS.yml @@ -34,7 +34,7 @@ jobs: compiler: ['gfortran'] mode: ['DEBUG'] comm: ['PARALLEL'] - enable_threads: ['NO'] + enable_threads: ['NO','YES'] mkl: ['NO'] hdf5: ['YES'] diff --git a/.github/workflows/CI_parallel_iNS.yml b/.github/workflows/CI_parallel_iNS.yml index bbef28c32..0453d3d74 100644 --- a/.github/workflows/CI_parallel_iNS.yml +++ b/.github/workflows/CI_parallel_iNS.yml @@ -34,7 +34,7 @@ jobs: compiler: ['gfortran'] mode: ['DEBUG'] comm: ['PARALLEL'] - enable_threads: ['NO'] + enable_threads: ['NO','YES'] name: iNS - ${{ matrix.compiler }} - ${{ matrix.mode }} - ${{ matrix.comm }} - ${{ matrix.enable_threads }} diff --git a/.github/workflows/CI_serial_GPU_OPENMP.yml b/.github/workflows/CI_serial_GPU_OPENMP.yml new file mode 100644 index 000000000..572abf5b1 --- /dev/null +++ b/.github/workflows/CI_serial_GPU_OPENMP.yml @@ -0,0 +1,1309 @@ +name: CI SERIAL GPU + +######################################################################## +on: + # Manual trigger only + workflow_dispatch: + +jobs: + wait-guard: + name: Runner availability guard + runs-on: ubuntu-latest + timeout-minutes: 2880 # 48 hours + steps: + # This job serves as a guard to prevent the workflow from pending indefinitely if the self-hosted GPU runner is not available. + - run: echo "GPU runner did not appear within 48 hours" + + serial-gpu-tests: + # Uses self-hosted runner on nibbler with GPU access + name: Serial GPU Tests (NVFORTRAN) + runs-on: [self-hosted, nibbler, gpu] + # Prevent runaway jobs from consuming cluster resources indefinitely + timeout-minutes: 120 + + env: + METIS_HOME: ${{ github.workspace }}/metis-5.1.0/build/Linux-x86_64 + + steps: + - uses: actions/checkout@v4 + + - name: Configure Horses3D + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cd Solver + ./configure + + - name: Compile horses3d + working-directory: ./Solver + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make allclean COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + make ns ins mu COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + +######################################################################## +####### NAVIER-STOKES TESTS ######## +######################################################################## + +# +# 1) CYLINDER +# ----------- + + - name: Build NSCylinder + working-directory: ./Solver/test/NavierStokes/Cylinder/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run NSCylinder + working-directory: ./Solver/test/NavierStokes/Cylinder + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinder_Serial + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns Cylinder.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 5) CYLINDER SMAGORINSKY +# ----------------------- + + # - name: Build NSCylinderSmagorinsky + # working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # make clean || true + # make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + # if: '!cancelled()' + + # - name: Run NSCylinderSmagorinsky + # working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=NSCylinderSmagorinsky + # #SBATCH --ntasks=1 + # #SBATCH --mem=20G + # #SBATCH --partition=gpu + # #SBATCH --gres=gpu:1 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # ./horses3d.ns CylinderSmagorinsky.control + # EOF + + # job_id=$(sbatch --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + +# +# 6) CYLINDER WALE +# ------------------ + + - name: Build NSCylinderWALE + working-directory: ./Solver/test/NavierStokes/CylinderWALE/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run NSCylinderWALE + working-directory: ./Solver/test/NavierStokes/CylinderWALE + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderWALE + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderWALE.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 7) CYLINDER VREMAN +# ------------------ + + - name: Build NSCylinderVreman + working-directory: ./Solver/test/NavierStokes/CylinderVreman/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run NSCylinderVreman + working-directory: ./Solver/test/NavierStokes/CylinderVreman + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderVreman + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderVreman.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 8) CYLINDER ChandrasekarRoe +# ------------------ + + - name: Build NSCylinderChandrasekarRoe + working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run NSCylinderChandrasekarRoe + working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderChandrasekarRoe + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderChandrasekarRoe.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 9) TAYLOR GREEN VORTEX +# ---------------------- + + # - name: Build TaylorGreen + # working-directory: ./Solver/test/NavierStokes/TaylorGreen/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # make clean || true + # make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + # if: '!cancelled()' + + # - name: Run TaylorGreen + # working-directory: ./Solver/test/NavierStokes/TaylorGreen + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=TaylorGreen_Serial + # #SBATCH --ntasks=1 + # #SBATCH --mem=20G + # #SBATCH --partition=gpu + # #SBATCH --gres=gpu:1 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # ./horses3d.ns TaylorGreen.control + # EOF + + # job_id=$(sbatch --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # # Check exit code (ExitCode:Signal, e.g., "0:0" means success) + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + +# # +# # 18) NACA0012 Unsteady Dual Time Stepping +# # ---------------------------- + +# - name: Build DualTimeStepping +# working-directory: ./Solver/test/NavierStokes/DualTimeStepping/SETUP +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# make clean || true +# make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES +# if: '!cancelled()' + +# - name: Run DualTimeStepping +# working-directory: ./Solver/test/NavierStokes/DualTimeStepping +# run: | +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + +# cat > run_test.sh << 'EOF' +# #!/bin/bash +# #SBATCH --job-name=DualTimeStepping +# #SBATCH --ntasks=1 +# #SBATCH --mem=20G +# #SBATCH --partition=gpu +# #SBATCH --gres=gpu:1 +# #SBATCH --time=00:15:00 + +# module purge +# module load hpc_sdk +# module load nvhpc/24.11 +# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda +# export CUDA_HOME=$NVHPC_CUDA_HOME +# export CUDA_PATH=$NVHPC_CUDA_HOME +# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH +# ./horses3d.ns DualTimeStepping.control +# EOF + +# job_id=$(sbatch --parsable run_test.sh) +# while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + +# cat slurm-${job_id}.out + +# exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) +# if [ "$exit_code" != "0" ]; then +# echo "Test failed with exit code: $exit_code" +# exit 1 +# fi +# if: '!cancelled()' + +# +# 26) IBM CYLINDER +# ------------------- + + - name: Build IBM_Cylinder + working-directory: ./Solver/test/NavierStokes/IBM_Cylinder/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run IBM_Cylinder + working-directory: ./Solver/test/NavierStokes/IBM_Cylinder + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=IBM_Cylinder + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns IBMCylinder.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 40) CYLINDER EntropyConservingCentral +# ------------------------------------- + - name: Build NSCylinderEntropyConservingCentral + working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderEntropyConservingCentral + working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderEntropyConservingCentral + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderEntropyConservingCentral.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 41) CYLINDER RoePikePirozzoli +# ---------------------------- + - name: Build NSCylinderRoePikePirozzoli + working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderRoePikePirozzoli + working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderRoePikePirozzoli + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderRoePikePirozzoli.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 42) CYLINDER KennedyGruberLaxFriedrichs +# --------------------------------------- + - name: Build NSCylinderKennedyGruberLaxFriedrichs + working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderKennedyGruberLaxFriedrichs + working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderKennedyGruberLaxFriedrichs + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderKennedyGruberLaxFriedrichs.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 43) CYLINDER MorinishiLowDissipationRoe +# ---------------------------------------- + - name: Build NSCylinderMorinishiLowDissipationRoe + working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderMorinishiLowDissipationRoe + working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderMorinishiLowDissipationRoe + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderMorinishiLowDissipationRoe.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 44) CYLINDER DucrosMatrixDissipation +# ------------------------------------- + - name: Build NSCylinderDucrosMatrixDissipation + working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderDucrosMatrixDissipation + working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderDucrosMatrixDissipation + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderDucrosMatrixDissipation.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 45) CYLINDER ViscousStandard +# ------------------------- + - name: Build NSCylinderViscousStandard + working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderViscousStandard + working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderViscousStandard + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderViscousStandard.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + +# +# 46) CYLINDER RusanovStandard +# ------------------------- + - name: Build NSCylinderRusanovStandard + working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderRusanovStandard + working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderRusanovStandard + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderRusanovStandard.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 47) CYLINDER UdissStandard +# ------------------------- + - name: Build NSCylinderUdissStandard + working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderUdissStandard + working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderUdissStandard + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns CylinderUdissStandard.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 48) CYLINDER sprk33 +# ------------------- + - name: Build NSCylinderssprk33 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk33/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderssprk33 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk33 + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderssprk33 + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns Cylinderssprk33.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +# +# 49) CYLINDER sprk43 +# ------------------- + - name: Build NSCylinderssprk43 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk43/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSCylinderssprk43 + working-directory: ./Solver/test/NavierStokes/Cylinderssprk43 + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSCylinderssprk43 + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns Cylinderssprk43.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' + + +# +# 50) Limiter Test +# ------------------- + - name: Build NSLimiterTest + working-directory: ./Solver/test/NavierStokes/LimiterTest/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make ns COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + - name: Run NSLimiterTest + working-directory: ./Solver/test/NavierStokes/LimiterTest + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=NSLimiterTest + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.ns LimiterTest.control + EOF + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + cat slurm-${job_id}.out + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' +######################################################################## +####### INCOMPRESSIBLE_NS TESTS ######## +######################################################################## + +### Sometimes this test passes and sometimes it failscc +# +# 1) NACA0012 +# ---------------------------- + + # - name: Build iNS-NACA0012 + # working-directory: ./Solver/test/IncompressibleNS/NACA0012/SETUP + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # make clean || true + # make ins COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + # if: '!cancelled()' + + # - name: Run iNS-NACA0012 + # working-directory: ./Solver/test/IncompressibleNS/NACA0012 + # run: | + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + # cat > run_test.sh << 'EOF' + # #!/bin/bash + # #SBATCH --job-name=NACA0012 + # #SBATCH --ntasks=1 + # #SBATCH --mem=20G + # #SBATCH --partition=gpu + # #SBATCH --gres=gpu:1 + # #SBATCH --time=00:15:00 + + # module purge + # module load hpc_sdk + # module load nvhpc/24.11 + # export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + # export CUDA_HOME=$NVHPC_CUDA_HOME + # export CUDA_PATH=$NVHPC_CUDA_HOME + # export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + # export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + # ./horses3d.ins NACA0012.control + # EOF + + # job_id=$(sbatch --parsable run_test.sh) + # while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + # cat slurm-${job_id}.out + + # exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + # if [ "$exit_code" != "0" ]; then + # echo "Test failed with exit code: $exit_code" + # exit 1 + # fi + # if: '!cancelled()' + + +######################################################################## +####### MULTIPHASE TESTS ######## +######################################################################## + +# +# 4) Entropy conserving test +# ---------------------------- + + - name: Build MultiphaseEntropyConservingTest + working-directory: ./Solver/test/Multiphase/EntropyConservingTest/SETUP + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + make clean || true + make mu COMPILER=nvfortran COMM=SERIAL ENABLE_THREADS=YES + if: '!cancelled()' + + - name: Run MultiphaseEntropyConservingTest + working-directory: ./Solver/test/Multiphase/EntropyConservingTest + run: | + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + + cat > run_test.sh << 'EOF' + #!/bin/bash + #SBATCH --job-name=MultiphaseEntropyConservingTest + #SBATCH --ntasks=1 + #SBATCH --mem=20G + #SBATCH --partition=gpu + #SBATCH --gres=gpu:1 + #SBATCH --time=00:15:00 + + module purge + module load hpc_sdk + module load nvhpc/24.11 + export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda + export CUDA_HOME=$NVHPC_CUDA_HOME + export CUDA_PATH=$NVHPC_CUDA_HOME + export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH + export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH + ./horses3d.mu EntropyConservingTest.control + EOF + + job_id=$(sbatch --parsable run_test.sh) + while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done + + cat slurm-${job_id}.out + + exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1) + if [ "$exit_code" != "0" ]; then + echo "Test failed with exit code: $exit_code" + exit 1 + fi + if: '!cancelled()' \ No newline at end of file diff --git a/.github/workflows/CI_serial_MU.yml b/.github/workflows/CI_serial_MU.yml index ae58d4241..bfe650722 100644 --- a/.github/workflows/CI_serial_MU.yml +++ b/.github/workflows/CI_serial_MU.yml @@ -34,7 +34,7 @@ jobs: compiler: ['gfortran'] mode: ['DEBUG'] comm: ['SEQUENTIAL'] - enable_threads: ['NO'] + enable_threads: ['NO','YES'] name: MU - ${{ matrix.compiler }} - ${{ matrix.mode }} - ${{ matrix.comm }} - ${{ matrix.enable_threads }} From eb627e11463398c1fb80c571a670714d28396230 Mon Sep 17 00:00:00 2001 From: Gonzalo Rubio Date: Wed, 8 Jul 2026 11:01:50 +0200 Subject: [PATCH 3/9] Fix name CI GPU OPENMP --- .github/workflows/CI_parallel_GPU_OPENMP.yml | 2 +- .github/workflows/CI_serial_GPU_OPENMP.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI_parallel_GPU_OPENMP.yml b/.github/workflows/CI_parallel_GPU_OPENMP.yml index 47d945b68..c06f8c557 100644 --- a/.github/workflows/CI_parallel_GPU_OPENMP.yml +++ b/.github/workflows/CI_parallel_GPU_OPENMP.yml @@ -1,4 +1,4 @@ -name: CI PARALLEL (MPI) GPU +name: CI PARALLEL (MPI) GPU OPENMP ######################################################################## on: diff --git a/.github/workflows/CI_serial_GPU_OPENMP.yml b/.github/workflows/CI_serial_GPU_OPENMP.yml index 572abf5b1..f12f945e1 100644 --- a/.github/workflows/CI_serial_GPU_OPENMP.yml +++ b/.github/workflows/CI_serial_GPU_OPENMP.yml @@ -1,4 +1,4 @@ -name: CI SERIAL GPU +name: CI SERIAL GPU OPENMP ######################################################################## on: From b6edf351c4ea66a69b269422c81ce7e0a19d563e Mon Sep 17 00:00:00 2001 From: Miguel Chavez Modena Date: Thu, 9 Jul 2026 12:06:59 +0200 Subject: [PATCH 4/9] Update missing chages --- .gitignore | 11 +- .../SpatialDiscretization.f90 | 183 +- .../src/NavierStokesSolver/split_template.inc | 88 + Solver/src/libs/mesh/HexElementClass.f90 | 95 +- Solver/src/libs/mesh/HexMesh.f90 | 81 +- Solver/src/libs/mesh/Monitors.f90 | 1503 +++++++++++++++++ Solver/src/libs/mesh/Probe.f90 | 967 +++++++++++ Solver/src/libs/mesh/SurfaceIntegrals.f90 | 1269 ++++++++++++++ Solver/src/libs/mesh/SurfaceMonitor.f90 | 560 ++++++ Solver/src/libs/sources/ActuatorLine.f90 | 1 - doc/basicManualAndTutorial.pdf | Bin 215966 -> 239299 bytes 11 files changed, 4540 insertions(+), 218 deletions(-) create mode 100644 Solver/src/NavierStokesSolver/split_template.inc create mode 100644 Solver/src/libs/mesh/Monitors.f90 create mode 100644 Solver/src/libs/mesh/Probe.f90 create mode 100644 Solver/src/libs/mesh/SurfaceIntegrals.f90 create mode 100644 Solver/src/libs/mesh/SurfaceMonitor.f90 diff --git a/.gitignore b/.gitignore index 281295a2e..ce380f3ce 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,13 @@ Solver/include/ horsesConverter #ford -doc/doc_output \ No newline at end of file +doc/doc_output +*.aux +*.fls +*.out +*.fdb_latexmk +*.gz +*.log +*.toc +*.blg +*.bbl diff --git a/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 b/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 index f076f5acb..1067a050f 100644 --- a/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 +++ b/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 @@ -1447,126 +1447,91 @@ end subroutine TimeDerivative_VolumetricContribution ! !/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - subroutine TimeDerivative_VolumetricContribution_Split_StandardDG(mesh) - use HexMeshClass - use NodalStorageClass, only: NodalStorage - use ElementClass - use DGIntegrals - use RiemannSolvers_NS, only: twopointflux_selector => StandardDG_TwoPointFlux - implicit none - type(HexMesh), intent(inout) :: mesh - integer :: i, j, k, l, eq, eID - real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc - include 'split_template.inc' - end subroutine - - subroutine TimeDerivative_VolumetricContribution_Split_Morinishi(mesh) - use HexMeshClass - use NodalStorageClass, only: NodalStorage - use ElementClass - use DGIntegrals - use RiemannSolvers_NS, only: twopointflux_selector => Morinishi_TwoPointFlux - implicit none - type(HexMesh), intent(inout) :: mesh - integer :: i, j, k, l, eq, eID - real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc - include 'split_template.inc' - end subroutine +! - subroutine TimeDerivative_VolumetricContribution_Split_Ducros(mesh) + subroutine TimeDerivative_VolumetricContribution_Split(mesh) use HexMeshClass - use NodalStorageClass, only: NodalStorage + use NodalStorageClass, only: NodalStorage use ElementClass use DGIntegrals - use RiemannSolvers_NS, only: twopointflux_selector => Ducros_TwoPointFlux + use RiemannSolvers_NS implicit none - type(HexMesh), intent(inout) :: mesh - integer :: i, j, k, l, eq, eID - real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc - include 'split_template.inc' - end subroutine + type(HexMesh), intent (inout) :: mesh +! +! --------------- +! Local variables +! --------------- +! + integer :: i, j, k,l,eq, eID + real(kind=RP) :: Flux(1:NCONS, 1:NDIM) - subroutine TimeDerivative_VolumetricContribution_Split_KennedyGruber(mesh) - use HexMeshClass - use NodalStorageClass, only: NodalStorage - use ElementClass - use DGIntegrals - use RiemannSolvers_NS, only: twopointflux_selector => KennedyGruber_TwoPointFlux - implicit none - type(HexMesh), intent(inout) :: mesh - integer :: i, j, k, l, eq, eID - real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc - include 'split_template.inc' - end subroutine + !$acc parallel present(mesh) vector_length(128) num_gangs(9750) async(1) + !$acc loop gang + do eID = 1 , size(mesh % elements) + + !$acc loop vector collapse(3) private(Flux) + do k = 0, mesh % elements(eID) % Nxyz(3) + do j = 0, mesh % elements(eID) % Nxyz(2) + do i = 0, mesh % elements(eID) % Nxyz(1) + + call ViscousFlux_STATE( NCONS, NGRAD, mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % U_x(:,i,j,k), & + mesh % elements(eID) % storage % U_y(:,i,j,k) , mesh % elements(eID) % storage % U_z(:,i,j,k), & + mesh % elements(eID) % storage % mu_ns(1,i,j,k), 0.0_RP, & + mesh % elements(eID) % storage % mu_ns(2,i,j,k), Flux) + !$acc loop seq + do eq = 1, NCONS + + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IX) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradXi(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradXi(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradXi(IZ,i,j,k) - subroutine TimeDerivative_VolumetricContribution_Split_Pirozzoli(mesh) - use HexMeshClass - use NodalStorageClass, only: NodalStorage - use ElementClass - use DGIntegrals - use RiemannSolvers_NS, only: twopointflux_selector => Pirozzoli_TwoPointFlux - implicit none - type(HexMesh), intent(inout) :: mesh - integer :: i, j, k, l, eq, eID - real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc - include 'split_template.inc' - end subroutine + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IY) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradEta(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradEta(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradEta(IZ,i,j,k) + + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IZ) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradZeta(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradZeta(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradZeta(IZ,i,j,k) + ! initialize to 0 to accumulate + mesh % elements(eID) % storage % Qdot(eq,i,j,k) = 0.0_RP + end do + end do ; end do ; end do - subroutine TimeDerivative_VolumetricContribution_Split_EntropyConserving(mesh) - use HexMeshClass - use NodalStorageClass, only: NodalStorage - use ElementClass - use DGIntegrals - use RiemannSolvers_NS, only: twopointflux_selector => EntropyConserving_TwoPointFlux - implicit none - type(HexMesh), intent(inout) :: mesh - integer :: i, j, k, l, eq, eID - real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc - include 'split_template.inc' - end subroutine + call ScalarWeakIntegrals_StdVolumeGreen( mesh % elements(eID) % Nxyz, NCONS, mesh % elements(eID) % storage % contravariantFlux, & + mesh % elements(eID) % storage % QDot) +! +! ************************************* +! Compute interior contravariant fluxes +! ************************************* +! +! Compute inviscid contravariant flux +! ----------------------------------- + !$acc loop vector collapse(3) private(Flux) + do k = 0, mesh % elements(eID) % Nxyz(3) + do j = 0, mesh % elements(eID) % Nxyz(2) + do i = 0, mesh % elements(eID) % Nxyz(1) + !$acc loop seq + do l = 0, mesh % elements(eID) % Nxyz(1) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,l,j,k), mesh % elements(eID) % geom % jGradXi(:,i,j,k), mesh % elements(eID) % geom % jGradXi(:,l,j,k), Flux(:,IX)) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,i,l,k), mesh % elements(eID) % geom % jGradEta(:,i,j,k), mesh % elements(eID) % geom % jGradEta(:,i,l,k), Flux(:,IY)) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,i,j,l), mesh % elements(eID) % geom % jGradZeta(:,i,j,k), mesh % elements(eID) % geom % jGradZeta(:,i,j,l), Flux(:,IZ)) + + !$acc loop seq + do eq = 1, NCONS + mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) & + - NodalStorage(mesh % elements(eID) % Nxyz(1)) % sharpD(i,l) * Flux(eq,IX) & + - NodalStorage(mesh % elements(eID) % Nxyz(2)) % sharpD(j,l) * Flux(eq,IY) & + - NodalStorage(mesh % elements(eID) % Nxyz(3)) % sharpD(k,l) * Flux(eq,IZ) + end do + end do - subroutine TimeDerivative_VolumetricContribution_Split_Chandrasekar(mesh) - use HexMeshClass - use NodalStorageClass, only: NodalStorage - use ElementClass - use DGIntegrals - use RiemannSolvers_NS, only: twopointflux_selector => Chandrasekar_TwoPointFlux - implicit none - type(HexMesh), intent(inout) :: mesh - integer :: i, j, k, l, eq, eID - real(kind=RP) :: Flux(1:NCONS, 1:NDIM), Q_acc - include 'split_template.inc' - end subroutine + end do ; end do ; end do - subroutine TimeDerivative_VolumetricContribution_Split(mesh) - use HexMeshClass - use RiemannSolvers_NSKeywordsModule - use RiemannSolvers_NS, only : whichAverage - implicit none - type(HexMesh), intent(inout) :: mesh - - select case (whichAverage) - case (STANDARD_AVG) - call TimeDerivative_VolumetricContribution_Split_StandardDG(mesh) - case (MORINISHI_AVG) - call TimeDerivative_VolumetricContribution_Split_Morinishi(mesh) - case (DUCROS_AVG) - call TimeDerivative_VolumetricContribution_Split_Ducros(mesh) - case (KENNEDYGRUBER_AVG) - call TimeDerivative_VolumetricContribution_Split_KennedyGruber(mesh) - case (PIROZZOLI_AVG) - call TimeDerivative_VolumetricContribution_Split_Pirozzoli(mesh) - case (ENTROPYCONS_AVG) - call TimeDerivative_VolumetricContribution_Split_EntropyConserving(mesh) - case (CHANDRASEKAR_AVG) - call TimeDerivative_VolumetricContribution_Split_Chandrasekar(mesh) - case default - print*, "Averaging not recognized." - errorMessage(STD_OUT) - error stop - end select - end subroutine + enddo + !$acc end parallel loop + end subroutine TimeDerivative_VolumetricContribution_Split +! !/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ! subroutine TimeDerivative_FacesContribution(e, mesh) diff --git a/Solver/src/NavierStokesSolver/split_template.inc b/Solver/src/NavierStokesSolver/split_template.inc new file mode 100644 index 000000000..40ddac7bc --- /dev/null +++ b/Solver/src/NavierStokesSolver/split_template.inc @@ -0,0 +1,88 @@ + !$acc parallel present(mesh) vector_length(128) num_gangs(9750) async(1) + !$acc loop gang + do eID = 1 , size(mesh % elements) + + !$acc loop vector collapse(3) private(Flux) + do k = 0, mesh % elements(eID) % Nxyz(3) + do j = 0, mesh % elements(eID) % Nxyz(2) + do i = 0, mesh % elements(eID) % Nxyz(1) + + call ViscousFlux_STATE( NCONS, NGRAD, mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % U_x(:,i,j,k), & + mesh % elements(eID) % storage % U_y(:,i,j,k) , mesh % elements(eID) % storage % U_z(:,i,j,k), & + mesh % elements(eID) % storage % mu_ns(1,i,j,k), 0.0_RP, & + mesh % elements(eID) % storage % mu_ns(2,i,j,k), Flux) + !$acc loop seq + do eq = 1, NCONS + + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IX) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradXi(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradXi(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradXi(IZ,i,j,k) + + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IY) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradEta(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradEta(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradEta(IZ,i,j,k) + + mesh % elements(eID) % storage % contravariantFlux(eq,i,j,k,IZ) = - Flux(eq,IX) * mesh % elements(eID) % geom % jGradZeta(IX,i,j,k) & + - Flux(eq,IY) * mesh % elements(eID) % geom % jGradZeta(IY,i,j,k) & + - Flux(eq,IZ) * mesh % elements(eID) % geom % jGradZeta(IZ,i,j,k) + ! initialize to 0 to accumulate + mesh % elements(eID) % storage % Qdot(eq,i,j,k) = 0.0_RP + end do + end do ; end do ; end do + + call ScalarWeakIntegrals_StdVolumeGreen( mesh % elements(eID) % Nxyz, NCONS, mesh % elements(eID) % storage % contravariantFlux, & + mesh % elements(eID) % storage % QDot) +! +! ************************************* +! Compute interior contravariant fluxes +! ************************************* +! +! Compute inviscid contravariant flux +! ----------------------------------- + !$acc loop vector collapse(3) private(Flux) + do k = 0, mesh % elements(eID) % Nxyz(3) + do j = 0, mesh % elements(eID) % Nxyz(2) + do i = 0, mesh % elements(eID) % Nxyz(1) + if ( mesh % elements(eID) % Nxyz(1) > 0 ) then + !$acc loop seq + do l = 0, mesh % elements(eID) % Nxyz(1) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,l,j,k), mesh % elements(eID) % geom % jGradXi(:,i,j,k), mesh % elements(eID) % geom % jGradXi(:,l,j,k), Flux(:,IX)) + + !$acc loop seq + do eq = 1, NCONS + mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) & + - NodalStorage(mesh % elements(eID) % Nxyz(1)) % sharpD(i,l) * Flux(eq,IX) + end do + end do + end if + + if ( mesh % elements(eID) % Nxyz(2) > 0 ) then + !$acc loop seq + do l = 0, mesh % elements(eID) % Nxyz(2) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,i,l,k), mesh % elements(eID) % geom % jGradEta(:,i,j,k), mesh % elements(eID) % geom % jGradEta(:,i,l,k), Flux(:,IY)) + + !$acc loop seq + do eq = 1, NCONS + mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) & + - NodalStorage(mesh % elements(eID) % Nxyz(2)) % sharpD(j,l) * Flux(eq,IY) + end do + end do + end if + + if ( mesh % elements(eID) % Nxyz(3) > 0 ) then + !$acc loop seq + do l = 0, mesh % elements(eID) % Nxyz(3) + call TwoPointFlux_Selector(mesh % elements(eID) % storage % Q(:,i,j,k), mesh % elements(eID) % storage % Q(:,i,j,l), mesh % elements(eID) % geom % jGradZeta(:,i,j,k), mesh % elements(eID) % geom % jGradZeta(:,i,j,l), Flux(:,IZ)) + + !$acc loop seq + do eq = 1, NCONS + mesh % elements(eID) % storage % QDot(eq,i,j,k) = mesh % elements(eID) % storage % QDot(eq,i,j,k) & + - NodalStorage(mesh % elements(eID) % Nxyz(3)) % sharpD(k,l) * Flux(eq,IZ) + end do + end do + end if + + end do ; end do ; end do + + enddo + !$acc end parallel loop diff --git a/Solver/src/libs/mesh/HexElementClass.f90 b/Solver/src/libs/mesh/HexElementClass.f90 index 3c03413e2..86576a189 100644 --- a/Solver/src/libs/mesh/HexElementClass.f90 +++ b/Solver/src/libs/mesh/HexElementClass.f90 @@ -35,12 +35,7 @@ Module ElementClass private public Element - public HexElement_ComputeLocalGradient, HexElement_ProlongSolToFaces, & - HexElement_ProlongGradientsToFaces, HexElement_ProlongGradientsToFaces_GL, & - HexElement_ProlongSolToFaces_GL -#ifdef NAVIERSTOKES - public HexElement_NSGradientVariables -#endif + public HexElement_ComputeLocalGradient, HexElement_ProlongSolToFaces, HexElement_ProlongGradientsToFaces, HexElement_ProlongGradientsToFaces_GL, HexElement_ProlongSolToFaces_GL public PrintElement, SetElementBoundaryNames, SurfInfo_t ! @@ -346,50 +341,50 @@ subroutine HexElement_ProlongSolToFaces(self, nEqn, f_side, side) case(1) !$acc loop vector collapse(2) private(Qlocal) - do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) - Qlocal(1:nEqn)= self % storage % Q(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(2)) % v(0,FRONT) + do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) + Qlocal(1:nEqn)= self % storage % Q(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(1)) % v(0,FRONT) !$acc loop seq - do j = 1, self % Nxyz(2) + do j = 1, self % Nxyz(3) !$acc loop seq do eq = 1, nEqn - Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(2)) % v(j,FRONT) + Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(j,FRONT) enddo end do f_side % storage(self % faceSide(EFRONT)) % Q_aux(1:nEqn,i,k) = Qlocal(1:nEqn) end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT)) case(2) !$acc loop vector collapse(2) private(Qlocal) - do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) - Qlocal(1:nEqn) = self % storage % Q(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(2)) % v(0,BACK) + do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) + Qlocal(1:nEqn) = self % storage % Q(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(1)) % v(0,BACK) !$acc loop seq - do j = 1, self % Nxyz(2) + do j = 1, self % Nxyz(3) !$acc loop seq do eq = 1, nEqn - Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(2)) % v(j,BACK) + Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(j,BACK) enddo end do f_side % storage(self % faceSide(EBACK)) % Q_aux(1:nEqn,i,k) = Qlocal(1:nEqn) end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK)) case(3) !$acc loop vector collapse(2) private(Qlocal) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Qlocal(1:nEqn)= self % storage % Q(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(3)) % v(0,BOTTOM) + Qlocal(1:nEqn)= self % storage % Q(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(1)) % v(0,BOTTOM) !$acc loop seq do k = 1, self % Nxyz(3) !$acc loop seq do eq = 1, nEqn - Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(3)) % v(k,BOTTOM) + Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(k,BOTTOM) enddo end do f_side % storage(self % faceSide(EBOTTOM)) % Q_aux(1:nEqn,i,j) = Qlocal(1:nEqn) end do ; end do - call Face_AdaptSolToFace(f_side,nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM)) + call Face_AdaptSolToFace(f_side,nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM)) case(4) @@ -410,17 +405,17 @@ subroutine HexElement_ProlongSolToFaces(self, nEqn, f_side, side) case(5) !$acc loop vector collapse(2) private(Qlocal) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Qlocal(1:nEqn) = self % storage % Q(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(3)) % v(0,TOP) + Qlocal(1:nEqn) = self % storage % Q(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(1)) % v(0,TOP) !$acc loop seq do k = 1, self % Nxyz(3) !$acc loop seq do eq = 1, nEqn - Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(3)) % v(k,TOP) + Qlocal(eq) = Qlocal(eq) + self % storage % Q(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(k,TOP) enddo end do f_side % storage(self % faceSide(ETOP)) % Q_aux(1:nEqn,i,j) = Qlocal(1:nEqn) end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP)) case(6) @@ -464,20 +459,20 @@ subroutine HexElement_ProlongSolToFaces_GL(self, nEqn, f_side, side) select case(side) case(1) !$acc loop vector collapse(3) - do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) + do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) do eq = 1, nEqn f_side % storage(self % faceSide(EFRONT)) % Q_aux(eq,i,k) = self % storage % Q(eq,i,0,k) enddo end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT)) case(2) !$acc loop vector collapse(3) - do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) + do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) do eq = 1, nEqn f_side % storage(self % faceSide(EBACK)) % Q_aux(eq,i,k) = self % storage % Q(eq,i,self % Nxyz(2),k) enddo end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK)) case(3) !$acc loop vector collapse(3) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) @@ -485,7 +480,7 @@ subroutine HexElement_ProlongSolToFaces_GL(self, nEqn, f_side, side) f_side % storage(self % faceSide(EBOTTOM)) % Q_aux(eq,i,j) = self % storage % Q(eq,i,j,0) enddo end do ; end do - call Face_AdaptSolToFace(f_side,nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM)) + call Face_AdaptSolToFace(f_side,nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM)) case(4) !$acc loop vector collapse(3) @@ -503,7 +498,7 @@ subroutine HexElement_ProlongSolToFaces_GL(self, nEqn, f_side, side) f_side % storage(self % faceSide(ETOP)) % Q_aux(eq,i,j) = self % storage % Q(eq,i,j,self % Nxyz(3)) enddo end do ; end do - call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP)) + call Face_AdaptSolToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP)) case(6) !$acc loop vector collapse(3) @@ -542,12 +537,12 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d select case(side) case(1) !$acc loop vector collapse(2) private(Q_grad) - do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) - Q_grad(1:nEqn)= U_xyz(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(2)) % v(0,FRONT) + do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) + Q_grad(1:nEqn)= U_xyz(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(1)) % v(0,FRONT) !$acc loop seq - do j = 1, self % Nxyz(2) + do j = 1, self % Nxyz(3) do eq = 1, nEqn - Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(2)) % v(j,FRONT) + Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(j,FRONT) enddo end do !$acc loop seq @@ -555,16 +550,16 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d f_side % storage(self % faceSide(EFRONT)) % Q_aux(eq,i,k) = Q_grad(eq) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT), dir) case(2) !$acc loop vector collapse(2) private(Q_grad) - do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) - Q_grad(1:nEqn) = U_xyz(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(2)) % v(0,BACK) + do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) + Q_grad(1:nEqn) = U_xyz(1:nEqn,i,0,k)* NodalStorage(self % Nxyz(1)) % v(0,BACK) !$acc loop seq - do j = 1, self % Nxyz(2) + do j = 1, self % Nxyz(3) do eq = 1, nEqn - Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(2)) % v(j,BACK) + Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(j,BACK) enddo end do !$acc loop seq @@ -572,16 +567,16 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d f_side % storage(self % faceSide(EBACK)) % Q_aux(eq,i,k) = Q_grad(eq) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK), dir) case(3) !$acc loop vector collapse(2) private(Q_grad) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Q_grad(1:nEqn)= U_xyz(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(3)) % v(0,BOTTOM) + Q_grad(1:nEqn)= U_xyz(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(1)) % v(0,BOTTOM) !$acc loop seq do k = 1, self % Nxyz(3) do eq = 1, nEqn - Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(3)) % v(k,BOTTOM) + Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(k,BOTTOM) enddo end do !$acc loop seq @@ -589,7 +584,7 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d f_side % storage(self % faceSide(EBOTTOM)) % Q_aux(eq,i,j) = Q_grad(eq) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side,nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM), dir) + call Face_AdaptGradientsToFace(f_side,nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM), dir) case(4) !$acc loop vector collapse(2) private(Q_grad) @@ -611,11 +606,11 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d case(5) !$acc loop vector collapse(2) private(Q_grad) do j = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) - Q_grad(1:nEqn) = U_xyz(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(3)) % v(0,TOP) + Q_grad(1:nEqn) = U_xyz(1:nEqn,i,j,0)* NodalStorage(self % Nxyz(1)) % v(0,TOP) !$acc loop seq do k = 1, self % Nxyz(3) do eq = 1, nEqn - Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(3)) % v(k,TOP) + Q_grad(eq) = Q_grad(eq) + U_xyz(eq,i,j,k)* NodalStorage(self % Nxyz(1)) % v(k,TOP) enddo end do !$acc loop seq @@ -623,7 +618,7 @@ subroutine HexElement_ProlongGradientsToFaces(self, nEqn, f_side, U_xyz, side, d f_side % storage(self % faceSide(ETOP)) % Q_aux(eq,i,j) = Q_grad(eq) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP), dir) case(6) !$acc loop vector collapse(2) private(Q_grad) @@ -669,20 +664,20 @@ subroutine HexElement_ProlongGradientsToFaces_GL(self, nEqn, f_side, U_xyz, side select case(side) case(1) !$acc loop vector collapse(3) - do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) + do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) do eq = 1, nEqn f_side % storage(self % faceSide(EFRONT)) % Q_aux(eq,i,k) = U_xyz(eq,i,0,k) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EFRONT)) % Q_aux, self % faceSide(EFRONT), dir) case(2) !$acc loop vector collapse(3) - do k = 0, self % Nxyz(3) ; do i = 0, self % Nxyz(1) + do k = 0, self % Nxyz(2) ; do i = 0, self % Nxyz(1) do eq = 1, nEqn f_side % storage(self % faceSide(EBACK)) % Q_aux(eq,i,k) = U_xyz(eq,i,self % Nxyz(2),k) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBACK)) % Q_aux, self % faceSide(EBACK), dir) case(3) !$acc loop vector collapse(3) @@ -691,7 +686,7 @@ subroutine HexElement_ProlongGradientsToFaces_GL(self, nEqn, f_side, U_xyz, side f_side % storage(self % faceSide(EBOTTOM)) % Q_aux(eq,i,j) = U_xyz(eq,i,j,0) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side,nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM), dir) + call Face_AdaptGradientsToFace(f_side,nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(EBOTTOM)) % Q_aux, self % faceSide(EBOTTOM), dir) case(4) !$acc loop vector collapse(3) @@ -709,7 +704,7 @@ subroutine HexElement_ProlongGradientsToFaces_GL(self, nEqn, f_side, U_xyz, side f_side % storage(self % faceSide(ETOP)) % Q_aux(eq,i,j) = U_xyz(eq,i,j,self % Nxyz(3)) enddo end do ; end do - call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(1), self % Nxyz(2), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP), dir) + call Face_AdaptGradientsToFace(f_side, nEqn, self % Nxyz(2), self % Nxyz(3), f_side % storage(self % faceSide(ETOP)) % Q_aux, self % faceSide(ETOP), dir) case(6) !$acc loop vector collapse(3) diff --git a/Solver/src/libs/mesh/HexMesh.f90 b/Solver/src/libs/mesh/HexMesh.f90 index a5eef59e6..a96aa7b26 100644 --- a/Solver/src/libs/mesh/HexMesh.f90 +++ b/Solver/src/libs/mesh/HexMesh.f90 @@ -4171,8 +4171,6 @@ subroutine HexMesh_ComputeWallDistances(self,facesList,elementList) else num_of_elems = self % no_of_elements end if - - !$acc data copyin(self, Xwall) if(no_of_wallDOFS .gt. 0) do ii = 1, num_of_elems if ( present(elementList) ) then eID = elementList (ii) @@ -4187,34 +4185,18 @@ subroutine HexMesh_ComputeWallDistances(self,facesList,elementList) endif if( .not. self% IBM% active ) then - if (no_of_wallDOFS .ne. 0) then - !$acc data copyin(self % elements(eID), self % elements(eID) % geom, self % elements(eID) % geom % x) & - !$acc& copyout(self % elements(eID) % geom % dWall) - !$acc parallel loop gang collapse(3) present(self, Xwall) & - !$acc& private(xP, currentDistance, minimumDistance) vector_length(128) - do k = 0, self % elements(eID) % Nxyz(3) - do j = 0, self % elements(eID) % Nxyz(2) - do i = 0, self % elements(eID) % Nxyz(1) - xP = self % elements(eID) % geom % x(:,i,j,k) - - minimumDistance = HUGE(1.0_RP) - !$acc loop vector reduction(min:minimumDistance) private(currentDistance) - do fID = 1, no_of_wallDOFS - currentDistance = POW2(xP(1) - Xwall(1,fID)) + & - POW2(xP(2) - Xwall(2,fID)) + & - POW2(xP(3) - Xwall(3,fID)) - minimumDistance = min(minimumDistance, currentDistance) - end do - - self % elements(eID) % geom % dWall(i,j,k) = sqrt(minimumDistance) - end do - end do + do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) + xP = e % geom % x(:,i,j,k) + + minimumDistance = HUGE(1.0_RP) + do fID = 1, no_of_wallDOFS + currentDistance = sum(POW2(xP - Xwall(:,fID))) + minimumDistance = min(minimumDistance, currentDistance) end do - !$acc end parallel loop - !$acc end data - else - e % geom % dWall = huge(1.0_RP) - end if + + e % geom % dWall(i,j,k) = sqrt(minimumDistance) + + end do ; end do ; end do end if end associate @@ -4241,37 +4223,22 @@ subroutine HexMesh_ComputeWallDistances(self,facesList,elementList) endif if( .not. self% IBM% active ) then - if (no_of_wallDOFS .ne. 0) then - !$acc data copyin(self % faces(eID), self % faces(eID) % geom, self % faces(eID) % geom % x) & - !$acc& copyout(self % faces(eID) % geom % dWall) - !$acc parallel loop gang collapse(2) present(self, Xwall) & - !$acc& private(xP, currentDistance, minimumDistance) vector_length(128) - do j = 0, self % faces(eID) % Nf(2) - do i = 0, self % faces(eID) % Nf(1) - xP = self % faces(eID) % geom % x(:,i,j) - - minimumDistance = HUGE(1.0_RP) - !$acc loop vector reduction(min:minimumDistance) private(currentDistance) - do fID = 1, no_of_wallDOFS - currentDistance = POW2(xP(1) - Xwall(1,fID)) + & - POW2(xP(2) - Xwall(2,fID)) + & - POW2(xP(3) - Xwall(3,fID)) - minimumDistance = min(minimumDistance, currentDistance) - end do - - self % faces(eID) % geom % dWall(i,j) = sqrt(minimumDistance) - end do + do j = 0, fe % Nf(2) ; do i = 0, fe % Nf(1) + xP = fe % geom % x(:,i,j) + + minimumDistance = HUGE(1.0_RP) + do fID = 1, no_of_wallDOFS + currentDistance = sum(POW2(xP - Xwall(:,fID))) + minimumDistance = min(minimumDistance, currentDistance) end do - !$acc end parallel loop - !$acc end data - else - fe % geom % dWall = huge(1.0_RP) - end if - endif + + fe % geom % dWall(i,j) = sqrt(minimumDistance) + + end do ; end do + end if end associate end do - !$acc end data deallocate(Xwall) @@ -5849,4 +5816,4 @@ subroutine HexMesh_ComputeLocalGradientMU(self, set_mu) end subroutine HexMesh_ComputeLocalGradientMU #endif -END MODULE HexMeshClass \ No newline at end of file +END MODULE HexMeshClass diff --git a/Solver/src/libs/mesh/Monitors.f90 b/Solver/src/libs/mesh/Monitors.f90 new file mode 100644 index 000000000..eaa4f1cff --- /dev/null +++ b/Solver/src/libs/mesh/Monitors.f90 @@ -0,0 +1,1503 @@ +#include "Includes.h" +module MonitorsClass + use SMConstants + use NodalStorageClass + use HexMeshClass + use MonitorDefinitions + use ResidualsMonitorClass + use VolumeMonitorClass + use LoadBalancingMonitorClass + use FileReadingUtilities , only: getFileName +#ifdef FLOW + use ProbeClass +#endif +#if defined(NAVIERSTOKES) || defined(INCNS) + use StatisticsMonitor + use SurfaceMonitorClass +#endif + implicit none +! + + private + public Monitor_t +! +! ***************************** +! Main monitor class definition +! ***************************** +! + type Monitor_t + character(len=LINE_LENGTH) :: solution_file + character(len=LINE_LENGTH) :: probes_solution_file = "" + integer :: no_of_probes + integer :: no_of_surfaceMonitors + integer :: no_of_volumeMonitors + integer :: no_of_loadBalancingMonitors + integer :: no_of_fileProbes = 0 + character(len=LINE_LENGTH) :: probesFileName = "" + character(len=STR_LEN_MONITORS), allocatable :: probesVariables(:) + real(kind=RP) :: probeFileSaveTimestep = 0.0_RP + character(len=8) :: probeFileOutputFormat = "ASCII" + real(kind=RP) :: fp_lastSavedTime = -huge(0.0_RP) + integer :: bufferLine + integer , allocatable :: iter(:) + integer :: dt_restriction + logical :: write_dt_restriction + real(kind=RP) , allocatable :: t(:) + real(kind=RP) , allocatable :: SolverSimuTime(:) + real(kind=RP) , allocatable :: TotalSimuTime(:) + type(Residuals_t) :: residuals + class(VolumeMonitor_t) , allocatable :: volumeMonitors(:) + class(LoadBalancingMonitor_t), allocatable :: loadBalancingMonitors(:) +#ifdef FLOW + class(Probe_t) , allocatable :: probes(:) +#endif +#if defined(NAVIERSTOKES) || defined(INCNS) + class(SurfaceMonitor_t) , allocatable :: surfaceMonitors(:) + type(StatisticsMonitor_t) :: stats +#endif + contains + procedure :: Construct => Monitors_Construct + procedure :: WriteLabel => Monitor_WriteLabel + procedure :: WriteUnderlines => Monitor_WriteUnderlines + procedure :: WriteValues => Monitor_WriteValues + procedure :: UpdateValues => Monitor_UpdateValues + procedure :: WriteToFile => Monitor_WriteToFile + procedure :: WriteProbesFileSummary => Monitor_WriteProbesFileSummary + procedure :: destruct => Monitor_Destruct + procedure :: copy => Monitor_Assign + generic :: assignment(=) => copy + end type Monitor_t +! +! ======== + contains +! ======== +! +!/////////////////////////////////////////////////////////////////////////////////////// +! + subroutine Monitors_Construct( Monitors, mesh, controlVariables ) + use FTValueDictionaryClass + use mainKeywordsModule + use MPI_Process_Info + implicit none + class(Monitor_t) :: Monitors + class(HexMesh), intent(in) :: mesh + class(FTValueDictionary), intent(in) :: controlVariables + +! +! --------------- +! Local variables +! --------------- +! + integer :: fID , io + integer :: i + character(len=STR_LEN_MONITORS) :: line + character(len=STR_LEN_MONITORS) :: solution_file + logical, save :: FirstCall = .TRUE. + logical :: saveGradients + character(len=LINE_LENGTH) :: probesFileName + character(len=LINE_LENGTH) :: probesVariablesLine + character(len=STR_LEN_MONITORS), allocatable :: probesVariables(:) + integer :: no_of_fileProbes + integer :: no_of_probesVariables + real(kind=RP) :: probeFileSaveTimestep + character(len=8) :: probeFileOutputFormat + character(len=LINE_LENGTH) :: probes_solution_file + character(len=LINE_LENGTH) :: probes_dir + integer :: last_slash +! +! Setup the buffer +! ---------------- + if (controlVariables % containsKey("monitors flush interval") ) then + BUFFER_SIZE = controlVariables % integerValueForKey("monitors flush interval") + end if + + allocate ( Monitors % TotalSimuTime(BUFFER_SIZE), & + Monitors % SolverSimuTime(BUFFER_SIZE), & + Monitors % t(BUFFER_SIZE), & + Monitors % iter(BUFFER_SIZE) ) +! +! Get the solution file name +! -------------------------- + solution_file = controlVariables % stringValueForKey( solutionFileNameKey, requestedLength = STR_LEN_MONITORS ) +! +! Remove the *.hsol termination +! ----------------------------- + solution_file = trim(getFileName(solution_file)) + Monitors % solution_file = trim(solution_file) +! +! Build the probes output subdirectory: /probes/ +! ---------------------------------------------------------- + last_slash = index(trim(solution_file), '/', back=.true.) + if (last_slash .gt. 0) then + probes_dir = trim(solution_file(1:last_slash)) // "probes" + probes_solution_file = trim(solution_file(1:last_slash)) // "probes/" // trim(solution_file(last_slash+1:)) + else + probes_dir = "probes" + probes_solution_file = "probes/" // trim(solution_file) + end if + if (MPI_Process % isRoot) then + call execute_command_line('mkdir -p "' // trim(probes_dir) // '"', wait=.true.) + end if + Monitors % probes_solution_file = trim(probes_solution_file) +! +! Search in case file for probes, surface monitors, and volume monitors +! --------------------------------------------------------------------- + no_of_fileProbes = 0 + if (mesh % child) then ! Return doing nothing if this is a child mesh + Monitors % no_of_probes = 0 + Monitors % no_of_surfaceMonitors = 0 + Monitors % no_of_volumeMonitors = 0 + Monitors % no_of_loadBalancingMonitors = 0 + else + call getNoOfMonitors( Monitors % no_of_probes, Monitors % no_of_surfaceMonitors, Monitors % no_of_volumeMonitors, Monitors % no_of_loadBalancingMonitors ) +! +! Check for an additional probes definition file (one probe per +! line: "x y z variable1 [variable2 ...]"), allowing several +! variables to be sampled and saved for the same probe location +! --------------------------------------------------------------- +#ifdef FLOW + call readProbesFileBlock( probesFileName, probesVariablesLine, no_of_probesVariables, probeFileSaveTimestep, probeFileOutputFormat ) + + if ( len_trim(probesFileName) .gt. 0 ) then + call countProbesInFile( trim(probesFileName), no_of_fileProbes ) + + if ( len_trim(probesVariablesLine) .gt. 0 ) then + call splitIntoTokens( trim(probesVariablesLine), probesVariables, no_of_probesVariables ) + else + write(STD_OUT,*) "Error: 'variables' must be specified inside the '#define probe file' block." + stop + end if + end if +#endif + end if +! +! Initialize the Monitors class in the GPU +! ---------------------------------------- + !!$acc enter data copyin(Monitors) + !$acc update device(Monitors) + +! Pro tip: This is necessary to avoid the compiler doing behind the scenes copies of the class. +! Pro tip(cont): Its not really necessary for the class itself, but for the arrays inside the class. +! +! Initialize +! ---------- + call Monitors % residuals % Initialization( solution_file , FirstCall ) + + allocate ( Monitors % volumeMonitors ( Monitors % no_of_volumeMonitors ) ) + !$acc update device(Monitors) + do i = 1 , Monitors % no_of_volumeMonitors + call Monitors % volumeMonitors(i) % Initialization ( mesh , i, solution_file , FirstCall ) + end do + + allocate ( Monitors % loadBalancingMonitors ( Monitors % no_of_loadBalancingMonitors ) ) + !$acc update device(Monitors) + do i = 1 , Monitors % no_of_loadBalancingMonitors + call Monitors % loadBalancingMonitors(i) % Initialization ( mesh , i, solution_file , FirstCall ) + end do + +#ifdef FLOW + allocate ( Monitors % probes ( Monitors % no_of_probes + no_of_fileProbes ) ) + !$acc update device(Monitors) + do i = 1 , Monitors % no_of_probes + call Monitors % probes(i) % Initialization ( mesh , i, probes_solution_file , FirstCall ) + end do + + if ( no_of_fileProbes .gt. 0 ) then + call InitializeProbesFromFile( trim(probesFileName), Monitors % probes, Monitors % no_of_probes, & + mesh, probes_solution_file, FirstCall, probesVariables, probeFileSaveTimestep, & + probeFileOutputFormat ) + Monitors % probesFileName = trim(probesFileName) + Monitors % probeFileSaveTimestep = probeFileSaveTimestep + Monitors % probeFileOutputFormat = probeFileOutputFormat + Monitors % fp_lastSavedTime = -huge(0.0_RP) + allocate( Monitors % probesVariables(size(probesVariables)) ) + Monitors % probesVariables = probesVariables +#ifdef HAS_HDF5 + if ( trim(probeFileOutputFormat) .eq. "HDF5" ) then + call Monitor_InitFileProbesHDF5( Monitors, no_of_fileProbes ) + end if +#endif + end if + + Monitors % no_of_fileProbes = no_of_fileProbes + Monitors % no_of_probes = Monitors % no_of_probes + no_of_fileProbes +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) + saveGradients = controlVariables % logicalValueForKey(saveGradientsToSolutionKey) + call Monitors % stats % Construct(mesh, saveGradients) + + + allocate ( Monitors % surfaceMonitors ( Monitors % no_of_surfaceMonitors ) ) + !$acc update device(Monitors) + do i = 1 , Monitors % no_of_surfaceMonitors + call Monitors % surfaceMonitors(i) % Initialization ( mesh , i, solution_file , FirstCall ) + end do +#endif + + Monitors % write_dt_restriction = controlVariables % logicalValueForKey( "write dt restriction" ) + + Monitors % bufferLine = 0 + + FirstCall = .FALSE. +! +! Include the latest changes in the GPU +! ---------------------------------------- + !$acc update device(Monitors) + + end subroutine Monitors_Construct + + subroutine Monitor_WriteLabel ( self ) +! +! *************************************************** +! This subroutine prints the labels for the time +! integrator Display procedure. +! *************************************************** +! + use MPI_Process_Info + implicit none + class(Monitor_t) :: self + integer :: i + + if ( .not. MPI_Process % isRoot ) return +! +! Write "Iteration" and "Time" +! ---------------------------- + write ( STD_OUT , ' ( A10 ) ' , advance = "no" ) "Iteration" + write ( STD_OUT , ' ( 3X,A10 ) ' , advance = "no" ) "Time" +! +! Write residuals labels +! ---------------------- + call self % residuals % WriteLabel +! +! Write volume monitors labels +! ----------------------------- + do i = 1 , self % no_of_volumeMonitors + call self % volumeMonitors(i) % WriteLabel + end do +! +! Write load balancing monitor labels +! ------------------------------------ + do i = 1 , self % no_of_loadBalancingMonitors + call self % loadBalancingMonitors(i) % WriteLabel + end do + +#ifdef FLOW +! +! Write probes labels (file-based probes excluded for readability) +! --------------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes + call self % probes(i) % WriteLabel + end do +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) +! +! Write surface monitors labels +! ----------------------------- + do i = 1 , self % no_of_surfaceMonitors + call self % surfaceMonitors(i) % WriteLabel + end do + + call self % stats % WriteLabel +#endif +! +! Write label for dt restriction +! ------------------------------ + if (self % write_dt_restriction) write ( STD_OUT , ' ( 3X,A10 ) ' , advance = "no" ) "dt restr." + + write(STD_OUT , *) + + end subroutine Monitor_WriteLabel + + subroutine Monitor_WriteUnderlines( self ) +! +! ******************************************************** +! This subroutine displays the underlines for the +! time integrator Display procedure. +! ******************************************************** +! + use PhysicsStorage + use MPI_Process_Info + implicit none + class(Monitor_t) :: self +! +! --------------- +! Local variables +! --------------- +! + integer :: i, j + character(len=MONITOR_LENGTH), parameter :: dashes = "----------" + + if ( .not. MPI_Process % isRoot ) return +! +! Print dashes for "Iteration" and "Time" +! --------------------------------------- + write ( STD_OUT , ' ( A10 ) ' , advance = "no" ) trim ( dashes ) + write ( STD_OUT , ' ( 3X,A10 ) ' , advance = "no" ) trim ( dashes ) +! +! Print dashes for residuals +! -------------------------- + do i = 1 , NCONS + write(STD_OUT , '(3X,A10)' , advance = "no" ) trim(dashes) + end do +! +! Print dashes for volume monitors +! -------------------------------- + do i = 1 , self % no_of_volumeMonitors ; do j=1, size ( self % volumeMonitors(i) % values, 1 ) + write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % volumeMonitors(i) % monitorName ) + 2 ) ) + end do ; end do +! +! Print dashes for load balancing monitor +! -------------------------------------- + do i = 1 , self % no_of_loadBalancingMonitors ; do j=1, size ( self % loadBalancingMonitors(i) % values, 1 ) + write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % loadBalancingMonitors(i) % monitorName ) + 2 ) ) + end do ; end do + +#ifdef FLOW +! +! Print dashes for probes (file-based probes excluded for readability) +! ---------------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes + if ( self % probes(i) % active ) then + write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % probes(i) % monitorName ) + 2 ) ) + end if + end do +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) +! +! Print dashes for surface monitors +! --------------------------------- + do i = 1 , self % no_of_surfaceMonitors + write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % surfaceMonitors(i) % monitorName ) + 2 ) ) + end do + + if ( self % stats % state .ne. 0 ) write(STD_OUT,'(3X,A10)',advance="no") trim(dashes) +#endif + + +! +! Print dashes for dt restriction +! ------------------------------- + if (self % write_dt_restriction) write ( STD_OUT , ' ( 3X,A10 ) ' , advance = "no" ) trim ( dashes ) + + write(STD_OUT , *) + + end subroutine Monitor_WriteUnderlines + + subroutine Monitor_WriteValues ( self ) +! +! ******************************************************* +! This subroutine prints the values for the time +! integrator Display procedure. +! ******************************************************* +! + use MPI_Process_Info + implicit none + class(Monitor_t) :: self + integer :: i + + if ( .not. MPI_Process % isRoot ) return +! +! Print iteration and time +! ------------------------ + write ( STD_OUT , ' ( I10 ) ' , advance = "no" ) self % iter ( self % bufferLine ) + write ( STD_OUT , ' ( 1X,A,1X,ES10.3 ) ' , advance = "no" ) "|" , self % t ( self % bufferLine ) +! +! Print residuals +! --------------- + call self % residuals % WriteValues( self % bufferLine ) +! +! Print volume monitors +! --------------------- + do i = 1 , self % no_of_volumeMonitors + call self % volumeMonitors(i) % WriteValues ( self % bufferLine ) + end do +! +! Print load balancing monitors +! ----------------------------- + do i = 1 , self % no_of_loadBalancingMonitors + call self % loadBalancingMonitors(i) % WriteValues ( self % bufferLine ) + end do + +#ifdef FLOW +! +! Print probes (file-based probes excluded for readability) +! ---------------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes + call self % probes(i) % WriteValues ( self % bufferLine ) + end do +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) +! +! Print surface monitors +! ---------------------- + do i = 1 , self % no_of_surfaceMonitors + call self % surfaceMonitors(i) % WriteValues ( self % bufferLine ) + end do + + call self % stats % WriteValue +#endif +! +! Print dt restriction +! -------------------- + if (self % write_dt_restriction) then + select case (self % dt_restriction) + case (DT_FIXED) ; write ( STD_OUT , ' ( 1X,A,1X,A10) ' , advance = "no" ) "|" , 'Fixed' + case (DT_DIFF) ; write ( STD_OUT , ' ( 1X,A,1X,A10) ' , advance = "no" ) "|" , 'Diffusive' + case (DT_CONV) ; write ( STD_OUT , ' ( 1X,A,1X,A10) ' , advance = "no" ) "|" , 'Convective' + end select + end if + + write(STD_OUT , *) + + end subroutine Monitor_WriteValues + + subroutine Monitor_UpdateValues ( self, mesh, t , iter, maxResiduals, Autosave, dt ) +! +! *************************************************************** +! This subroutine updates the values for the residuals, +! for the probes, surface and volume monitors. +! *************************************************************** +! + use PhysicsStorage + use StopwatchClass + implicit none + class(Monitor_t) :: self + class(HexMesh) :: mesh + real(kind=RP) :: t + integer :: iter + real(kind=RP) :: maxResiduals(NCONS), dt + logical :: Autosave +! +! --------------- +! Local variables +! --------------- +! + integer :: i +! +! Move to next buffer line +! ------------------------ + self % bufferLine = self % bufferLine + 1 +! +! Save time, iteration and CPU-time +! ----------------------- + self % t ( self % bufferLine ) = t + self % iter ( self % bufferLine ) = iter + self % SolverSimuTime ( self % bufferLine ) = Stopwatch % ElapsedTime("Solver") + self % TotalSimuTime ( self % bufferLine ) = Stopwatch % ElapsedTime("TotalTime") +! +! Compute current residuals +! ------------------------- + call self % residuals % Update( mesh, maxResiduals, self % bufferLine ) +! +! Update volume monitors +! ---------------------- + do i = 1 , self % no_of_volumeMonitors + call self % volumeMonitors(i) % Update( mesh , self % bufferLine ) + end do +! +! Update load balancing monitors +! ------------------------------ + do i = 1 , self % no_of_loadBalancingMonitors + call self % loadBalancingMonitors(i) % Update( mesh , self % bufferLine ) + end do + +#ifdef FLOW +! +! Update standard probes (GPU path with per-probe MPI) +! ----------------------------------------------------- + do i = 1 , self % no_of_probes - self % no_of_fileProbes + call self % probes(i) % Update( mesh , self % bufferLine ) + end do +! +! Update file probes: compute into slot 1, then flush immediately. +! File probes use a size-1 values buffer to avoid large memory +! allocations when the number of probes is O(1e6). +! ---------------------------------------------------------- + if ( self % no_of_fileProbes .gt. 0 ) then + call Monitor_UpdateFileProbes( self, mesh, 1 ) + call Monitor_FlushFileProbesNow( self, t, iter ) + end if +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) +! +! Update surface monitors +! ----------------------- + do i = 1 , self % no_of_surfaceMonitors + call self % surfaceMonitors(i) % Update( mesh , self % bufferLine, iter, autosave, dt ) + end do +! +! Update statistics +! ----------------- + call self % stats % Update(mesh, iter, t, trim(self % solution_file) ) +#endif + +! +! Update dt restriction +! --------------------- + if (self % write_dt_restriction) self % dt_restriction = mesh % dt_restriction + + end subroutine Monitor_UpdateValues + + subroutine Monitor_WriteToFile ( self , mesh, force) +! +! ****************************************************************** +! This routine has a double behaviour: +! force = .true. -> Writes to file and resets buffers +! force = .false. -> Just writes to file if the buffer is full +! ****************************************************************** +! + use MPI_Process_Info + implicit none + class(Monitor_t) :: self + class(HexMesh) :: mesh + logical, optional :: force +! ------------------------------------------------ + integer :: i + logical :: forceVal + + if ( present ( force ) ) then + forceVal = force + + else + forceVal = .false. + + end if + + if ( forceVal ) then +! +! In this case the monitors are exported to their files and the buffer is reset +! ----------------------------------------------------------------------------- + call self % residuals % WriteToFile ( self % iter , self % t, self % TotalSimuTime, self % SolverSimuTime , self % bufferLine ) + + do i = 1 , self % no_of_volumeMonitors + call self % volumeMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + end do + + do i = 1 , self % no_of_loadBalancingMonitors + call self % loadBalancingMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + end do + +#ifdef FLOW + do i = 1 , self % no_of_probes - self % no_of_fileProbes + call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + end do +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) + do i = 1 , self % no_of_surfaceMonitors + call self % surfaceMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + end do +! +! Write statistics +! ---------------- + if ( self % bufferLine .eq. 0 ) then + i = 1 + else + i = self % bufferLine + end if + call self % stats % WriteFile(mesh, self % iter(i), self % t(i), self % solution_file) +#endif +! +! Reset buffer +! ------------ + self % bufferLine = 0 + + else +! +! The monitors are exported just if the buffer is full +! ---------------------------------------------------- + if ( self % bufferLine .eq. BUFFER_SIZE ) then + + call self % residuals % WriteToFile ( self % iter , self % t, self % TotalSimuTime, self % SolverSimuTime, BUFFER_SIZE ) + + do i = 1 , self % no_of_volumeMonitors + call self % volumeMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + end do + + do i = 1 , self % no_of_loadBalancingMonitors + call self % loadBalancingMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + end do + +#ifdef FLOW + do i = 1 , self % no_of_probes - self % no_of_fileProbes + call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + end do +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) + do i = 1 , self % no_of_surfaceMonitors + call self % surfaceMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) + end do +#endif +! +! Reset buffer +! ------------ + self % bufferLine = 0 + + end if + end if + + end subroutine Monitor_WriteToFile + + subroutine Monitor_WriteProbesFileSummary ( self ) +! +! ******************************************************************** +! Prints a startup-log summary of the bulk probes-file monitor, +! shown instead of including the file-based probes in the +! per-iteration screen log. +! ******************************************************************** +! + use Headers + use MPI_Process_Info + implicit none + class(Monitor_t) :: self +! +! --------------- +! Local variables +! --------------- +! + integer :: j + + if ( .not. MPI_Process % isRoot ) return + if ( self % no_of_fileProbes .le. 0 ) return + + write(STD_OUT,'(/)') + call Section_Header("Probes file monitor") + write(STD_OUT,'(/)') + + write(STD_OUT,'(30X,A,A28,A)') "->" , "File: " , trim(self % probesFileName) + write(STD_OUT,'(30X,A,A28,I10)') "->" , "Number of probes: " , self % no_of_fileProbes + write(STD_OUT,'(30X,A,A28)',advance="no") "->" , "Variables: " + do j = 1 , size(self % probesVariables) + write(STD_OUT,'(A)',advance="no") trim(self % probesVariables(j)) // " " + end do + write(STD_OUT,*) + if ( self % probeFileSaveTimestep .gt. 0.0_RP ) then + write(STD_OUT,'(30X,A,A28,ES14.6)') "->" , "Save timestep: " , self % probeFileSaveTimestep + end if + write(STD_OUT,'(30X,A,A28,A)') "->" , "Output format: " , trim(self % probeFileOutputFormat) + + end subroutine Monitor_WriteProbesFileSummary + + subroutine Monitor_Destruct (self) + implicit none + class(Monitor_t) :: self + + deallocate (self % iter) + deallocate (self % t) + deallocate (self % TotalSimuTime) + deallocate (self % SolverSimuTime) + + call self % residuals % destruct + + call self % volumeMonitors % destruct + safedeallocate(self % volumeMonitors) + + call self % loadBalancingMonitors % destruct + safedeallocate(self % loadBalancingMonitors) + +#ifdef FLOW + call self % probes % destruct + safedeallocate (self % probes) +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) + call self % surfaceMonitors % destruct + safedeallocate (self % surfaceMonitors) + + !call self % stats % destruct +#endif + end subroutine + + impure elemental subroutine Monitor_Assign ( to, from ) + implicit none + !-arguments-------------------------------------- + class(Monitor_t), intent(inout) :: to + type(Monitor_t) , intent(in) :: from + !-local-variables-------------------------------- + !------------------------------------------------ + + to % solution_file = from % solution_file + to % probes_solution_file = from % probes_solution_file + to % no_of_probes = from % no_of_probes + to % no_of_surfaceMonitors = from % no_of_surfaceMonitors + to % no_of_volumeMonitors = from % no_of_volumeMonitors + to % no_of_loadBalancingMonitors = from % no_of_loadBalancingMonitors + to % no_of_fileProbes = from % no_of_fileProbes + to % probesFileName = from % probesFileName + to % probeFileSaveTimestep = from % probeFileSaveTimestep + to % probeFileOutputFormat = from % probeFileOutputFormat + to % fp_lastSavedTime = from % fp_lastSavedTime + if ( allocated(from % probesVariables) ) then + safedeallocate ( to % probesVariables ) + allocate ( to % probesVariables ( size(from % probesVariables) ) ) + to % probesVariables = from % probesVariables + end if + to % bufferLine = from % bufferLine + + safedeallocate ( to % iter ) + allocate ( to % iter ( size(from % iter) ) ) + to % iter = from % iter + + to % dt_restriction = from % dt_restriction + to % write_dt_restriction = from % write_dt_restriction + + safedeallocate (to % t) + allocate (to % t (size (from % t) ) ) + to % t = from % t + + safedeallocate ( to % TotalSimuTime ) + allocate ( to % TotalSimuTime ( size(from % TotalSimuTime) ) ) + to % TotalSimuTime = from % TotalSimuTime + + safedeallocate ( to % SolverSimuTime ) + allocate ( to % SolverSimuTime ( size(from % SolverSimuTime) ) ) + to % SolverSimuTime = from % SolverSimuTime + + to % residuals = from % residuals + + safedeallocate ( to % volumeMonitors ) + allocate ( to % volumeMonitors ( size(from % volumeMonitors) ) ) + to % volumeMonitors = from % volumeMonitors + + safedeallocate ( to % loadBalancingMonitors ) + allocate ( to % loadBalancingMonitors ( size(from % loadBalancingMonitors) ) ) + to % loadBalancingMonitors = from % loadBalancingMonitors + +#ifdef FLOW + safedeallocate ( to % probes ) + allocate ( to % probes ( size(from % probes) ) ) + to % probes = from % probes +#endif + +#if defined(NAVIERSTOKES) || defined(INCNS) + safedeallocate ( to % surfaceMonitors ) + allocate ( to % surfaceMonitors ( size(from % surfaceMonitors) ) ) + to % surfaceMonitors = from % surfaceMonitors + + to % stats = from % stats +#endif + + end subroutine Monitor_Assign + +! +!////////////////////////////////////////////////////////////////////////////// +! +! Auxiliars +! +!////////////////////////////////////////////////////////////////////////////// +! + subroutine getNoOfMonitors(no_of_probes, no_of_surfaceMonitors, no_of_volumeMonitors, no_of_loadBalancingMonitors) + use ParamfileRegions + implicit none + integer, intent(out) :: no_of_probes + integer, intent(out) :: no_of_surfaceMonitors + integer, intent(out) :: no_of_volumeMonitors + integer, intent(out) :: no_of_loadBalancingMonitors +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: case_name, line + integer :: fID + integer :: io +! +! Initialize +! ---------- + no_of_probes = 0 + no_of_surfaceMonitors = 0 + no_of_volumeMonitors = 0 + no_of_loadBalancingMonitors = 0 +! +! Get case file name +! ------------------ + call get_command_argument(1, case_name) + +! +! Open case file +! -------------- + open ( newunit = fID , file = case_name , status = "old" , action = "read" ) + +! +! Read the whole file to find monitors +! ------------------------------------ +readloop:do + read ( fID , '(A)' , iostat = io ) line + + if ( io .lt. 0 ) then +! +! End of file +! ----------- + line = "" + exit readloop + + elseif ( io .gt. 0 ) then +! +! Error +! ----- + errorMessage(STD_OUT) + error stop "Stopped." + + else +! +! Succeeded +! --------- + line = getSquashedLine( line ) + + if ( index ( line , '#defineprobefile' ) .gt. 0 ) then +! +! The probe-file block is not an individual probe definition +! ----------------------------------------------------------- + + elseif ( index ( line , '#defineprobe' ) .gt. 0 ) then + no_of_probes = no_of_probes + 1 + + elseif ( index ( line , '#definesurfacemonitor' ) .gt. 0 ) then + no_of_surfaceMonitors = no_of_surfaceMonitors + 1 + + elseif ( index ( line , '#definevolumemonitor' ) .gt. 0 ) then + no_of_volumeMonitors = no_of_volumeMonitors + 1 + + elseif ( index ( line , '#defineloadbalancingmonitor' ) .gt. 0 ) then + no_of_loadBalancingMonitors = no_of_loadBalancingMonitors + 1 + + end if + + end if + + end do readloop +! +! Close case file +! --------------- + close(fID) + +end subroutine getNoOfMonitors + +! +!/////////////////////////////////////////////////////////////////////////////////// +! +! Probes-from-file auxiliary routines +! +! The probes file is a plain text file. Blank lines and lines starting +! with "#" are ignored. Every remaining line holds the coordinates of +! one probe: +! +! x y z +! +! The list of variables to sample (shared by every probe in the file) +! is given through the "probes file variables" control-file keyword. +! +! A separate output file ".probe_.probe" is created for +! each probe, with one column per variable, and one row written per +! saved time step. +! +!/////////////////////////////////////////////////////////////////////////////////// +! + subroutine readProbesFileBlock(fileName, variablesLine, no_of_probesVariables, saveTimestep, outputFormat) +! +! ****************************************************************** +! Reads the "#define probe file ... #end" block from the case +! file, if present: +! +! #define probe file +! file = Probe.dat +! variables = u +! probe save timestep = 1.0E-3 +! output format = HDF5 +! #end +! +! Note: this is a hand-rolled parser (rather than readValueInRegion) +! because readValueInRegion lower-cases every line it reads, which +! would corrupt a case-sensitive file path. +! ****************************************************************** +! + use ParamfileRegions, only: getSquashedLine + implicit none + character(len=LINE_LENGTH), intent(out) :: fileName + character(len=LINE_LENGTH), intent(out) :: variablesLine + integer, intent(out) :: no_of_probesVariables + real(kind=RP), intent(out) :: saveTimestep + character(len=8), intent(out) :: outputFormat +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: paramFile + character(len=LINE_LENGTH) :: line, squashed, valStr + integer :: fID, io, position + + logical :: inside + + fileName = "" + variablesLine = "" + no_of_probesVariables = 0 + saveTimestep = 0.0_RP + outputFormat = "ASCII" + inside = .false. + + call get_command_argument(1, paramFile) + + open ( newunit = fID , file = trim(paramFile) , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + squashed = getSquashedLine(line) + + if ( squashed .eq. getSquashedLine("#define probe file") ) then + inside = .true. + cycle + elseif ( squashed .eq. getSquashedLine("#end") ) then + inside = .false. + cycle + end if + + if ( .not. inside ) cycle +! +! Strip a trailing comment, keeping the value's original case +! ------------------------------------------------------------- + position = index(line , '!') + if ( position .gt. 0 ) line = line(1:position-1) + + position = max( index(line,'='), index(line,':') ) + if ( position .eq. 0 ) cycle + + if ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("file") ) then + fileName = adjustl( removeQuotes( line(position+1:) ) ) + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("variables") ) then + variablesLine = adjustl( removeQuotes( line(position+1:) ) ) + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("probe save timestep") ) then + valStr = adjustl( removeQuotes( line(position+1:) ) ) + read( valStr , * ) saveTimestep + elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("output format") ) then + valStr = adjustl( removeQuotes( line(position+1:) ) ) + valStr = adjustl( getSquashedLine(valStr) ) + if ( index(valStr, "HDF5") .gt. 0 ) then + outputFormat = "HDF5" + else + outputFormat = "ASCII" + end if + end if + end do + + close(fID) + + end subroutine readProbesFileBlock + + function removeQuotes(str) result(res) + implicit none + character(len=*), intent(in) :: str + character(len=LINE_LENGTH) :: res + character(len=LINE_LENGTH) :: auxstr + integer :: i, j + + auxstr = trim(adjustl(str)) + res = "" + j = 0 + + do i = 1 , len_trim(auxstr) + if ( auxstr(i:i) .eq. '"' .or. auxstr(i:i) .eq. "'" ) cycle + j = j + 1 + res(j:j) = auxstr(i:i) + end do + + end function removeQuotes + + subroutine countProbesInFile(fileName, n) + implicit none + character(len=*), intent(in) :: fileName + integer, intent(out) :: n +! +! --------------- +! Local variables +! --------------- +! + integer :: fID, io + character(len=LINE_LENGTH) :: line + + n = 0 + open ( newunit = fID , file = fileName , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + line = adjustl(line) + if ( len_trim(line) .eq. 0 ) cycle + if ( line(1:1) .eq. '#' ) cycle + + n = n + 1 + end do + + close(fID) + + end subroutine countProbesInFile + + subroutine splitIntoTokens(line, tokens, n) + implicit none + character(len=*), intent(in) :: line + character(len=STR_LEN_MONITORS), allocatable, intent(out) :: tokens(:) + integer, intent(out) :: n +! +! --------------- +! Local variables +! --------------- +! + character(len=LINE_LENGTH) :: auxline + character(len=STR_LEN_MONITORS) :: buffer(64) + integer :: pos + + auxline = adjustl(line) + n = 0 + + do while ( len_trim(auxline) .gt. 0 ) + pos = index(trim(auxline), " ") + n = n + 1 + + if ( pos .eq. 0 ) then + buffer(n) = trim(auxline) + auxline = "" + else + buffer(n) = auxline(1:pos-1) + auxline = adjustl(auxline(pos+1:)) + end if + end do + + allocate( tokens(n) ) + tokens(1:n) = buffer(1:n) + + end subroutine splitIntoTokens + +#ifdef FLOW + subroutine Monitor_FlushFileProbesNow(self, t_now, iter_now) +! +! Writes the current (single-slot) file-probe values to disk, +! applying the probeFileSaveTimestep filter. Called every timestep +! from Monitor_UpdateValues after Monitor_UpdateFileProbes. +! + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + real(kind=RP), intent(in) :: t_now + integer, intent(in) :: iter_now +! +! --------------- +! Local variables +! --------------- +! + integer :: i + integer :: iter_arr(1) + real(kind=RP) :: t_arr(1) + + iter_arr(1) = iter_now + t_arr(1) = t_now + +#ifdef HAS_HDF5 + if ( trim(self % probeFileOutputFormat) .eq. "HDF5" ) then + call Monitor_WriteFileProbesHDF5( self, iter_arr, t_arr, 1 ) + else +#endif + !$omp parallel do schedule(dynamic,16) default(shared) + do i = self % no_of_probes - self % no_of_fileProbes + 1, self % no_of_probes + call self % probes(i) % WriteToFile( iter_arr, t_arr, 1 ) + end do + !$omp end parallel do +#ifdef HAS_HDF5 + end if +#endif + + end subroutine Monitor_FlushFileProbesNow + + subroutine Monitor_UpdateFileProbes(self, mesh, bufferPos) +! +! Evaluates all file-probes using OpenMP threads (CPU path, no OpenACC), +! then performs a single MPI_Allreduce(SUM) to share results across ranks. +! This replaces N_fileProbes individual MPI_Bcast calls with one collective. +! + use MPI_Process_Info +#ifdef _HAS_MPI_ + use mpi +#endif + implicit none + class(Monitor_t), intent(inout) :: self + class(HexMesh), intent(in) :: mesh + integer, intent(in) :: bufferPos +! +! --------------- +! Local variables +! --------------- +! + integer :: i, v, j, nfp, nv, fp_offset, ierr + real(kind=RP), allocatable :: buf(:) + + nfp = self % no_of_fileProbes + nv = size(self % probesVariables) + fp_offset = self % no_of_probes - nfp + + ! Parallel CPU computation — each probe writes to its own values(:,bufferPos). + ! Non-owning ranks store 0 so MPI_Allreduce(SUM) gives the correct result. + !$omp parallel do schedule(dynamic,16) default(shared) + do i = fp_offset + 1, self % no_of_probes + call self % probes(i) % ComputeLocal(mesh, bufferPos) + end do + !$omp end parallel do + +#ifdef _HAS_MPI_ + if ( MPI_Process % doMPIAction ) then + ! Pack all file-probe values into a flat buffer + allocate( buf(nfp * nv) ) + j = 0 + do i = fp_offset + 1, self % no_of_probes + do v = 1, nv + j = j + 1 + buf(j) = self % probes(i) % values(v, bufferPos) + end do + end do + + ! Single collective instead of nfp individual Bcasts + call MPI_Allreduce(MPI_IN_PLACE, buf, nfp * nv, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_WORLD, ierr) + + ! Unpack + j = 0 + do i = fp_offset + 1, self % no_of_probes + do v = 1, nv + j = j + 1 + self % probes(i) % values(v, bufferPos) = buf(j) + end do + end do + deallocate(buf) + end if +#endif + + end subroutine Monitor_UpdateFileProbes + + subroutine InitializeProbesFromFile(fileName, probes, offset, mesh, solution_file, FirstCall, variables, saveTimestep, outputFormat) + implicit none + character(len=*), intent(in) :: fileName + class(Probe_t), intent(inout) :: probes(:) + integer, intent(in) :: offset + class(HexMesh), intent(in) :: mesh + character(len=*), intent(in) :: solution_file + logical, intent(in) :: FirstCall + character(len=*), intent(in) :: variables(:) + real(kind=RP), intent(in) :: saveTimestep + character(len=*), intent(in) :: outputFormat +! +! --------------- +! Local variables +! --------------- +! + integer :: fID, io, idx, nTok + character(len=LINE_LENGTH) :: line + real(kind=RP) :: x(NDIM) + character(len=STR_LEN_MONITORS), allocatable :: tokens(:) + character(len=STR_LEN_MONITORS) :: pname + + idx = offset + open ( newunit = fID , file = fileName , status = "old" , action = "read" ) + + do + read ( fID , '(A)' , iostat = io ) line + if ( io .ne. 0 ) exit + + line = adjustl(line) + if ( len_trim(line) .eq. 0 ) cycle + if ( line(1:1) .eq. '#' ) cycle + + call splitIntoTokens(line, tokens, nTok) + + if ( nTok .lt. 3 ) then + deallocate(tokens) + cycle + end if + + read(tokens(1),*) x(1) + read(tokens(2),*) x(2) + read(tokens(3),*) x(3) + + idx = idx + 1 + write(pname,'(A,I0)') "probe_", idx + + call probes(idx) % Initialization( mesh, idx, solution_file, FirstCall, & + x_in = x, variables_in = variables, name_in = trim(pname), & + isFileProbe_in = .true., outputFormat_in = trim(outputFormat) ) + probes(idx) % saveTimestep = saveTimestep + + deallocate(tokens) + end do + + close(fID) + + end subroutine InitializeProbesFromFile + +! ============================================================ +! HDF5 collective I/O for file-based probes +! Compiled only when HAS_HDF5 is defined. +! ============================================================ + +#ifdef HAS_HDF5 + subroutine Monitor_InitFileProbesHDF5(self, no_of_fileProbes) +! +! Creates the HDF5 output file for bulk file-probes with +! the following structure: +! +! /coordinates (3, nProbes) — fixed, written once +! /time (extendible) — one value per saved step +! /iteration (extendible) — one value per saved step +! / (nProbes, ext) — one row per saved step +! + use HDF5 + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + integer, intent(in) :: no_of_fileProbes +! +! --------------- +! Local variables +! --------------- +! + integer(HID_T) :: file_id, dset_id, dspace_id, dcpl_id + integer(HSIZE_T) :: dims2(2), maxdims2(2), chunk2(2) + integer(HSIZE_T) :: dims1(1), maxdims1(1), chunk1(1) + integer :: iError, j, v, fp_offset, nv + character(len=LINE_LENGTH) :: fname + real(kind=RP), allocatable :: coords(:,:) + + if ( .not. MPI_Process % isRoot ) return + + nv = size(self % probesVariables) + fp_offset = self % no_of_probes ! probes are allocated 1..no_of_probes+no_of_fileProbes + ! but at this point no_of_probes has not yet been bumped + + write(fname,'(A,A)') trim(self % probes_solution_file), ".probes.h5" + + call h5open_f(iError) + call h5fcreate_f(trim(fname), H5F_ACC_TRUNC_F, file_id, iError) + + ! /coordinates (3, nProbes) — fixed at creation + allocate( coords(3, no_of_fileProbes) ) + do j = 1, no_of_fileProbes + coords(:, j) = self % probes(fp_offset + j) % x + end do + dims2 = [ int(3, HSIZE_T), int(no_of_fileProbes, HSIZE_T) ] + call h5screate_simple_f(2, dims2, dspace_id, iError) + call h5dcreate_f(file_id, "coordinates", H5T_NATIVE_DOUBLE, dspace_id, dset_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, coords, dims2, iError) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + deallocate(coords) + + ! /time (extendible 1-D) + dims1(1) = 0 + maxdims1(1) = H5S_UNLIMITED_F + chunk1(1) = int(max(BUFFER_SIZE, 1), HSIZE_T) + call h5screate_simple_f(1, dims1, dspace_id, iError, maxdims1) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 1, chunk1, iError) + call h5dcreate_f(file_id, "time", H5T_NATIVE_DOUBLE, dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + + ! /iteration (extendible 1-D) + call h5screate_simple_f(1, dims1, dspace_id, iError, maxdims1) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 1, chunk1, iError) + call h5dcreate_f(file_id, "iteration", H5T_NATIVE_INTEGER, dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + + ! / (extendible × nProbes) — h5ls shows {nProbes, Inf} + ! chunk2(1)=1: one time step per chunk avoids pre-allocation waste when + ! the save-timestep filter makes n_write << BUFFER_SIZE per flush. + dims2(1) = 0 + dims2(2) = int(no_of_fileProbes, HSIZE_T) + maxdims2(1) = H5S_UNLIMITED_F + maxdims2(2) = int(no_of_fileProbes, HSIZE_T) + chunk2(1) = int(1, HSIZE_T) + chunk2(2) = int(no_of_fileProbes, HSIZE_T) + + do v = 1, nv + call h5screate_simple_f(2, dims2, dspace_id, iError, maxdims2) + call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) + call h5pset_chunk_f(dcpl_id, 2, chunk2, iError) + call h5dcreate_f(file_id, trim(self % probesVariables(v)), H5T_NATIVE_DOUBLE, & + dspace_id, dset_id, iError, dcpl_id) + call h5dclose_f(dset_id, iError) + call h5sclose_f(dspace_id, iError) + call h5pclose_f(dcpl_id, iError) + end do + + call h5fclose_f(file_id, iError) + call h5close_f(iError) + + end subroutine Monitor_InitFileProbesHDF5 + + subroutine Monitor_WriteFileProbesHDF5(self, iter, t, no_of_lines) +! +! Appends one buffer of file-probe data to the HDF5 output file. +! Only lines that satisfy the saveTimestep filter are written. +! Called from Monitor_WriteToFile in place of the per-probe ASCII loop. +! + use HDF5 + use MPI_Process_Info + implicit none + class(Monitor_t), intent(inout) :: self + integer, intent(in) :: iter(:) + real(kind=RP), intent(in) :: t(:) + integer, intent(in) :: no_of_lines +! +! --------------- +! Local variables +! --------------- +! + integer(HID_T) :: file_id, dset_id, dspace_id, mspace_id + integer(HSIZE_T) :: cur1(1), max1(1), off1(1), cnt1(1) + integer(HSIZE_T) :: new1(1) + integer(HSIZE_T) :: cur2(2), max2(2), off2(2), cnt2(2) + integer(HSIZE_T) :: new2(2) + integer :: iError, i, j, k, v, nfp, nv, fp_offset, n_write + character(len=LINE_LENGTH) :: fname + logical, allocatable :: wmask(:) + integer, allocatable :: ibuf(:) + real(kind=RP), allocatable :: rbuf(:), vbuf(:) + + if ( .not. MPI_Process % isRoot ) return + + nfp = self % no_of_fileProbes + nv = size(self % probesVariables) + fp_offset = self % no_of_probes - nfp + + ! Build write mask (apply saveTimestep filter) + allocate( wmask(no_of_lines) ) + wmask = .false. + do i = 1, no_of_lines + if ( self % probeFileSaveTimestep .gt. 0.0_RP ) then + if ( t(i) .lt. self % fp_lastSavedTime + self % probeFileSaveTimestep ) cycle + end if + wmask(i) = .true. + self % fp_lastSavedTime = t(i) + end do + n_write = count(wmask) + if ( n_write .eq. 0 ) then + deallocate(wmask) + return + end if + + write(fname,'(A,A)') trim(self % probes_solution_file), ".probes.h5" + + call h5open_f(iError) + call h5fopen_f(trim(fname), H5F_ACC_RDWR_F, file_id, iError) + + ! Query current extent of /time to get append offset + call h5dopen_f(file_id, "time", dset_id, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sget_simple_extent_dims_f(dspace_id, cur1, max1, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + ! cur1(1) = number of time steps already written + + cnt1(1) = int(n_write, HSIZE_T) + off1(1) = cur1(1) + new1(1) = cur1(1) + cnt1(1) + + ! Pack filtered iteration values + allocate( ibuf(n_write) ) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + ibuf(j) = iter(i) + end do + + call h5dopen_f(file_id, "iteration", dset_id, iError) + call h5dextend_f(dset_id, new1, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off1, cnt1, iError) + call h5screate_simple_f(1, cnt1, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_INTEGER, ibuf, cnt1, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + deallocate(ibuf) + + ! Pack filtered time values + allocate( rbuf(n_write) ) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + rbuf(j) = t(i) + end do + + call h5dopen_f(file_id, "time", dset_id, iError) + call h5dextend_f(dset_id, new1, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off1, cnt1, iError) + call h5screate_simple_f(1, cnt1, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, rbuf, cnt1, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + call h5dclose_f(dset_id, iError) + deallocate(rbuf) + + ! Write each variable: one extend + one write per flush (avoids chunk waste). + ! vbuf2(nfp, n_write) packs all filtered timesteps into a contiguous 2D block. + off2(1) = cur1(1) ! append after existing time steps + off2(2) = int(0, HSIZE_T) + cnt2(1) = int(n_write, HSIZE_T) + cnt2(2) = int(nfp, HSIZE_T) + new2(1) = cur1(1) + int(n_write, HSIZE_T) + new2(2) = int(nfp, HSIZE_T) + + allocate( vbuf(nfp * n_write) ) + + do v = 1, nv + call h5dopen_f(file_id, trim(self % probesVariables(v)), dset_id, iError) + + ! Pack into column-major 2D layout vbuf(n_write, nfp): + ! time index j varies fastest (Fortran column-major with cnt2=[n_write,nfp]) + j = 0 + do i = 1, no_of_lines + if ( .not. wmask(i) ) cycle + j = j + 1 + do k = 1, nfp + vbuf( j + (k-1)*n_write ) = self % probes(fp_offset + k) % values(v, i) + end do + end do + + call h5dextend_f(dset_id, new2, iError) + call h5dget_space_f(dset_id, dspace_id, iError) + call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off2, cnt2, iError) + call h5screate_simple_f(2, cnt2, mspace_id, iError) + call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, vbuf, cnt2, iError, mspace_id, dspace_id) + call h5sclose_f(mspace_id, iError) + call h5sclose_f(dspace_id, iError) + + call h5dclose_f(dset_id, iError) + end do + + deallocate(vbuf) + deallocate(wmask) + + call h5fclose_f(file_id, iError) + call h5close_f(iError) + + end subroutine Monitor_WriteFileProbesHDF5 +#endif ! HAS_HDF5 + +#endif ! FLOW + +end module MonitorsClass +! +!/////////////////////////////////////////////////////////////////////////////////// +! diff --git a/Solver/src/libs/mesh/Probe.f90 b/Solver/src/libs/mesh/Probe.f90 new file mode 100644 index 000000000..e91279dbc --- /dev/null +++ b/Solver/src/libs/mesh/Probe.f90 @@ -0,0 +1,967 @@ +#include "Includes.h" +#ifdef FLOW +module ProbeClass + use SMConstants + use HexMeshClass + use MonitorDefinitions + use PhysicsStorage + use VariableConversion + use MPI_Process_Info + use FluidData + use FileReadingUtilities, only: getRealArrayFromString + use NodalStorageClass , only: NodalStorage +#ifdef _HAS_MPI_ + use mpi +#endif + implicit none + + private + public Probe_t +! +! ********************** +! Probe class definition +! ********************** +! + type Probe_t + logical :: active + logical :: isFileProbe + integer :: rank + integer :: ID + integer :: eID + integer :: nVars + real(kind=RP) :: x(NDIM) + real(kind=RP) :: xi(NDIM) + real(kind=RP), allocatable :: values(:,:) + real(kind=RP), allocatable :: lxi(:) , leta(:), lzeta(:) + real(kind=RP), allocatable :: var(:,:,:) + real(kind=RP) :: saveTimestep + real(kind=RP) :: lastSavedTime + character(len=STR_LEN_MONITORS) :: fileName + character(len=STR_LEN_MONITORS) :: monitorName + character(len=STR_LEN_MONITORS), allocatable :: variableNames(:) + contains + procedure :: Initialization => Probe_Initialization + procedure :: Update => Probe_Update + procedure :: ComputeLocal => Probe_ComputeLocal + procedure :: WriteLabel => Probe_WriteLabel + procedure :: WriteValues => Probe_WriteValue + procedure :: WriteToFile => Probe_WriteToFile + procedure :: LookInOtherPartitions => Probe_LookInOtherPartitions + procedure :: destruct => Probe_Destruct + procedure :: copy => Probe_Assign + generic :: assignment(=) => copy + end type Probe_t + + contains + + subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall, x_in, variables_in, name_in, isFileProbe_in, outputFormat_in) + use ParamfileRegions + use MPI_Process_Info + use Utilities, only: toLower + implicit none + class(Probe_t) :: self + class(HexMesh) :: mesh + integer :: ID + character(len=*) :: solution_file + logical, intent(in) :: FirstCall + real(kind=RP), intent(in), optional :: x_in(NDIM) + character(len=*), intent(in), optional :: variables_in(:) + character(len=*), intent(in), optional :: name_in + logical, intent(in), optional :: isFileProbe_in + character(len=*), intent(in), optional :: outputFormat_in +! +! --------------- +! Local variables +! --------------- +! + integer :: i, j, k, v, fid + character(len=STR_LEN_MONITORS) :: in_label + character(len=STR_LEN_MONITORS) :: fileName + character(len=STR_LEN_MONITORS) :: paramFile + character(len=STR_LEN_MONITORS) :: coordinates + character(len=STR_LEN_MONITORS) :: variable + character(len=STR_LEN_MONITORS) :: outputFormat + + self % isFileProbe = .false. + if ( present(isFileProbe_in) ) self % isFileProbe = isFileProbe_in + + outputFormat = "ASCII" + if ( present(outputFormat_in) ) outputFormat = trim(outputFormat_in) + + if (FirstCall) then +! +! Get monitor ID +! -------------- + self % ID = ID +! +! Get the probe definition, either from a probes file (x_in/variables_in +! provided) or from a "#define probe" block in the case file +! ------------------------------------------------------------------------ + if ( present(x_in) ) then + self % monitorName = name_in + self % x = x_in + self % nVars = size(variables_in) + allocate( self % variableNames(self % nVars) ) + self % variableNames = variables_in + + else +! +! Search for the parameters in the case file +! ------------------------------------------ + write(in_label , '(A,I0)') "#define probe " , self % ID + + call get_command_argument(1, paramFile) + call readValueInRegion(trim(paramFile), "name" , self % monitorName, in_label, "# end" ) + call readValueInRegion(trim(paramFile), "variable", variable , in_label, "# end" ) + call readValueInRegion(trim(paramFile), "position", coordinates , in_label, "# end" ) +! +! Get the coordinates +! ------------------- + self % x = getRealArrayFromString(coordinates) + + self % nVars = 1 + allocate( self % variableNames(1) ) + self % variableNames(1) = variable + end if +! +! Allocate memory +! --------------- + if (self % isFileProbe) then + allocate ( self % values(self % nVars, 1) ) + else + allocate ( self % values(self % nVars, BUFFER_SIZE) ) + end if + self % saveTimestep = 0.0_RP + self % lastSavedTime = -huge(self % lastSavedTime) +! +! Check the variables +! -------------------- + do v = 1, self % nVars + call tolower(self % variableNames(v)) + + select case ( trim(self % variableNames(v)) ) +#ifdef NAVIERSTOKES + case ("pressure") + case ("velocity") + case ("u") + case ("v") + case ("w") + case ("mach") + case ("k") + case ("rho") + case default + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' + print*, "Options available are:" + print*, " * pressure" + print*, " * velocity" + print*, " * u" + print*, " * v" + print*, " * w" + print*, " * Mach" + print*, " * K" + print*, " * rho" + end select +#endif +#ifdef INCNS + case ("pressure") + case ("velocity") + case ("u") + case ("v") + case ("w") + case ("rho") + case default + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' + print*, "Options available are:" + print*, " * pressure" + print*, " * velocity" + print*, " * u" + print*, " * v" + print*, " * w" + print*, " * rho" + end select +#endif +#ifdef MULTIPHASE + case ("static-pressure") + + case default + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' + print*, "Options available are:" + print*, " * static-pressure" + + end select +#endif +#ifdef ACOUSTIC + case ("pressure") + case ("density") + case ("u") + case ("v") + case ("w") + case default + print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' + print*, "Options available are:" + print*, " * pressure" + print*, " * velocity" + print*, " * u" + print*, " * v" + print*, " * w" + end select +#endif + end do + +! +! Find the requested point in the mesh +! ------------------------------------ + self % active = mesh % FindPointWithCoords(self % x, self % eID, self % xi) +! +! Check whether the probe is located in other partition +! ----------------------------------------------------- + call self % LookInOtherPartitions +! +! Disable the probe if the point is not found +! ------------------------------------------- + if ( .not. self % active ) then + if ( MPI_Process % isRoot ) then + write(STD_OUT,'(A,I0,A)') "Probe ", ID, " was not successfully initialized." + print*, "Probe is set to inactive." + end if + + return + end if +! +! Set the fileName +! ---------------- + write( self % fileName , '(A,A,A,A)') trim(solution_file) , "." , & + trim(self % monitorName) , ".probe" + end if +! +! +! Return if the process does not contain the partition +! ---------------------------------------------------- + if ( self % rank .ne. MPI_Process % rank ) then + self % eID = 1 + return + end if + +! +! If this is not the first call, just reload the reference frame coordinates +! -------------------------------------------------------------------------- + if (.not. firstCall) self % active = mesh % elements(self % eID) % FindPointWithCoords(self % x,mesh % dir2D_ctrl, self % xi) +! +! Get the Lagrange interpolants +! ----------------------------- + associate(e => mesh % elements(self % eID)) + associate( spAxi => NodalStorage(e % Nxyz(1)), & + spAeta => NodalStorage(e % Nxyz(2)), & + spAzeta => NodalStorage(e % Nxyz(3)) ) + safedeallocate(self % lxi ) ; allocate( self % lxi(0 : e % Nxyz(1)) ) + safedeallocate(self % leta ) ; allocate( self % leta(0 : e % Nxyz(2)) ) + safedeallocate(self % lzeta) ; allocate( self % lzeta(0 : e % Nxyz(3)) ) + self % lxi = spAxi % lj(self % xi(1)) + self % leta = spAeta % lj(self % xi(2)) + self % lzeta = spAzeta % lj(self % xi(3)) +! +! Allocate storage for the probe +! ----------------------------- + safedeallocate(self % var ) ; allocate( self % var(0 : e % Nxyz(1),0 : e % Nxyz(2),0 : e % Nxyz(3)) ) + self % var = 0.0_RP + + if ( .not. self % isFileProbe ) then + !$acc enter data copyin(self) + !$acc enter data copyin(self % eiD) + !$acc enter data copyin(self % id) + !$acc enter data copyin(self % var) + !$acc enter data copyin(self % lxi) + !$acc enter data copyin(self % leta) + !$acc enter data copyin(self % lzeta) + end if +! +! **************** +! Prepare the file +! **************** +! +! Create file (skip for file-probes using HDF5 output) +! ----------------------------------------------------- + if (FirstCall .and. .not. (self % isFileProbe .and. trim(outputFormat) .eq. "HDF5")) then + open ( newunit = fID , file = trim(self % fileName) , status = "unknown" , action = "write" ) +! +! Write the file headers +! ---------------------- + write( fID , '(A20,A )') "Monitor name: ", trim(self % monitorName) + write( fID , '(A25,ES24.10,2(4X,ES24.10))') "x, y, z coordinates: ", self % x(1), self % x(2), self % x(3) + + write( fID , * ) + write( fID , '(A10,2X,A24)' , advance = "no") "Iteration" , "Time" + do v = 1 , self % nVars + write( fID , '(2X,A24)' , advance = "no") trim(self % variableNames(v)) + end do + write( fID , * ) + + close ( fID ) + end if + end associate + end associate + end subroutine Probe_Initialization + + subroutine Probe_Update(self, mesh, bufferPosition) + use Physics + use MPI_Process_Info + implicit none + class(Probe_t) :: self + type(HexMesh) :: mesh + integer :: bufferPosition +! +! --------------- +! Local variables +! --------------- +! + integer :: i, j, k, v, ierr + real(kind=RP) :: value + + if ( .not. self % active ) return + + if ( MPI_Process % rank .eq. self % rank ) then + +! +! Update the probe +! ---------------- + do v = 1, self % nVars + + if ( self % isFileProbe ) then +! +! CPU path for file-probes (no OpenACC to avoid runtime table overflow) +! ---------------------------------------------------------------------- + select case (trim(self % variableNames(v))) +#ifdef NAVIERSTOKES + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("mach") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) + self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& + (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) + end do ; end do ; end do + case("k") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef INCNS + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef MULTIPHASE + case("static-pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & + - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & + - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) + end do ; end do ; end do +#endif +#ifdef ACOUSTIC + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAP,i,j,k) + end do ; end do ; end do + case("density") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAARHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAU,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAV,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAW,i,j,k) + end do ; end do ; end do +#endif + end select + + value = 0.0_RP + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) + end do ; end do ; end do + + else +! +! GPU path for standard probes (OpenACC) +! --------------------------------------- + select case (trim(self % variableNames(v))) +#ifdef NAVIERSTOKES + case("pressure") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) + end do ; end do ; end do + !$acc end parallel loop + + case("velocity") + !$acc parallel loop collapse(3) present(mesh, self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("u") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("v") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("w") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("mach") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) ! Vabs**2 + self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& + (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) + end do ; end do ; end do + !$acc end parallel loop + + case("k") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("rho") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop +#endif +#ifdef INCNS + + case("pressure") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("velocity") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & + POW2( mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("u") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("v") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("w") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop + + case("rho") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + !$acc end parallel loop +#endif +#ifdef MULTIPHASE + case("static-pressure") + !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & + - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & + - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) + end do ; end do ; end do + !$acc end parallel loop +#endif +#ifdef ACOUSTIC + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAP,i,j,k) + end do ; end do ; end do + + case("density") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAARHO,i,j,k) + end do ; end do ; end do + + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAU,i,j,k) + end do ; end do ; end do + + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAV,i,j,k) + end do ; end do ; end do + + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAW,i,j,k) + end do ; end do ; end do +#endif + end select + + value = 0.0_RP + !$acc parallel loop collapse(3) present(mesh, self) reduction(+:value) async(self % ID) + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) + end do ; end do ; end do + !$acc end parallel loop + + !$acc wait + + end if ! isFileProbe + + self % values(v, bufferPosition) = value + + end do +! +#ifdef _HAS_MPI_ + if ( MPI_Process % doMPIAction ) then +! +! Share the result with the rest of the processes +! ----------------------------------------------- + call mpi_bcast(self % values(:,bufferPosition), self % nVars, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) + + end if +#endif + else +! +! Receive the result from the rank that contains the probe +! -------------------------------------------------------- +#ifdef _HAS_MPI_ + if ( MPI_Process % doMPIAction ) then + call mpi_bcast(self % values(:,bufferPosition), self % nVars, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) + end if +#endif + end if + end subroutine Probe_Update + + subroutine Probe_ComputeLocal(self, mesh, bufferPosition) +! +! ************************************************************* +! CPU-only computation for file-probes, no MPI. +! Non-owning ranks set values to 0 so a caller can +! accumulate results with a single MPI_Allreduce(SUM). +! ************************************************************* +! + use Physics + use MPI_Process_Info + implicit none + class(Probe_t) :: self + type(HexMesh) :: mesh + integer :: bufferPosition +! +! --------------- +! Local variables +! --------------- +! + integer :: i, j, k, v + real(kind=RP) :: value + + if ( .not. self % active ) then + self % values(:, bufferPosition) = 0.0_RP + return + end if + + if ( MPI_Process % rank .ne. self % rank ) then + self % values(:, bufferPosition) = 0.0_RP + return + end if + + do v = 1, self % nVars + + select case (trim(self % variableNames(v))) +#ifdef NAVIERSTOKES + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("mach") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) + self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& + (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) + end do ; end do ; end do + case("k") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef INCNS + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) + end do ; end do ; end do + case("velocity") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & + POW2(mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do + case("rho") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) + end do ; end do ; end do +#endif +#ifdef MULTIPHASE + case("static-pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & + - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & + - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) + end do ; end do ; end do +#endif +#ifdef ACOUSTIC + case("pressure") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAP,i,j,k) + end do ; end do ; end do + case("density") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAARHO,i,j,k) + end do ; end do ; end do + case("u") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAU,i,j,k) + end do ; end do ; end do + case("v") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAV,i,j,k) + end do ; end do ; end do + case("w") + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + self % var(i,j,k) = Q(ICAAW,i,j,k) + end do ; end do ; end do +#endif + end select + + value = 0.0_RP + do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) + value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) + end do ; end do ; end do + + self % values(v, bufferPosition) = value + + end do + + end subroutine Probe_ComputeLocal + + subroutine Probe_WriteLabel ( self ) +! +! ************************************************************* +! This subroutine writes the label for the probe, +! when invoked from the time integrator Display +! procedure. +! ************************************************************* +! + implicit none + class(Probe_t) :: self + + write(STD_OUT , '(3X,A10)' , advance = "no") trim(self % monitorName(1 : MONITOR_LENGTH)) + + end subroutine Probe_WriteLabel + + subroutine Probe_WriteValue ( self , bufferLine ) +! +! ************************************************************* +! This subroutine writes the monitor value for the time +! integrator Display procedure. +! ************************************************************* +! + implicit none + class(Probe_t) :: self + integer :: bufferLine + + write(STD_OUT , '(1X,A,1X,ES10.3)' , advance = "no") "|" , self % values ( 1 , bufferLine ) + + end subroutine Probe_WriteValue + + subroutine Probe_WriteToFile ( self , iter , t , no_of_lines) +! +! ************************************************************* +! This subroutine writes the buffer to the file. +! ************************************************************* +! + implicit none + class(Probe_t) :: self + integer :: iter(:) + real(kind=RP) :: t(:) + integer :: no_of_lines +! +! --------------- +! Local variables +! --------------- +! + integer :: i, v + integer :: fID + + if ( .not. self % active ) then + if ( no_of_lines .ne. 0 ) self % values(:,1) = self % values(:,no_of_lines) + return + end if + + if ( MPI_Process % isRoot ) then + open( newunit = fID , file = trim ( self % fileName ) , action = "write" , access = "append" , status = "old" ) + + do i = 1 , no_of_lines + if ( self % saveTimestep > 0.0_RP ) then + if ( t(i) < self % lastSavedTime + self % saveTimestep ) cycle + end if + write( fID , '(I10,2X,ES24.16)' , advance = "no" ) iter(i) , t(i) + do v = 1 , self % nVars + write( fID , '(2X,ES24.16)' , advance = "no" ) self % values(v,i) + end do + write( fID , * ) + self % lastSavedTime = t(i) + end do + + close ( fID ) + end if + + + if ( no_of_lines .ne. 0 ) self % values(:,1) = self % values(:,no_of_lines) + + end subroutine Probe_WriteToFile + + subroutine Probe_LookInOtherPartitions(self) + use MPI_Process_Info + implicit none + class(Probe_t) :: self + integer :: allActives(MPI_Process % nProcs) + integer :: active, ierr + + if ( MPI_Process % doMPIAction ) then +#ifdef _HAS_MPI_ +! +! Cast the logicals onto integers +! ------------------------------- + if ( self % active ) then + active = 1 + else + active = 0 + end if +! +! Gather all data from all processes +! ---------------------------------- + call mpi_allgather(active, 1, MPI_INT, allActives, 1, MPI_INT, MPI_COMM_WORLD, ierr) +! +! Check if any of them found the probe +! ------------------------------------ + if ( any(allActives .eq. 1) ) then +! +! Assign the domain of the partition that contains the probe +! ---------------------------------------------------------- + self % active = .true. + self % rank = maxloc(allActives, dim = 1) - 1 + + else +! +! Disable the probe +! ----------------- + self % active = .false. + self % rank = -1 + + end if +#endif + else +! +! Without MPI select the rank 0 as default +! ---------------------------------------- + self % rank = 0 + + end if + + end subroutine Probe_LookInOtherPartitions + + elemental subroutine Probe_Destruct (self) + implicit none + class(Probe_t), intent(inout) :: self + + safedeallocate (self % values) + safedeallocate (self % lxi) + safedeallocate (self % leta) + safedeallocate (self % lzeta) + safedeallocate (self % variableNames) + end subroutine Probe_Destruct + + elemental subroutine Probe_Assign (to, from) + implicit none + class(Probe_t), intent(inout) :: to + type(Probe_t) , intent(in) :: from + + to % active = from % active + to % isFileProbe = from % isFileProbe + to % rank = from % rank + to % ID = from % ID + to % eID = from % eID + to % nVars = from % nVars + to % x = from % x + to % xi = from % xi + + if ( allocated(from % values) ) then + safedeallocate ( to % values ) + allocate ( to % values ( size(from % values, 1), size(from % values, 2) ) ) + to % values = from % values + end if + + if ( allocated(from % lxi) ) then + safedeallocate ( to % lxi ) + allocate ( to % lxi ( size(from % lxi) ) ) + to % lxi = from % lxi + end if + + if ( allocated(from % leta) ) then + safedeallocate ( to % leta ) + allocate ( to % leta ( size(from % leta) ) ) + to % leta = from % leta + end if + + if ( allocated(from % lzeta) ) then + safedeallocate ( to % lzeta ) + allocate ( to % lzeta ( size(from % lzeta) ) ) + to % lzeta = from % lzeta + end if + + to % saveTimestep = from % saveTimestep + to % lastSavedTime = from % lastSavedTime + to % fileName = from % fileName + to % monitorName = from % monitorName + + safedeallocate ( to % variableNames ) + allocate ( to % variableNames ( size(from % variableNames) ) ) + to % variableNames = from % variableNames + + end subroutine Probe_Assign + +end module ProbeClass +#endif diff --git a/Solver/src/libs/mesh/SurfaceIntegrals.f90 b/Solver/src/libs/mesh/SurfaceIntegrals.f90 new file mode 100644 index 000000000..6807203c8 --- /dev/null +++ b/Solver/src/libs/mesh/SurfaceIntegrals.f90 @@ -0,0 +1,1269 @@ +#include "Includes.h" +#if defined(NAVIERSTOKES) || (defined(INCNS)) +module SurfaceIntegrals + use SMConstants + use PhysicsStorage + use Physics + use FaceClass + use ElementClass + use HexMeshClass + use VariableConversion, only: Pressure + use NodalStorageClass +#ifdef _HAS_MPI_ + use mpi +#endif + implicit none + + private + public SURFACE, TOTAL_FORCE, PRESSURE_FORCE, VISCOUS_FORCE, MASS_FLOW, FLOW_RATE, PRESSURE_DISTRIBUTION + public ScalarSurfaceIntegral, VectorSurfaceIntegral, ScalarDataReconstruction, VectorDataReconstruction + + integer, parameter :: SURFACE = 1 + integer, parameter :: TOTAL_FORCE = 2 + integer, parameter :: PRESSURE_FORCE = 3 + integer, parameter :: VISCOUS_FORCE = 4 + integer, parameter :: MASS_FLOW = 5 + integer, parameter :: FLOW_RATE = 6 + integer, parameter :: PRESSURE_DISTRIBUTION = 7 + integer, parameter :: USER_DEFINED = 99 + !$acc declare copyin(SURFACE, TOTAL_FORCE, PRESSURE_FORCE, VISCOUS_FORCE, MASS_FLOW, FLOW_RATE, PRESSURE_DISTRIBUTION, USER_DEFINED) +! +! ======== + contains +! ======== +! +!//////////////////////////////////////////////////////////////////////////////////////// +! +! SCALAR INTEGRALS PROCEDURES +! +!//////////////////////////////////////////////////////////////////////////////////////// +! + function ScalarSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) +! +! ----------------------------------------------------------- +! This function computes scalar integrals, that is, those +! in the form: +! val = \int \vec{v}·\vec{n}dS +! Implemented integrals are: +! * Surface: computes the zone surface. +! * Mass flow: computes the mass flow across the zone. +! * Flow: computes the volumetric flow across the zone. +! ----------------------------------------------------------- +! + implicit none + class(HexMesh), intent(inout), target :: mesh + integer, intent(in) :: zoneID + integer, intent(in) :: integralType, iter + real(kind=RP) :: val, localval +! +! --------------- +! Local variables +! --------------- +! + integer :: zonefID, fID, eID, fIDs(6), ierr +! +! Initialization +! -------------- + val = 0.0_RP + + call ProlongToFaces(mesh, zoneID) +! +! Loop the zone to get faces and elements +! --------------------------------------- +#ifdef _OPENACC +!$acc parallel loop gang reduction(+:val) present(mesh) +#else +!$omp parallel do private(fID) reduction(+:val) schedule(runtime) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces +! +! Face global ID +! -------------- + fID = mesh % zones(zoneID) % faces(zonefID) +! +! Compute the integral +! -------------------- + val = val + ScalarSurfaceIntegral_Face(mesh % faces(fID), integralType) + + end do +#ifdef _OPENACC +!$acc end parallel loop +#else +!$omp end parallel do +#endif + +#ifdef _HAS_MPI_ + localval = val + call mpi_allreduce(localval, val, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) +#endif + + end function ScalarSurfaceIntegral + + function ScalarSurfaceIntegral_Face(f, integralType) result(val) + !$acc routine vector + implicit none + class(Face), intent(in) :: f + integer, intent(in) :: integralType + real(kind=RP) :: val +! +! --------------- +! Local variables +! --------------- +! + integer :: i, j ! Face indices + real(kind=RP) :: p +! +! Initialization +! -------------- + val = 0.0_RP +! +! Perform the numerical integration +! --------------------------------- + select case ( integralType ) + case ( SURFACE ) +! +! ********************************** +! Computes the surface integral +! val = \int dS +! ********************************** +! + !$acc loop vector collapse(2) reduction(+:val) + do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) + val = val + NodalStorage(f % Nf(1)) % w(i) & + * NodalStorage(f % Nf(2)) % w(j) & + * f % geom % jacobian(i,j) + end do ; end do + + case ( MASS_FLOW ) +! +! *********************************** +! Computes the mass-flow integral +! I = \int rho \vec{v}·\vec{n}dS +! *********************************** +! +#if defined(NAVIERSTOKES) + !$acc loop vector collapse(2) reduction(+:val) + do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) +! +! Compute the integral +! -------------------- + val = val +(f % storage(1) % Q(IRHOU,i,j) * f % geom % normal(1,i,j) & + + f % storage(1) % Q(IRHOV,i,j) * f % geom % normal(2,i,j) & + + f % storage(1) % Q(IRHOW,i,j) * f % geom % normal(3,i,j) ) & + * NodalStorage(f % Nf(1)) % w(i) & + * NodalStorage(f % Nf(2)) % w(j) & + * f % geom % jacobian(i,j) + + end do ; end do +#endif +#if defined(INCNS) + !$acc loop vector collapse(2) reduction(+:val) + do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) +! +! Compute the integral +! -------------------- + val = val +(f % storage(1) % Q(INSRHOU,i,j) * f % geom % normal(1,i,j) & + + f % storage(1) % Q(INSRHOV,i,j) * f % geom % normal(2,i,j) & + + f % storage(1) % Q(INSRHOW,i,j) * f % geom % normal(3,i,j) ) & + * NodalStorage(f % Nf(1)) % w(i) & + * NodalStorage(f % Nf(2)) % w(j) & + * f % geom % jacobian(i,j) + + end do ; end do +#endif + case ( FLOW_RATE ) +! +! *********************************** +! Computes the flow integral +! val = \int \vec{v}·\vec{n}dS +! *********************************** +! +#if defined(NAVIERSTOKES) + !$acc loop vector collapse(2) reduction(+:val) + do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) +! +! Compute the integral +! -------------------- + val = val + (1.0_RP / f % storage(1) % Q(IRHO,i,j))*(f % storage(1) % Q(IRHOU,i,j) * f % geom % normal(1,i,j) & + + f % storage(1) % Q(IRHOV,i,j) * f % geom % normal(2,i,j) & + + f % storage(1) % Q(IRHOW,i,j) * f % geom % normal(3,i,j) ) & + * NodalStorage(f % Nf(1)) % w(i) & + * NodalStorage(f % Nf(2)) % w(j) & + * f % geom % jacobian(i,j) + end do ; end do +#endif +#if defined(INCNS) + !$acc loop vector collapse(2) reduction(+:val) + do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) +! +! Compute the integral +! -------------------- + val = val + (1.0_RP / f % storage(1) % Q(INSRHO,i,j))*(f % storage(1) % Q(INSRHOU,i,j) * f % geom % normal(1,i,j) & + + f % storage(1) % Q(INSRHOV,i,j) * f % geom % normal(2,i,j) & + + f % storage(1) % Q(INSRHOW,i,j) * f % geom % normal(3,i,j) ) & + * NodalStorage(f % Nf(1)) % w(i) & + * NodalStorage(f % Nf(2)) % w(j) & + * f % geom % jacobian(i,j) + end do ; end do +#endif + case ( PRESSURE_FORCE ) +! +! *********************************** +! Computes the pressure integral +! val = \int pdS +! *********************************** +! + !$acc loop vector collapse(2) reduction(+:val) + do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) +! +! Compute the integral +! -------------------- + p = Pressure(f % storage(1) % Q(:,i,j)) + val = val + p * NodalStorage(f % Nf(1)) % w(i) & + * NodalStorage(f % Nf(2)) % w(j) & + * f % geom % jacobian(i,j) + end do ; end do + + + case ( USER_DEFINED ) ! TODO + end select + + end function ScalarSurfaceIntegral_Face +! +!//////////////////////////////////////////////////////////////////////////////////////// +! +! VECTOR INTEGRALS PROCEDURES +! +!//////////////////////////////////////////////////////////////////////////////////////// +! + function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) +! +! ----------------------------------------------------------- +! This function computes scalar integrals, that is, those +! in the form: +! val = \int \vec{v}·\vec{n}dS +! Implemented integrals are: +! * Surface: computes the zone surface. +! * Mass flow: computes the mass flow across the zone. +! * Flow: computes the volumetric flow across the zone. +! ----------------------------------------------------------- +! +#ifdef _HAS_MPI_ + use mpi +#endif + implicit none + type(HexMesh), intent(inout) :: mesh + integer, intent(in) :: zoneID + integer, intent(in) :: integralType, iter + real(kind=RP) :: val(NDIM) +! +! --------------- +! Local variables +! --------------- +! + integer :: zonefID, fID, eID, i, j, ierr + real(kind=RP) :: localval(NDIM) + real(kind=RP) :: valx, valy, valz + real(kind=RP) :: localx, localy, localz + real(kind=RP) :: p, tau(1:NDIM, 1:NDIM) +! +! Initialization +! -------------- + val = 0.0_RP + localval = 0.0_RP + valx = 0.0_RP + valy = 0.0_RP + valz = 0.0_RP + + call ProlongToFaces(mesh, zoneID) + + select case ( integralType ) + case ( SURFACE ) + +#ifdef _OPENACC +!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#else +!$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + ! + ! Face global ID + ! -------------- + fID = mesh % zones(zoneID) % faces(zonefID) + + localx = 0.0_RP + localy = 0.0_RP + localz = 0.0_RP + ! + ! ********************************** + ! Computes the surface integral + ! val = \int \vec{n} dS + ! ********************************** + ! + !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(val) + do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) + val = NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) & + * mesh % faces(fid) % geom % jacobian(i,j) * mesh % faces(fid) % geom % normal(:,i,j) + + localx = localx + val(1) + localy = localy + val(2) + localz = localz + val(3) + + end do ; end do + + valx = valx + localx + valy = valy + localy + valz = valz + localz + + end do +#ifdef _OPENACC +!$acc end parallel loop +#else +!$omp end parallel do +#endif + + case ( TOTAL_FORCE ) + +#ifdef _OPENACC +!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#else +!$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + ! + ! Face global ID + ! -------------- + fID = mesh % zones(zoneID) % faces(zonefID) + + localx = 0.0_RP + localy = 0.0_RP + localz = 0.0_RP + ! + ! ************************************************ + ! Computes the total force experienced by the zone + ! F = \int p \vec{n}ds - \int tau'·\vec{n}ds + ! ************************************************ + ! + !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(tau, localval) + do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) + ! + ! Compute the integral + ! -------------------- + p = Pressure(mesh % faces(fid) % storage(1) % Q(:,i,j)) + call getStressTensor(mesh % faces(fid) % storage(1) % Q(:,i,j), mesh % faces(fid) % storage(1) % U_x(:,i,j),& + mesh % faces(fid) % storage(1) % U_y(:,i,j),mesh % faces(fid) % storage(1) % U_z(:,i,j), tau) + + localval = ( p * mesh % faces(fid) % geom % normal(:,i,j) - matmul(tau,mesh % faces(fid) % geom % normal(:,i,j)) ) & + * mesh % faces(fid) % geom % jacobian(i,j) & + * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) + + localx = localx + localval(1) + localy = localy + localval(2) + localz = localz + localval(3) + end do ; end do + + valx = valx + localx + valy = valy + localy + valz = valz + localz + + end do +#ifdef _OPENACC +!$acc end parallel loop +#else +!$omp end parallel do +#endif + + case ( PRESSURE_FORCE ) + +#ifdef _OPENACC +!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#else +!$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + ! + ! Face global ID + ! -------------- + fID = mesh % zones(zoneID) % faces(zonefID) + + localx = 0.0_RP + localy = 0.0_RP + localz = 0.0_RP + ! + ! **************************************************** + ! Computes the pressure forces experienced by the zone + ! F = \int p \vec{n}ds + ! **************************************************** + ! + !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(val) + do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) + ! + ! Compute the integral + ! -------------------- + p = Pressure(mesh % faces(fid) % storage(1) % Q(:,i,j)) + + val = ( p * mesh % faces(fid) % geom % normal(:,i,j) ) * mesh % faces(fid) % geom % jacobian(i,j) & + * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) & + * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) + + localx = localx + val(1) + localy = localy + val(2) + localz = localz + val(3) + end do ; end do + + valx = valx + localx + valy = valy + localy + valz = valz + localz + + end do +#ifdef _OPENACC +!$acc end parallel loop +#else +!$omp end parallel do +#endif + + case ( VISCOUS_FORCE ) + +#ifdef _OPENACC +!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) +#else +!$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + ! + ! Face global ID + ! -------------- + fID = mesh % zones(zoneID) % faces(zonefID) + + localx = 0.0_RP + localy = 0.0_RP + localz = 0.0_RP + ! + ! ************************************************ + ! Computes the total force experienced by the zone + ! F = - \int tau'·\vec{n}ds + ! ************************************************ + ! + !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(tau, val) + do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) + ! + ! Compute the integral + ! -------------------- + call getStressTensor(mesh % faces(fid) % storage(1) % Q(:,i,j), mesh % faces(fid) % storage(1) % U_x(:,i,j),& + mesh % faces(fid) % storage(1) % U_y(:,i,j),mesh % faces(fid) % storage(1) % U_z(:,i,j), tau) + + val = - matmul(tau,mesh % faces(fid) % geom % normal(:,i,j)) * mesh % faces(fid) % geom % jacobian(i,j) & + * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) & + * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) + + localx = localx + val(1) + localy = localy + val(2) + localz = localz + val(3) + end do ; end do + + valx = valx + localx + valy = valy + localy + valz = valz + localz + + end do +#ifdef _OPENACC +!$acc end parallel loop +#else +!$omp end parallel do +#endif + + end select + + val(1:3) = [valx, valy, valz] + +#ifdef _HAS_MPI_ + localVal = val + call mpi_allreduce(localVal, val, NDIM, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) +#endif + + end function VectorSurfaceIntegral +! +!//////////////////////////////////////////////////////////////////////////////////////// +! +! INTEGRALS PROCEDURES FOR IBM DATA RECONSTRUCTION +! +! SURFACE INTEGRALS +! +!//////////////////////////////////////////////////////////////////////////////////////// + subroutine ScalarDataReconstruction( IBM, elements, STLNum, integralType, iter, autosave, dt ) + use TessellationTypes + use MappedGeometryClass + use IBMClass + use OrientedBoundingBox + use KDClass + use MPI_Process_Info + use MPI_IBMUtilities +#ifdef _HAS_MPI_ + use mpi +#endif +! +! ----------------------------------------------------------------------------------------- +! This function computes Scalar integrals, that is, those +! in the form: +! val = \int \vec{v}·\vec{n}dS +! The data at the boundary point (BP) is computed through a Inverse Distance Weight +! procedure. +! ----------------------------------------------------------------------------------------- + implicit none + !-arguments-------------------------------------------------------- + type(IBM_type), intent(inout) :: IBM + type(element), intent(inout) :: elements(:) + integer, intent(in) :: integralType, STLNum, iter + real(kind=RP), intent(in) :: dt + !-local-variables------------------------------------------------- + real(kind=rp), allocatable :: Qsurf(:,:), U_xsurf(:,:), U_ysurf(:,:), U_zsurf(:,:) + integer :: i, j + logical :: found, autosave + + if( .not. IBM% Integral(STLNum)% compute ) return + + allocate( Qsurf(NCONS,IBM% NumOfInterPoints) ) + call IBM% BandPoint_state( elements, STLNum, .true. ) + + if( IBM% stlSurfaceIntegrals(STLNum)% move ) then + if( IBM% stlSurfaceIntegrals(STLNum)% motionType .eq. ROTATION ) then + call IBM% stlSurfaceIntegrals(STLNum)% getRotationaMatrix( dt ) + call OBB(STLNum)% STL_rotate( IBM% stlSurfaceIntegrals(STLNum), .true. ) + elseif( IBM% stlSurfaceIntegrals(STLNum)% motionType .eq. LINEAR ) then + call IBM% stlSurfaceIntegrals(STLNum)% getDisplacement( dt ) + call OBB(STLNum)% STL_translate( IBM% stlSurfaceIntegrals(STLNum), .true. ) + end if + end if + + if( .not. MPI_Process% isRoot ) return +!$omp parallel +!$omp do schedule(runtime) private(j,found) + do i = 1, IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs + + do j = 1, NumOfVertices + 4 + call GetSurfaceState( IBM, IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i), IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j), STLNum ) + Qsurf = IBM% BandRegion(STLNum)% Q(:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) + end do + + do j = 1, NumOfVertices + 4 + IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% ScalarValue = IntegratedScalarValue( Q = Qsurf, & + vertex = IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j), & + normal = IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% normal, & + integralType = integralType, & + InterpolationType = IBM% InterpolationType ) + end do + end do +!$omp end do +!$omp end parallel + if( IBM% stl(STLNum)% move ) then + IBM% Integral(STLNum)% ListComputed = .false. + else + IBM% Integral(STLNum)% ListComputed = .true. + end if + + if( autosave ) call GenerateScalarmonitorTECfile( IBM, STLNum, integralType, iter ) + + end subroutine ScalarDataReconstruction +! +!//////////////////////////////////////////////////////////////////////////////////////// +! +! VECTOR INTEGRALS +! +!//////////////////////////////////////////////////////////////////////////////////////// + subroutine VectorDataReconstruction( IBM, elements, STLNum, integralType, iter, autosave, dt ) + use TessellationTypes + use MappedGeometryClass + use IBMClass + use OrientedBoundingBox + use KDClass + use MPI_Process_Info + use MPI_IBMUtilities + use omp_lib +#ifdef _HAS_MPI_ + use mpi +#endif +! +! ----------------------------------------------------------------------------------------- +! This function computes Vector integrals, that is, those +! in the form: +! val = \int \vec{v}·\vec{n}dS +! The data at the boundary point (BP) is computed through a Inverse Distance Weight +! procedure. +! ----------------------------------------------------------------------------------------- + implicit none + !-arguments--------------------------------------------------------------------------------- + type(IBM_type), intent(inout) :: IBM + type(element), intent(inout) :: elements(:) + integer, intent(in) :: integralType, STLNum, iter + real(kind=RP), intent(in) :: dt + !-local-variables--------------------------------------------------------------------------- + real(kind=rp), allocatable :: Qsurf(:,:,:), U_xsurf(:,:,:), U_ysurf(:,:,:), U_zsurf(:,:,:) + integer :: i, j + logical :: found, autosave + + if( .not. IBM% Integral(STLNum)% compute ) return + + allocate( Qsurf(NCONS,IBM% NumOfInterPoints,NumOfVertices + 4), & + U_xsurf(NCONS,IBM% NumOfInterPoints,NumOfVertices + 4), & + U_ysurf(NCONS,IBM% NumOfInterPoints,NumOfVertices + 4), & + U_zsurf(NCONS,IBM% NumOfInterPoints,NumOfVertices + 4) ) + call IBM% BandPoint_state( elements, STLNum, .true. ) + + if( .not. MPI_Process% isRoot ) return + + if( IBM% stlSurfaceIntegrals(STLNum)% move ) then + if( IBM% stlSurfaceIntegrals(STLNum)% motionType .eq. ROTATION ) then + call IBM% stlSurfaceIntegrals(STLNum)% getRotationaMatrix( dt ) + call OBB(STLNum)% STL_rotate( IBM% stlSurfaceIntegrals(STLNum), .true. ) + elseif( IBM% stlSurfaceIntegrals(STLNum)% motionType .eq. LINEAR ) then + call IBM% stlSurfaceIntegrals(STLNum)% getDisplacement( dt ) + call OBB(STLNum)% STL_translate( IBM% stlSurfaceIntegrals(STLNum), .true. ) + end if + end if +!$omp parallel +!$omp do schedule(runtime) private(j,found,Qsurf,U_xsurf,U_ysurf,U_zsurf) + do i = 1, IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs + + do j = 1, NumOfVertices + 4 + call GetSurfaceState( IBM, IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i), IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j), STLNum ) + + Qsurf(:,:,j) = IBM% BandRegion(STLNum)% Q (:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) + U_xsurf(:,:,j) = IBM% BandRegion(STLNum)% U_x(:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) + U_ysurf(:,:,j) = IBM% BandRegion(STLNum)% U_y(:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) + U_zsurf(:,:,j) = IBM% BandRegion(STLNum)% U_z(:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) + end do + + do j = 1, NumOfVertices + 4 + IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% VectorValue = IntegratedVectorValue( Q = Qsurf(:,:,j), & + U_x = U_xsurf(:,:,j), & + U_y = U_ysurf(:,:,j), & + U_z = U_zsurf(:,:,j), & + vertex = IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j), & + normal = IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% normal, & + y = IBM% IP_Distance, & + Wallfunction = IBM% Wallfunction, & + integralType = integralType, & + InterpolationType = IBM% InterpolationType ) + end do + end do +!$omp end do +!$omp end parallel + deallocate( Qsurf, U_xsurf, U_ysurf, U_zsurf ) + + if( IBM% stl(STLNum)% move ) then + IBM% Integral(STLNum)% ListComputed = .false. + else + IBM% Integral(STLNum)% ListComputed = .true. + end if + + if( autosave ) call GenerateVectormonitorTECfile( IBM, STLNum, integralType, iter ) + + end subroutine VectorDataReconstruction + + subroutine GetSurfaceState( IBM, obj, vertex, STLNum ) + use TessellationTypes + use IBMClass + use MPI_Process_Info + use omp_lib +#ifdef _HAS_MPI_ + use mpi +#endif + implicit none + + class(IBM_type), intent(inout) :: IBM + type(object_type), intent(inout) :: obj + type(point_type), intent(inout) :: vertex + integer, intent(in) :: STLNum + + real(kind=RP) :: Dist + integer :: i, j, k + + if( IBM% Integral(STLNum)% ListComputed ) return + + vertex% nearestPoints = 0 + do k = 1, IBM% NumOfInterPoints + if( IBM% Wallfunction ) then + call MinimumDistancePoints( vertex% coords + IBM% IP_Distance * obj% Normal, & + IBM% rootPoints(STLNum), IBM% BandRegion(STLNum), & + Dist, k, vertex% nearestPoints ) + else + call MinimumDistancePoints( vertex% coords, IBM% rootPoints(STLNum), & + IBM% BandRegion(STLNum), Dist, k, & + vertex% nearestPoints ) + end if + end do + call GetMatrixInterpolationSystem( vertex% coords, & + IBM% BandRegion(STLNum)% x(vertex% nearestPoints), & + vertex% invPhi, & + vertex% b, IBM% InterpolationType ) + + end subroutine GetSurfaceState + + subroutine GetSurfaceState_HO( IBM, obj, vertex, STLNum, elements, Qs, U_xs, U_ys, U_zs, gradients, found ) + use TessellationTypes + use IBMClass + use MPI_Process_Info + use omp_lib +#ifdef _HAS_MPI_ + use mpi +#endif + implicit none + + class(IBM_type), intent(in) :: IBM + type(object_type), intent(inout) :: obj + type(point_type), intent(inout) :: vertex + integer, intent(in) :: STLNum + type(element), intent(inout) :: elements(:) + real(kind=RP), intent(inout) :: Qs(NCONS,1) + real(kind=RP), intent(inout) :: U_xs(NCONS,1), U_ys(NCONS,1), U_zs(NCONS,1) + logical, intent(in) :: gradients + logical, intent(out) :: found + + real(kind=RP) :: xi(NDIM) + integer :: eID, i, j, k + + Qs = 0.0_RP + if( gradients ) then + U_xs = 0.0_RP; U_ys = 0.0_RP; U_zs = 0.0_RP + end if + + if( IBM% Integral(STLNum)% ListComputed ) then + if( vertex% partition .eq. MPI_Process% rank ) then + eID = vertex% element_index + xi = vertex% xi + + associate( e => elements(eID) ) + + Qs(:,1) = elements(eID)% EvaluateSolutionAtPoint(NCONS, xi) + + if( gradients ) then + U_xs(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IX) + U_ys(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IY) + U_zs(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IZ) + end if + + end associate + found = .true. + else + found = .false. + end if + return + end if + + do eID = 1, size(elements) + associate(e => elements(eID) ) + found = e% FindPointWithCoords( vertex% coords, 0, xi ) + if( found ) then + vertex% element_index = eID + vertex% partition = MPI_Process% rank + vertex% xi = xi + + Qs(:,1) = elements(eID)% EvaluateSolutionAtPoint(NCONS, xi) + + if( gradients ) then + U_xs(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IX) + U_ys(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IY) + U_zs(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IZ) + end if + exit + end if + end associate + end do + + end subroutine GetSurfaceState_HO +! +!//////////////////////////////////////////////////////////////////////////////////////// +! +! INVERSE DISTANCE WEIGHTED INTERPOLATION PROCEDURES FOR IBM DATA RECONSTRUCTION +! +! SCALAR INTERPOLATION +! +!//////////////////////////////////////////////////////////////////////////////////////// + function IntegratedScalarValue( Q, vertex, normal, integralType, InterpolationType ) result( outvalue ) + use IBMClass + implicit none +! +! ----------------------------------------------------------- +! This function computes the IDW interpolat for a scalar +! quantity in the point "Point". +! Available scalars are: +! Mass flow +! Flow rate +! Pressure +! ----------------------------------------------------------- + !-arguments-------------------------------------------------------------- + real(kind=rp), intent(in) :: Q(:,:), normal(:) + type(point_type), intent(inout) :: vertex + integer, intent(in) :: integralType, InterpolationType + real(kind=rp) :: outvalue + !-local-variables-------------------------------------------------------- + real(kind=rp) :: Qi(NCONS), P + integer :: i + + outvalue = 0.0_RP + + select case( integralType ) + + case( MASS_FLOW ) + + do i = 1, NCONS + Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) + end do + +#if defined(NAVIERSTOKES) + outvalue = - (1.0_RP / Qi(IRHO))*(Qi(IRHOU)*normal(1) + Qi(IRHOV)*normal(2) + Qi(IRHOW)*normal(3)) +#endif +#if defined(INCNS) + outvalue = - (1.0_RP / Qi(INSRHO))*(Qi(INSRHOU)*normal(1) + Qi(INSRHOV)*normal(2) + Qi(INSRHOW)*normal(3)) +#endif + case ( FLOW_RATE ) + + do i = 1, NCONS + Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) + end do + +#if defined(NAVIERSTOKES) + outvalue = - (Qi(IRHOU)*normal(1) + Qi(IRHOV)*normal(2) + Qi(IRHOW)*normal(3)) +#endif +#if defined(INCNS) + outvalue = - (Qi(INSRHOU)*normal(1) + Qi(INSRHOV)*normal(2) + Qi(INSRHOW)*normal(3)) +#endif + case( PRESSURE_DISTRIBUTION ) + + do i = 1, NCONS + Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) + end do + + outvalue = pressure(Qi) + case ( USER_DEFINED ) ! TODO + + end select + + end function IntegratedScalarValue +! +!//////////////////////////////////////////////////////////////////////////////////////// +! +! VECTOR INTERPOLATION +! +!//////////////////////////////////////////////////////////////////////////////////////// + function IntegratedVectorValue( Q, U_x, U_y, U_z, vertex, normal, & + y, Wallfunction, integralType, & + InterpolationType ) result( outvalue ) + use IBMClass + use VariableConversion + use FluidData +#if defined(NAVIERSTOKES) + use WallFunctionBC +#endif + implicit none +! +! ----------------------------------------------------------- +! This function computes the IDW interpolat for a vector +! quantity in the point "Point". +! Available scalars are: +! Total force +! Pressure force +! Viscous force +! ----------------------------------------------------------- + !-arguments----------------------------------------------------------------- + real(kind=rp), intent(in) :: Q(:,:), U_x(:,:), U_y(:,:), & + U_z(:,:), normal(NDIM) + type(point_type), intent(inout) :: vertex + real(kind=rp), intent(in) :: y + logical, intent(in) :: Wallfunction + integer, intent(in) :: integralType, InterpolationType + real(kind=rp) :: outvalue(NDIM) + !-local-variables----------------------------------------------------------- + integer :: i + real(kind=rp) :: viscStress(NDIM), U(NDIM), U_t(NDIM), tangent(NDIM), & + Qi(NCONS), U_xi(NCONS), U_yi(NCONS), U_zi(NCONS), & + tau(NDIM,NDIM), P, T, T_w, rho_w, mu, nu, u_II, u_tau, & + tau_w, kappa_ + + outvalue = 0.0_RP + + select case( integralType ) + + case ( TOTAL_FORCE ) + + do i = 1, NCONS + Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) + end do + + P = pressure(Qi) + + if( Wallfunction ) then +#if defined(NAVIERSTOKES) + T = Temperature(Qi) + call get_laminar_mu_kappa(Qi,mu,kappa_) + nu = mu/Qi(IRHO) + + U = Qi(IRHOU:IRHOW)/Qi(IRHO) + U_t = U - ( dot_product(U,normal) * normal ) + + tangent = U_t/norm2(U_t) + + u_II = dot_product(U,tangent) + + u_tau = u_tau_f( u_II, y, nu, u_tau0=0.1_RP ) + + T_w = T + (dimensionless% Pr)**(1._RP/3._RP)/(2.0_RP*thermodynamics% cp) * POW2(u_II) + T_w = T_w * refvalues% T + rho_w = P*refvalues% p/(thermodynamics% R * T_w) + rho_w = rho_w/refvalues% rho +#endif + tau_w = rho_w*POW2(u_tau) + + viscStress = tau_w*tangent + else + + do i = 1, NCONS + U_xi(i) = GetInterpolatedValue( U_x(i,:), vertex% invPhi, vertex% b, InterpolationType ) + U_yi(i) = GetInterpolatedValue( U_y(i,:), vertex% invPhi, vertex% b, InterpolationType ) + U_zi(i) = GetInterpolatedValue( U_z(i,:), vertex% invPhi, vertex% b, InterpolationType ) + end do + + call getStressTensor(Qi, U_xi, U_yi, U_zi, tau) + + viscStress = matmul(tau,normal) + end if + + outvalue = -P * normal + viscStress + + case( PRESSURE_FORCE ) + + do i = 1, NCONS + Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) + end do + + P = pressure(Qi) + + outvalue = -P * normal + + case( VISCOUS_FORCE ) + + if( Wallfunction ) then +#if defined(NAVIERSTOKES) + T = Temperature(Qi) + call get_laminar_mu_kappa(Qi,mu,kappa_) + nu = mu/Qi(IRHO) + + U = Qi(IRHOU:IRHOW)/Qi(IRHO) + U_t = U - ( dot_product(U,normal) * normal ) + + tangent = U_t/norm2(U_t) + + u_II = dot_product(U,tangent) + + u_tau = u_tau_f( u_II, y, nu, u_tau0=0.1_RP ) + + T_w = T + (dimensionless% Pr)**(1._RP/3._RP)/(2.0_RP*thermodynamics% cp) * POW2(u_II) + T_w = T_w * refvalues% T + rho_w = P*refvalues% p/(thermodynamics% R * T_w) + rho_w = rho_w/refvalues% rho +#endif + tau_w = rho_w*POW2(u_tau) + + viscStress = tau_w*tangent + else + + do i = 1, NCONS + U_xi(i) = GetInterpolatedValue( U_x(i,:), vertex% invPhi, vertex% b, InterpolationType ) + U_yi(i) = GetInterpolatedValue( U_y(i,:), vertex% invPhi, vertex% b, InterpolationType ) + U_zi(i) = GetInterpolatedValue( U_z(i,:), vertex% invPhi, vertex% b, InterpolationType ) + end do + + call getStressTensor(Qi, U_xi, U_yi, U_zi, tau) + + viscStress = matmul(tau,normal) + end if + + outvalue = viscStress + + case ( USER_DEFINED ) ! TODO + + end select + + end function IntegratedVectorValue + + subroutine GenerateScalarmonitorTECfile( IBM, STLNum, integralType, iter ) + use MPI_Process_Info + use TessellationTypes + use MPI_IBMUtilities + use IBMClass + implicit none + !-arguments------------------------------------------------------- + type(IBM_type), intent(in) :: IBM + integer, intent(in) :: STLNum, integralType, iter + !-local-variables------------------------------------------------- + real(kind=RP), allocatable :: x(:), y(:), z(:), scalar(:), & + local_sum(:), global_sum(:) + integer :: i, j, index, NumOfObjs + character(len=LINE_LENGTH) :: FileName, FinalName +#ifdef _HAS_MPI_ + integer :: ierr +#endif + NumOfObjs = NumOfVertices * IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs + + allocate( x(NumOfObjs), & + y(NumOfObjs), & + z(NumOfObjs), & + scalar(NumOfObjs) ) + + index = 0 + + do i = 1, IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs + associate( obj => IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i) ) + do j = 1, NumOfVertices + index = index + 1 + x(index) = obj% vertices(j)% coords(IX) + y(index) = obj% vertices(j)% coords(IY) + z(index) = obj% vertices(j)% coords(IZ) + scalar(index) = obj% vertices(j)% ScalarValue + end do + end associate + end do +#ifdef _HAS_MPI_ + if( MPI_Process% doMPIAction ) then + allocate(local_sum(NumOfObjs),global_sum(NumOfObjs)) + local_sum = scalar + call mpi_allreduce(local_sum, global_sum, NumOfObjs, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) + scalar = global_sum + deallocate(local_sum,global_sum) + end if +#endif + if( .not. MPI_Process% isRoot ) then + deallocate(x, y, z, scalar) + return + end if + + if( .not. MPI_Process% isRoot ) return + + select case(integralType) + case( MASS_FLOW ) + FileName = 'MASS_FLOW_' + write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' + call STLScalarTEC( x, y, z, scalar, STLNum, FinalName, 'MASS FLOW', '"x","y","z","MassFlow"' ) + case( FLOW_RATE ) + FileName = 'FLOW_RATE_FORCE_' + write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' + call STLScalarTEC( x, y, z, scalar, STLNum, FinalName, 'FLOW RATE', '"x","y","z","FlowRate"' ) + case( PRESSURE_DISTRIBUTION ) + FileName = 'PRESSURE_' + write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' + call STLScalarTEC( x, y, z, scalar, STLNum, FinalName, 'PRESSURE DISTRIBUTION', '"x","y","z","Pressure"' ) + end select + + deallocate(x, y, z, scalar) + + end subroutine GenerateScalarmonitorTECfile + + subroutine GenerateVectormonitorTECfile( IBM, STLNum, integralType, iter ) + use MPI_Process_Info + use TessellationTypes + use MPI_IBMUtilities + use IBMClass + implicit none + !-arguments--------------------------------------------------------- + type(IBM_type), intent(in) :: IBM + integer, intent(in) :: STLNum, integralType, iter + !-local-variables--------------------------------------------------- + real(kind=RP), allocatable :: x(:), y(:), z(:), vector_x(:), & + vector_y(:), vector_z(:), & + local_sum(:), global_sum(:) + character(len=LINE_LENGTH) :: FileName, FinalName + integer :: index, NumOfObjs, i, j +#ifdef _HAS_MPI_ + integer :: ierr +#endif + NumOfObjs = NumOfVertices * IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs + + allocate( x(NumOfObjs), & + y(NumOfObjs), & + z(NumOfObjs), & + vector_x(NumOfObjs), & + vector_y(NumOfObjs), & + vector_z(NumOfObjs) ) + + index = 0 + + do i = 1, IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs + associate( obj => IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i) ) + do j = 1, NumOfVertices + index = index + 1 + x(index) = obj% vertices(j)% coords(IX) + y(index) = obj% vertices(j)% coords(IY) + z(index) = obj% vertices(j)% coords(IZ) + vector_x(index) = obj% vertices(j)% VectorValue(IX) + vector_y(index) = obj% vertices(j)% VectorValue(IY) + vector_z(index) = obj% vertices(j)% VectorValue(IZ) + end do + end associate + end do +#ifdef _HAS_MPI_ + if( MPI_Process% doMPIAction ) then + allocate(local_sum(NumOfObjs),global_sum(NumOfObjs)) + local_sum = vector_x + call mpi_allreduce(local_sum, global_sum, NumOfObjs, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) + vector_x = global_sum + local_sum = vector_y + call mpi_allreduce(local_sum, global_sum, NumOfObjs, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) + vector_y = global_sum + local_sum = vector_z + call mpi_allreduce(local_sum, global_sum, NumOfObjs, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) + vector_z = global_sum + deallocate(local_sum,global_sum) + end if +#endif + if( .not. MPI_Process% isRoot ) then + deallocate(x, y, z, vector_x, vector_y, vector_z) + return + end if + + select case(integralType) + case( TOTAL_FORCE ) + FileName = 'TOTAL_FORCE_' + write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' + call STLvectorTEC( x, y, z, vector_x, vector_y, vector_z, STLNum, FinalName, 'TOTAL FORCE', '"x","y","z","Ftot_x","Ftot_y","Ftot_z"' ) + case( PRESSURE_FORCE ) + FileName = 'PRESSURE_FORCE_' + write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' + call STLvectorTEC( x, y, z, vector_x, vector_y, vector_z, STLNum, FinalName, 'PRESSURE FORCE', '"x","y","z","Fpres_x","Fpres_y","Fpres_z"' ) + case( VISCOUS_FORCE ) + FileName = 'VISCOUS_FORCE_' + write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' + call STLvectorTEC( x, y, z, vector_x, vector_y, vector_z, STLNum, FinalName, 'VISCOUS FORCE', '"x","y","z","Fvisc_x","Fvisc_y","Fvisc_z"' ) + end select + + deallocate(x, y, z, vector_x, vector_y, vector_z) + + end subroutine GenerateVectormonitorTECfile + + subroutine ProlongToFaces(mesh, zoneID) + implicit none + type(HexMesh), intent(inout) :: mesh + integer, intent(in) :: zoneID + + !-------- local variables ---------- + integer :: zonefID, fID, eID + + !$acc wait + + select case ( mesh %nodeType ) + case(1) !Gauss-Legendre + !$acc parallel loop gang present(mesh) + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + fID = mesh % zones(zoneID) % faces(zonefID) + eID = mesh % faces(fID) % elementIDs(1) + call HexElement_ProlongSolToFaces(mesh % elements(eID), NCONS, mesh % faces(fID), mesh % faces(fID) % elementSide(1)) + end do + !$acc end parallel loop + + case(2) + !$acc parallel loop gang present(mesh) + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + fID = mesh % zones(zoneID) % faces(zonefID) + eID = mesh % faces(fID) % elementIDs(1) + call HexElement_ProlongSolToFaces_GL(mesh % elements(eID), NCONS, mesh % faces(fID), mesh % faces(fID) % elementSide(1)) + end do + !$acc end parallel loop + end select + + if ( computeGradients ) then + select case ( mesh %nodeType ) + case(1) !Gauss + +#ifdef _OPENACC + !$acc parallel loop gang present(mesh) +#else + !$omp do schedule(runtime) private(eID) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + fID = mesh % zones(zoneID) % faces(zonefID) + eID = mesh % faces(fID) % elementIDs(1) + call HexElement_ProlongGradientsToFaces(mesh % elements(eID), NGRAD, & + mesh % faces(fID), & + mesh % elements(eID) % storage % U_x, mesh % faces(fID) % elementSide(1), 1) + + call HexElement_ProlongGradientsToFaces(mesh % elements(eID), NGRAD, & + mesh % faces(fID), & + mesh % elements(eID) % storage % U_y, mesh % faces(fID) % elementSide(1), 2) + + call HexElement_ProlongGradientsToFaces(mesh % elements(eID), NGRAD, & + mesh % faces(fID), & + mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1), 3) + end do +#ifdef _OPENACC + !$acc end parallel loop +#else + !$omp end do +#endif + + case(2) !Gauss-Lobatto + +#ifdef _OPENACC + !$acc parallel loop gang present(mesh) private(fID) +#else + !$omp do schedule(runtime) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + fID = mesh % zones(zoneID) % faces(zonefID) + eID = mesh % faces(fID) % elementIDs(1) + call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & + mesh % faces(fID), & + mesh % elements(eID) % storage % U_x, mesh % faces(fID) % elementSide(1),1) + end do +#ifdef _OPENACC + !$acc end parallel loop +#else + !$omp end do +#endif + +#ifdef _OPENACC + !$acc parallel loop gang present(mesh) private(fID) +#else + !$omp do schedule(runtime) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + fID = mesh % zones(zoneID) % faces(zonefID) + eID = mesh % faces(fID) % elementIDs(1) + call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & + mesh % faces(fID), & + mesh % elements(eID) % storage % U_y, mesh % faces(fID) % elementSide(1),2) + end do +#ifdef _OPENACC + !$acc end parallel loop +#else + !$omp end do +#endif + +#ifdef _OPENACC + !$acc parallel loop gang present(mesh) private(fID) +#else + !$omp do schedule(runtime) +#endif + do zonefID = 1, mesh % zones(zoneID) % no_of_faces + fID = mesh % zones(zoneID) % faces(zonefID) + eID = mesh % faces(fID) % elementIDs(1) + call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & + mesh % faces(fID), & + mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1),3) + end do +#ifdef _OPENACC + !$acc end parallel loop +#else + !$omp end do +#endif + end select + end if +! +! Loop the zone to get faces and elements +! --------------------------------------- + !$acc wait + + end subroutine ProlongToFaces + +end module SurfaceIntegrals +#endif + + diff --git a/Solver/src/libs/mesh/SurfaceMonitor.f90 b/Solver/src/libs/mesh/SurfaceMonitor.f90 new file mode 100644 index 000000000..161dae0f9 --- /dev/null +++ b/Solver/src/libs/mesh/SurfaceMonitor.f90 @@ -0,0 +1,560 @@ +#include "Includes.h" +module SurfaceMonitorClass + use SMConstants + use HexMeshClass + use MonitorDefinitions + use PhysicsStorage + use MPI_Process_Info + use FluidData + use FileReadingUtilities, only: getRealArrayFromString + implicit none + + +#if defined(NAVIERSTOKES) || defined(INCNS) + private + public SurfaceMonitor_t + + +! +! ******************************** +! Surface monitor class definition +! ******************************** +! + type SurfaceMonitor_t + logical :: active + logical :: isDimensionless, IBM = .false. + integer :: ID + real(kind=RP) :: direction(NDIM) + integer :: marker + real(kind=RP), allocatable :: referenceSurface + real(kind=RP), allocatable :: values(:) + real(kind=RP) :: dynamicPressure + character(len=STR_LEN_MONITORS) :: monitorName + character(len=STR_LEN_MONITORS) :: fileName + character(len=STR_LEN_MONITORS) :: variable + contains + procedure :: Initialization => SurfaceMonitor_Initialization + procedure :: Update => SurfaceMonitor_Update + procedure :: WriteLabel => SurfaceMonitor_WriteLabel + procedure :: WriteValues => SurfaceMonitor_WriteValue + procedure :: WriteToFile => SurfaceMonitor_WriteToFile + procedure :: destruct => SurfaceMonitor_Destruct + procedure :: copy => SurfaceMonitor_Assign + generic :: assignment(=) => copy + end type SurfaceMonitor_t + + contains +! +!///////////////////////////////////////////////////////////////////////// +! +! SURFACE MONITOR PROCEDURES +! -------------------------- +!///////////////////////////////////////////////////////////////////////// +! + subroutine SurfaceMonitor_Initialization( self , mesh , ID, solution_file , FirstCall) +! +! ***************************************************************************** +! This subroutine initializes the surface monitor. The following +! data is obtained from the case file: +! -> Name: The monitor name (10 characters maximum) +! -> Marker: The surface marker in which the monitor will be computed. +! -> Variable: The variable to be monitorized. +! -> Reference surface (optional): Reference surface for lift/drag coefficients +! -> Direction (optional): Direction in which the forces are computed +! ***************************************************************************** +! + use ParamfileRegions + implicit none + class(SurfaceMonitor_t) :: self + class(HexMesh) :: mesh + integer :: ID + character(len=*) :: solution_file + logical, intent(in) :: FirstCall +! +! --------------- +! Local variables +! --------------- +! + character(len=STR_LEN_MONITORS) :: in_label + character(len=STR_LEN_MONITORS) :: fileName + character(len=STR_LEN_MONITORS) :: paramFile + character(len=STR_LEN_MONITORS) :: directionName + integer, allocatable :: marker + character(len=STR_LEN_MONITORS) :: markerName + integer :: pos, i, STLNum + integer :: fID + integer :: zoneID + real(kind=RP) :: directionValue(NDIM) +! +! Get monitor ID +! -------------- + self % ID = ID +! +! Search for the parameters in the case file +! ------------------------------------------ + write(in_label , '(A,I0)') "#define surface monitor " , self % ID + + call get_command_argument(1, paramFile) + call readValueInRegion ( trim ( paramFile ) , "name" , self % monitorName , in_label , "# end" ) + call readValueInRegion ( trim ( paramFile ) , "marker" , markerName , in_label , "# end" ) + call readValueInRegion ( trim ( paramFile ) , "variable" , self % variable , in_label , "# end" ) + call readValueInRegion ( trim ( paramFile ) , "reference surface" , self % referenceSurface , in_label , "# end" ) + call readValueInRegion ( trim ( paramFile ) , "direction" , directionName , in_label , "# end" ) +! +! Enable the monitor +! ------------------ + self % active = .true. + allocate ( self % values(BUFFER_SIZE) ) +! +! Get the surface marker +! ---------------------- + self % marker = -1 + if( mesh% IBM% active ) then + do STLNum = 1, mesh% IBM% NumOfSTL + if( mesh% IBM% Integral(STLNum)% compute ) cycle + if( trim(mesh% IBM% STLfilename(STLNum)) .eq. trim(markerName) ) then + if( .not. mesh% IBM% ComputeBandRegion ) then + write(*,'(A)') "Warning: for surface monitors with IBM, 'band region' must be set '.true.'" + error stop + end if + self% marker = STLNum + self% IBM = .true. + mesh% IBM% Integral(STLNum)% compute = .true. + mesh% IBM% Integral(STLNum)% ListComputed = .false. + if( MPI_Process% isRoot ) then + call mesh% IBM% stlSurfaceIntegrals(STLNum)% ReadTessellation( mesh% IBM% STLfilename(STLNum) ) + if( mesh% IBM% ClipAxis .ne. 0 ) call mesh% IBM% stlSurfaceIntegrals(STLNum)% Clip( mesh% IBM% minCOORDS, mesh% IBM% maxCOORDS, mesh% IBM% ClipAxis, .false. ) + call mesh% IBM% stlSurfaceIntegrals(STLNum)% SetIntegrationPoints() + call mesh% IBM% stlSurfaceIntegrals(STLNum)% SetIntegration( mesh% IBM% NumOfInterPoints ) + end if + call mesh% IBM% SetIntegration( STLNum ) + exit + end if + end do + else + do zoneID = 1, size(mesh % zones) + if ( trim(mesh % zones(zoneID) % name) .eq. trim(markerName) ) then + self % marker = zoneID + exit + end if + end do + end if + + if ( self % marker .eq. -1 ) then + self % active = .false. + write(*,'(A,I0)') "Warning: Marker not specified for surface monitor ", self % ID + write(*,'(A,I0,A)') " Surface monitor ", self % ID, " disabled." + end if +! +! Select the variable from the available list, and compute auxiliary variables if needed +! -------------------------------------------------------------------------------------- +! +! **************************************** + select case ( trim ( self % variable ) ) +! **************************************** +! + case ("mass-flow") + self % isDimensionless = .false. + + case ("flow") + self % isDimensionless = .false. + + case ("pressure-force") + self % isDimensionless = .false. + if ( len_trim(directionName) .eq. 0 ) then + print*, "Direction not specified for pressure-force in surface monitor " , self % ID , "." + error stop "error stopped" + + else + directionValue = getRealArrayFromString(directionName) + if ( size(directionValue) .ne. 3 ) then + print*, "Incorrect direction for monitor ", self % ID, "." + + else + self % direction = directionValue + + end if + end if + + case ("viscous-force") + self % isDimensionless = .false. + if ( len_trim(directionName) .eq. 0 ) then + print*, "Direction not specified for pressure-force in surface monitor " , self % ID , "." + error stop "error stopped" + + else + directionValue = getRealArrayFromString(directionName) + if ( size(directionValue) .ne. 3 ) then + print*, "Incorrect direction for monitor ", self % ID, "." + + else + self % direction = directionValue + + end if + end if + + case ("force") + self % isDimensionless = .false. + + if ( len_trim(directionName) .eq. 0 ) then + print*, "Direction not specified for pressure-force in surface monitor " , self % ID , "." + error stop "error stopped" + + else + directionValue = getRealArrayFromString(directionName) + if ( size(directionValue) .ne. 3 ) then + print*, "Incorrect direction for monitor ", self % ID, "." + + else + self % direction = directionValue + + end if + end if + + + case ("lift") + self % isDimensionless = .true. + + if ( .not. allocated ( self % referenceSurface ) ) then + print*, "Reference surface not specified for lift surface monitor " , self % ID , "." + error stop "error stopped" + end if + + if ( len_trim(directionName) .eq. 0 ) then + print*, "Direction not specified for lift in surface monitor " , self % ID , "." + print*, " ... Using [0,1,0] as default." + self % direction = [0._RP,1._RP,0._RP] + + else + directionValue = getRealArrayFromString(directionName) + if ( size(directionValue) .ne. 3 ) then + print*, "Incorrect direction for monitor ", self % ID, "." + + else + self % direction = directionValue + + end if + end if + + self % dynamicPressure = 0.5_RP * refValues % rho * POW2(refValues % V)* self % referenceSurface + + case ("drag") + self % isDimensionless = .true. + + if ( .not. allocated ( self % referenceSurface ) ) then + print*, "Reference surface not specified for drag surface monitor " , self % ID , "." + error stop "error stopped" + end if + + if ( len_trim(directionName) .eq. 0 ) then + print*, "Direction not specified for drag in surface monitor " , self % ID , "." + print*, " ... Using [1,0,0] as default." + self % direction = [1._RP,0._RP,0._RP] + + else + directionValue = getRealArrayFromString(directionName) + if ( size(directionValue) .ne. 3 ) then + print*, "Incorrect direction for monitor ", self % ID, "." + + else + self % direction = directionValue + + end if + end if + + self % dynamicPressure = 0.5_RP * refValues % rho * refValues % V * refValues % V * self % referenceSurface + + case ("pressure-average") + self % isDimensionless = .false. + + case default + + if ( len_trim (self % variable) .eq. 0 ) then + print*, "Variable was not specified for surface monitor " , self % ID , "." + else + print*, 'Variable "',trim(self % variable),'" surface monitor ', self % ID, ' not implemented yet.' + print*, "Options available are:" + print*, " * mass-flow" + print*, " * flow" + print*, " * pressure-force" + print*, " * viscous-force" + print*, " * force" + print*, " * lift" + print*, " * drag" + print*, " * pressure-average" + error stop "error stopped." + + end if +! +! ********** + end select +! ********** +! +! Prepare the file in which the monitor is exported +! ------------------------------------------------- + write( self % fileName , '(A,A,A,A)') trim(solution_file) , "." , trim(self % monitorName) , ".surface" +! +! Create file +! ----------- + if (FirstCall) then + open ( newunit = fID , file = trim(self % fileName) , status = "unknown" , action = "write" ) +! +! Write the file headers +! ---------------------- + write( fID , '(A20,A )') "Monitor name: ", trim(self % monitorName) + write( fID , '(A20,I0 )') "Surface marker: ", self % marker + write( fID , '(A20,A )') "Selected variable: " , trim(self % variable) + + if ( self % isDimensionless ) then + write(fID , '(A20,ES24.10)') "Dynamic pressure: " , self % dynamicPressure + end if + + write( fID , * ) + write( fID , '(A10,2X,A24,2X,A24)' ) "Iteration" , "Time" , trim(self % variable) + + close ( fID ) + end if + end subroutine SurfaceMonitor_Initialization + + subroutine SurfaceMonitor_Update ( self, mesh, bufferPosition, iter, autosave, dt ) +! +! ******************************************************************* +! This subroutine updates the monitor value computing it from +! the mesh. It is stored in the "bufferPosition" position of the +! buffer. +! ******************************************************************* +! + use SurfaceIntegrals + use IBMClass + implicit none + class ( SurfaceMonitor_t ) :: self + class ( HexMesh ) :: mesh + integer :: bufferPosition, iter, STLNum + real(kind=RP) :: F(NDIM) + real(kind=RP) :: dt + logical :: autosave + + select case ( trim ( self % variable ) ) + + case ("mass-flow") + if( self% IBM ) then + STLNum = self% marker + call ScalarDataReconstruction( mesh% IBM, mesh% elements, STLNum, MASS_FLOW, iter, autosave, dt ) + self % values(bufferPosition) = mesh% IBM% stlSurfaceIntegrals(STLNum)% ComputeScalarIntegral() + else + self % values(bufferPosition) = ScalarSurfaceIntegral(mesh, self % marker, MASS_FLOW, iter) + end if + case ("flow") + if( self% IBM ) then + STLNum = self% marker + call ScalarDataReconstruction( mesh% IBM, mesh% elements, STLNum, FLOW_RATE, iter, autosave, dt ) + self % values(bufferPosition) = mesh% IBM% stlSurfaceIntegrals(STLNum)% ComputeScalarIntegral() + else + self % values(bufferPosition) = ScalarSurfaceIntegral(mesh, self % marker, FLOW_RATE, iter) + end if + + case ("pressure-force") + if( self% IBM ) then + STLNum = self% marker + call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, PRESSURE_FORCE, iter, autosave, dt ) + F = mesh% IBM% stlSurfaceIntegrals(STLNum)% ComputeVectorIntegral() + else + F = VectorSurfaceIntegral(mesh, self % marker, PRESSURE_FORCE, iter) + end if + F = refValues % rho * POW2(refValues % V) * POW2(Lref) * F + self % values(bufferPosition) = dot_product(F, self % direction) + + case ("viscous-force") + if( self% IBM ) then + STLNum = self% marker + call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, VISCOUS_FORCE, iter, autosave, dt ) + F = mesh% IBM% stlSurfaceIntegrals(STLNum)% ComputeVectorIntegral() + else + F = VectorSurfaceIntegral(mesh, self % marker, VISCOUS_FORCE, iter) + end if + F = refValues % rho * POW2(refValues % V) * POW2(Lref) * F + self % values(bufferPosition) = dot_product(F, self % direction) + + case ("force") + if( self% IBM ) then + STLNum = self% marker + call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, TOTAL_FORCE, iter, autosave, dt ) + F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() + else + F = VectorSurfaceIntegral(mesh, self % marker, TOTAL_FORCE, iter) + end if + F = refValues % rho * POW2(refValues % V) * POW2(Lref) * F + self % values(bufferPosition) = dot_product(F, self % direction) + + case ("lift") + if( self% IBM ) then + STLNum = self% marker + call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, TOTAL_FORCE, iter, autosave, dt ) + F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() + else + F = VectorSurfaceIntegral(mesh, self % marker, TOTAL_FORCE, iter) + end if + F = 2.0_RP * POW2(Lref) * F / self % referenceSurface + self % values(bufferPosition) = dot_product(F, self % direction) + +#if defined (NAVIERSTOKES) + case ("drag") + if (flowIsNavierStokes) then + if( self% IBM ) then + STLNum = self% marker + call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, TOTAL_FORCE, iter, autosave, dt ) + F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() + else + F = VectorSurfaceIntegral(mesh, self % marker, TOTAL_FORCE, iter) + end if + else + if( self% IBM ) then + STLNum = self% marker + call VectorDataReconstruction( mesh% IBM, mesh% elements,STLNum, TOTAL_FORCE, iter, autosave, dt ) + F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() + else + F = VectorSurfaceIntegral(mesh, self % marker, PRESSURE_FORCE, iter) + end if + end if + F = 2.0_RP * POW2(Lref) * F / self % referenceSurface + self % values(bufferPosition) = dot_product(F, self % direction) +#endif +!TODO if true +#if defined (INCNS) + case ("drag") + if (.true.) then + if( self% IBM ) then + STLNum = self% marker + call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, TOTAL_FORCE, iter, autosave, dt ) + F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() + else + F = VectorSurfaceIntegral(mesh, self % marker, TOTAL_FORCE, iter) + end if + else + if( self% IBM ) then + STLNum = self% marker + call VectorDataReconstruction( mesh% IBM, mesh% elements,STLNum, TOTAL_FORCE, iter, autosave, dt ) + F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() + else + F = VectorSurfaceIntegral(mesh, self % marker, PRESSURE_FORCE, iter) + end if + end if + F = 2.0_RP * POW2(Lref) * F / self % referenceSurface + self % values(bufferPosition) = dot_product(F, self % direction) +#endif + case ("pressure-average") + self % values(bufferPosition) = ScalarSurfaceIntegral(mesh, self % marker, PRESSURE_FORCE, iter) / ScalarSurfaceIntegral(mesh, self % marker, SURFACE, iter) + + end select + + + end subroutine SurfaceMonitor_Update + + subroutine SurfaceMonitor_WriteLabel ( self ) +! +! ************************************************************* +! This subroutine writes the label for the surface +! monitor, when invoked from the time integrator Display +! procedure. +! ************************************************************* +! + implicit none + class(SurfaceMonitor_t), intent(in) :: self + + write(STD_OUT , '(3X,A10)' , advance = "no") trim(self % monitorName(1 : MONITOR_LENGTH)) + + end subroutine SurfaceMonitor_WriteLabel + + subroutine SurfaceMonitor_WriteValue ( self , bufferLine ) +! +! ************************************************************* +! This subroutine writes the monitor value for the time +! integrator Display procedure. +! ************************************************************* +! + implicit none + class(SurfaceMonitor_t) :: self + integer :: bufferLine + + write(STD_OUT , '(1X,A,1X,ES10.3)' , advance = "no") "|" , self % values ( bufferLine ) + + end subroutine SurfaceMonitor_WriteValue + + subroutine SurfaceMonitor_WriteToFile ( self , iter , t , no_of_lines) +! +! ************************************************************* +! This subroutine writes the buffer to the file. +! ************************************************************* +! + implicit none + class(SurfaceMonitor_t) :: self + integer :: iter(:) + real(kind=RP) :: t(:) + integer :: no_of_lines +! +! --------------- +! Local variables +! --------------- +! + integer :: i + integer :: fID + + if ( MPI_Process % isRoot ) then + + open( newunit = fID , file = trim ( self % fileName ) , action = "write" , access = "append" , status = "old" ) + + do i = 1 , no_of_lines + write( fID , '(I10,2X,ES24.16,2X,ES24.16)' ) iter(i) , t(i) , self % values(i) + + end do + + close ( fID ) + end if + + if ( no_of_lines .ne. 0 ) self % values(1) = self % values(no_of_lines) + + end subroutine SurfaceMonitor_WriteToFile +! + elemental subroutine SurfaceMonitor_Destruct (self) + implicit none + class(SurfaceMonitor_t), intent(inout) :: self + + safedeallocate (self % values) + safedeallocate (self % referenceSurface) + end subroutine SurfaceMonitor_Destruct +! +!////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +! + elemental subroutine SurfaceMonitor_Assign(to, from) + implicit none + class(SurfaceMonitor_t), intent(inout) :: to + type(SurfaceMonitor_t), intent(in) :: from + + if ( from % active) then + to % active = from % active + to % isDimensionless = from % isDimensionless + to % ID = from % ID + to % direction = from % direction + to % marker = from % marker + + safedeallocate(to % referenceSurface) + if ( allocated(from % referenceSurface) ) then + allocate (to % referenceSurface) + to % referenceSurface = from % referenceSurface + end if + + safedeallocate(to % values) + allocate ( to % values( size(from % values) ) ) + to % values = from % values + + to % dynamicPressure = from % dynamicPressure + to % monitorName = from % monitorName + to % fileName = from % fileName + to % variable = from % variable + else + to % active = .FALSE. + end if + + end subroutine SurfaceMonitor_Assign +#endif +end module SurfaceMonitorClass diff --git a/Solver/src/libs/sources/ActuatorLine.f90 b/Solver/src/libs/sources/ActuatorLine.f90 index 3c062aeb1..baf7149a7 100644 --- a/Solver/src/libs/sources/ActuatorLine.f90 +++ b/Solver/src/libs/sources/ActuatorLine.f90 @@ -752,7 +752,6 @@ subroutine UpdateFarm(self,time, mesh) enddo enddo !$omp end do - ! update MPI partitions and look for points that have changed ! ----------------------------------------------------------- if ( (MPI_Process % doMPIAction) ) then diff --git a/doc/basicManualAndTutorial.pdf b/doc/basicManualAndTutorial.pdf index 820d9c36f4ac905b6979e47b92700d31b49da74d..017493a33f58767dd7cfff46277c3c5c3e38d238 100644 GIT binary patch delta 90503 zcmcG12Rv2(`#2#&qCxgGGjiSS8cAj)BO@t_?9A*kjvXSE%qSyLBAdt_g(RdwvNvTU z5&EBd5TZ|?@Avopd%Zf`bI<#{pZD{A_Vc`-?ndmaY@h*nRAuFbkZ>#|&+9kU?}g(LZ4l9H50R)&O!L{=W=(?N^iz<(k- z2n6C7;bKL&*bpubgo{1lqJzMI{}2d$!UcN3PzM35-B%&pUwixyqvZ)tnGr4)go_d3 zV#mJ<5&jS0+Zh?yn^{}IksB9)L<0ZS68;$oB zBs{hH7NPYYAS5{w?m(0P0R?@8NCdj)^ou%Bh)vi4mdV@B;ldh;pf?f;3cVFdqzQ9E zD~Ir6K=^1*xU3N)>QBtHZZ-n`;lHSXumh+qxeA&NnhD~NRrXooCTQ803A3*v>9-vN zIc2d#%;|Rz?x~Zfn0Ha<_idvHIB6*EM<*X*60>`1iC=bF7M1QJmQx5 z2iRwz$_Ms=$&Z)M&wR* z6WJgV)=7kQh5}(h8+5P8mV!Z@fHY_UYe-)skF}LS`v?NH&i?){mIkfrx7nb3Z~_?r zxNyjSz`=m6-HkOK*tq5`g$bk_*CYre7NC!-d$JXnD@Y;8zs*=#qfzKPXy2AM?-vo` zx0WX`zBSM{$Z`CZ0!0bZf6!Jzlnz}W&^8Nlh)34gV@+T~Fs`n34dB%~YqlRGI-$!i zYyrz`4b8UUJ!@JH!k+2c!vCiDG4KtN6u+h1T2%n6_SY|f)=8=jd#LhD8bG%pfFX*7 zzBoeo7NUdIRsR|O4bo%1{$1NS$TV2B3`Dn-9IFdhvB>@k|4{;j5Ugs1ztjmAIf(zD zKhcEGvV_a(0wA<66CL3;wB67S{XsH8`wD#rN#+g6gT9042YMldo8@^x{6j!Ng!k_- z`)4}*MI)kr5@@Sl#4iLx@=tV2Q6@uJ_8PovtO2d>4@?4K2ogLS5CDlg=v@#u82y4& zMf4B$%vn(a-KxQa<<0DM*Tph{4vSNGLtDVTj{=$Pg2d!7~w=cLTMwC})xT(wG#h>@l> z?J0{VVgcfgI;s_cAL8rk9Shj6)?d)IaPHb=$(PsKVe9#Lp|9`sNx32OD6Qs#PR1X` zH65;{(Y`zI_2*_EX!~@imvl2}FTT6JojNG@&FMyUM0ag~)PA92Wu{TP;#a_~dKi_P z&?C#@YtFEneu$4XiucqigGZ)P;}<%RV)B8vxLl_X{Lmg`RuhZQB(T{EvR1-M5atQ` zfq-I-%Qi+ZVO4z-BT8WzYb$#rD|o+6$*B=GYGqO+Vojh^f)TG@`+r*d3`_`# zgQF25Fg#WijsQIwD+db;-3^0RWds3a;J49O92_T#hKb;DTMmXmUs)~;T%(1xt-6iA zff0k!E!_I>5mz9SJYZy7(!!|7Gmv6t0&_!{$KNI?R zRHxk?Jni$&X{`y0Sh`m0co%aQrS?0M_&qN3x4xB|2<}j)W+NT{xHyw9Eydb(!M>>I}m8cE1#MvLfLyc&H7_4bragniD9X z=-hj`>gZ`kAhQV=oI5Y9!b4^Mm?x}(uiXi;?-utd)a1_tbj=z*>{Cj*auj|$8RqI1 zySj7I=WALpXqOtAf?Ju*O!AiUi6vBt?Jd@wWd4<70kp)N+;44WQvLN=N!8ngeH`pK zV_zVy-zfY3RHxu=h=@4Rh%UT9O7=#T!RT2h|88B49J$sXz;k-Rcp&qgTeJo{O+fCA zk4sXa83NZ{-t5`Q-#E61{=3|>g-^sx{(z5b=djl7i`(bbr+0=Y&rxiX%4nBTWiqcg z%arw?Y0TPZdcxt9EJaYEefZ*q4vBew3+7UZ67OR>Ws*O9U^*%n`-Hy#up}kS=agBP zbj*#{g&7r74-a?(_x7<|)SEhco8kiHZK>Tb%Kb;1`TgZXP6SG%1x~Tul40Q&H5C?p z6|AH+lhDC_K1_@nllo@rXPglG8*qH_XJkZod4f zl)Pu3iP5A}Xm!j$uXGUO5S?t11O@9CuSalUhWb;)---_2x)t;cVA~OF+Ty)Mu9r=0)g4F3lzy0x&}b&&TvBp|tekCoj1sw6cBDyLs&% zDN*jBezG1NpLZF;jEW-KSe?XC=^6B;a5wk(T|z`eH%f>Q-mE2wCZ9=DvcL} z17=&MKeRLQaWM#V1NZKFcGPvk&1gTlsXRS+Z6=nYh)BF3>pO+<2|wd!`J|_G9rA9L zSqIMWHuybAO46zl%+{g~-7nZ2pl{ng`{an&1x#{VSa_I!=1{4Fb;yLH`xN_kV#U_Y zG8u_pFCJHfGO$r9BO#HoK`bLt1a-M0m;b$37CvHSWev*tv!Ki(=w~(X_p&-hA|OEz zegw!3<(8^!RJsb z??WxiT_h9+Cfr|n9|}uwUI;fG8$Bsa{;LpjM^*V3WHeb`U{|Q*&`q$tRLBMeYFp#2f5fCLv3>uEcfM6ovSP|@& zV6EEQtDgFLzz8_}E53k#g|Jn?5F&69FbaUgf+2TMkKvGbI1cp}z?Qww^?SaA>tN!E`IQuVA208%X4kA%lI2sSb zqA(x=k-ET^g!U`{u^j(d57(Nn`9HxJfP`TYU|ybE7I383@SR@)m;np;^wwcQT zn)Q(VSBe1GQbDb3CPow_8j!q@gwLRE#3GO&3jac7%W;AAfc=vq0RAh@iNS&FC<>ws z1W*(-Sui4^AfW%OIX7=5K@UK|i~kvoVL(3-4Q39Jcu<3a=>ZH*1dhau`~$`StE~j> zx-9qqmC9CZ4GbEL>4EkT5(9d&Xb~6&14<4MFklNgyDG>wkr-r8{cq&|Dun?k6dVJp za+IhD9DxPlLW4F3@(=QVg)LXJD(h$rvhe;J$bik(*^1`GAV3j|#e~GmWURl+D z9Q*^eivB^~u&_MnP$1E(!9Ng0BxoNK+*M(DFx!ZPifI0ABoOrK)||1GkT(*G7lmzy zf`MDayvI*>gtX(RfzndW5arB*CC8}(cPFVP{fPWgHv2qVdcCurM&~0IXSn2Q!|k3H z8OmKF9t|}MOyHYR2vo}`I7@nMR?Bm)r0ra>YlQ4IzKQIE#~Io?n!a#0ELiLjo`QYt zrN;O7)VSzox-}N{3%pVn26UF5E(RO(xR<=DJX!mlN-^zLa4C&J^v}$WU56TsEss?V z#M|#7k{o4p*~3TQPha}H z--0y^G1<#~&EiJ#c?~9oC0^tb^`Z^VytEyyK*_3U zd|R5d>aN3MVzI9-jI8Z{F>4i=YJ+Fh+ptmbI5s0mAKXv>z6(l{1Fcd!tiQ)OZJ{8)yKIRol- zCtt#1aL}PSW*p6NL+>-Bs*l;Ut{C>*nf>Y;sipQnE`W#zrJ`w@66}^Xb}R_!x%e$U zcbjoT`kCFH7Vj&S@5x;%-z!4apog3+32Y-(dV=Sut$KVTQgL>O`qQNCxy~!GmfQ1p zw!oQBFIr!U(Df-&{`B@4hxf#ZJ6Ysc%QDXVc;ty>HU`Mw@H%gEtrhY7@@~pK%DC&J zmzWMBSm-Gq7sYE`1WOlVldd z7sS&PT1f|HZha4a;|Z_+EJfYYBBj82H64++Pkw~j)l8K~f6Sq+{dBv7&W$981M!KC z=hL$sMl3V?Z*prKzf~?tZOa)DdwJ+sA~9<~&yVXdoX5E;hZ3Dtwne(U7%F410j>y| zQ;r6^ngm~PXb7H6;5o-1=T${?J4_4%2x>w@_b4?=Pos)`t19@igM{VL-?^(>V8 z>1<{`D>)m*NB=H&#PpeZkKXXU7m7!t3WSP%#M&KkPPj&l`Rk8Tr##g*xVI8y+_Eg=W0oC?JO9Fc>!2S zC=J_AW{k6CqO32MjC&cxC5GrTAbXQ)%#J>A?_}X0fU)-?uz9unf3i2@O0xD1d{us7 zo&S@n)QBO!{qbn^!#2Ywmw#M4Gd6vq=BnT8^G|?JJKaC+TpD%qR@r@q)y38RAa%`x1JcxK@t*-IjgzI_IP>6a&xm=zDaA2&_HelAdSrkCA5>xBy# zcZ?SIds4aA#nGF?KrAqBct>=1*N!X0x8yC;A@++UVjTN%|(muanGuw@q5IB@tXd zANKiIygzRe1@Aj)C@}Y|-@T}Ko?Iyp^>T~w^SZ*uRU>mlP7tsIy%Q*6yFM$3gbFeL zFX3mX@)Hc|f?f|4>;=7l!tV{iUBWZK-=mfPD)NQ_gE9y(ID-WP4k$34hXI{IJQ$kB zqCgi%^v^*WC={|<%e~G7hTz_4HT}0n6B>-@V!$#<6zDIDfX)LNi-&`e8qoVf!f^oh zpBz;y_!e##gv$RBXlMWdN8qtwo)<0(7M!5b1cy)*21Zu@n8q>P0&>f z6j+~thFC;cexub1n%Q6u`oA?n*SjSvCMf9XV^*pzgylo+(7QL*U38p1X4j6VFZrRB zQM4;LnbQxp1C9=pxs~VA+<$OSk;*YGUwZv(MuGNwZnHKn%w47`IF`j@bm`&%r^jjb zlKieiM7C*)^||iiHd@ULCgdt1B#pa0d&%QXVs#(8oI{4mReMoyZ=lrSuDh5Y&t!l#&g=WcDRMk&I((Qi^yv*=a*cm5f(|S8JU*x@3 ztH{&B>ZW^aKkkO%j}7_CzMY^lj=fUCC7nEIaXp(m;7}^Bx_-nSVXFX-ijZ^)$`;tW zP_wQ=$ChuW?_aK!t2AcNZ1A1QiYfEBe7UxhA9y$Fik#$h+2J79)~7jqB3Zwy{Rxwm zUyGi2%R=E%MI+u*_wFFIPhSylG&%Y;CN=!baO+%pd!E*N)sCngbr!6GvI-OjZr#Xr1e64q;It1sqKgER+Z?sD#eETWo$p& zxo@P_@nEyX&39F(p-~di!^EOTVB)qhI}JRPN5kOj54e)KY$I8%r$ff&+}m_1*4M+%d014tDN} z<)%sd&_h)3Ei1<9(ev6DlX=ldmu10RgwpTrD6t@&Tv`4LN`nmjo0+=LVR=OJLRp8U zM9Eosfig^lgu&DAd<$I-!;LjBy9I9St|KBws7OQ(k~m$`b>ggey=3N1acK`uLvoZs zIWQqDnBS-NPGD-Ir+8f7LVRkEpS`3@McStrTQY4%3k11kmjFuV^f%sWxSrzZ&ludk zcIxq<$KNUgvsCYA=D-d1>o4rS@2g0TFP0Qr90b%+{!UNgBfND>NuLg#>+@oJ+F)-< zO7qe*ukjY~kdZZ`e7!t}cGoA=xy8mWA<94TM_lY311UWAiKS7R&~UklG%>0%Qm1?s z@kXR-wv7eIC@X5yr^VhTLI`~>WY*3~J{pNVpJFCGdVP_U6Msxgb~al$Np96i0oFyYeKR`9lYWfN>9}lS0>mm~(WB?~xgspAtRp_o4Ib zGaBvlXMTL&F{>}oc~$PqjjBflf~j)K=3krk)H-KbeVHi99I=jaNNjt+#L9Hb$ux`m z06jLYfZ6Fd6&hwonqMWob5Xd{rp>xo#=v{lGwoEIet{gBH0RMsJcC($Shk>sevj22 z1C|-!exc!TWyq^9a$i{HOA8U|_llqQFJ9`4yD`V%&*p4xDLhe!avUf-TbXc!sO*tu zrN2@BWqmQn@4e)wJA&*O9CA(TwMQlN8eUeCy;b80cuS>SI|@6dGa@&<$eOH%dL(V%pSAhnI8JvQ0Ngn_ZEWq(p9+!Q6?wKd!?oKq zW>^8HC(@6L63#BV(LcKK-&#CopcQzw97m;U@v&np7V|MfhE;4Z{;%aBcI4K^6R%3dRX~pF4j3;##+2;PCg`J zn7xNL?;|5CFqTtohsr+rh&hWagNKUwquVDXrWw)(?Dsfshxh3bh5ku2RCb6xXw$T) zqj@JZO3e8C1f}W_2IM5@8Q(dgh~m?#pRI59 zFBz0fAA&WKnyP1a6kE5_V}MeRZ~d?>cf<5i;%y8Nagy=%kv4G(?G<-3puPY@aFnQuN2)JW`kq% z)$QVNHc3~T%8Pw2snWUjn>n?85tm>oaM6~OobUDsnqXzU&V1P>`pVNEj5O!6$6<6G zjLO2))oR0?XN4|`NLS|hUDIMx|MrD*y+4taw@9xK6 z=;iWeaQ-runB;R$u()2Gm`>as0)NSvNj;Xz(ir)&Rmup?P%~(i{aN7tOqC(k z!E?wrt)f<<7m-L2=9=0t!v_xO%!kOPb^9mefhZO`#ybq#=;kb58%mz~^zC@1 zweS#(ys7%c_=H%T=J2aqr{pt_+g0s*A$x)wEH)_POU$rla78b2__} zA2f`oX8*99X7ds$binP>dIodkeq37YaQ%?NmCT!^pPT2X?z^=2XWvFV$&eTpm97e! z5hjxMy2V=t>=jUk<7-9vkvlTs$IB*qJ@=CNXD)h8k@XAKHSz&7&Z_hAT!}-I@q5$l zWw|{g(z&GXWj^zwjqsq{!SSNA*;ou557y_f7KqAD*hjT4k7xC?^5 z`W9S$-`#|m*>>P-)vl^H*H{~aXKUWHGpK)=MC`k}#}DWZDZ`NJs(5(NI%2pUgqV)N zaKS#`Jx3i$o-6h|KbP-saNt&VNDIeJ=j-faCJ~05VOXIJRR<1e3UHN=+>bl0SufQJ&HaZ0 z1kW>#k2%)>GL0?QID2BWhQ$Oe9mMC4D&3m}1arlkl$UJ-g&w z#gm-jp2YX&jY1bnpYQ#U|H|_rxyE+A5AuWYci43w>)a$Wx2DT4VGXZ@VRRT64$3T6 z=j?jY<(>0tSLG3Bnxr%Q^cUEIQYmSQX!Q?gHcrTSiyrd^aC&nm-YtFJE^m3pOa8gg zL_?+(-w*m!;qtiII?bj%?qLnP3a{!izBxX+;M!XsG0gkeFXY|TT{=Hp>OSv4=fBWU zU>S8X7C5y{7ifJSrLKu{+;w-}SrX53Q-$=}i?JWQrLrE678za&e7>+%hyvK0-ro?T zAcQEOFv5lq1(;0&oucK+<^QTe4mycR2uT3z{Qh;vGgsp!|E=Fo@b?L&1%zKuHda%}~^C3rdxp)rBmHf%Mj<~d+cn?@>@%!_C z@uxQ=oc}Nu4d!R@n_Xfop{8&(G_gFC&?sPIP6q`h+>l`GXHzlwucM@d(ttlqjKYB> zE@1Fv^WaV4hh+=_>(fanEI2h-xVU-r`dsm!rp6%PNH9ILc`zYEv~ksIiw0J(jRJFK zD6j}~^XN^Xk9AwSI+zgE!frM7de+$zZt$Qo3aoV@q=Pq4y*@gyUdGu^SYrv3q<9yYf26Elz^e(VNrPqL7yMXNiIE z2dnFTernE)A&rXz(=6v|m7@%wxY&LQ)cc^rcj{hN(oOI2w@YKhX<^ina-K)fgSQdo z*`e9zSVYe4NR=Ro(+zLsVg4owbna+4S33JO?9@Sj4@zyI;Jv|H%2dyi5a}!R^JLjaRvHvkz7)Ry)eK^KiZG?MxZl z5bmmm`;E6^?%(7b_U0bNG(WyNmhv-wHmkE+mH$AHW-Ud(68Gf9(JU|CPC9@i+PV0+ zVq@g)a`NVyF)F$?QE6|f?8P1VO{qEijte*cXuopwZTJcM>(_)n<%E8d6Gm= zvsiliNfK9Q`45rvp(JO$XH80#zjZp{;S-`EM{kqr9(I-W%BjFB5CswKV|oS8k|=m_ zL`qUgtorU`^Yq|Nh6wuiue|ma+y?-~dDw6-2=6&bmYX$Hql;X00k_*9@9sJw)0fj< zDgXT9Om;ZhrP4_Bd@o=3k?^y&)kpJe7j8AP>wVmz$CZ{TV$ReyKx^In#(N_5Ve35I z99E5w;%C*9w(#>aHVXn=H+m+Uj%QCv$D1i;V|j0RJQBJc5YW$1uU6#hWN!lK?XAyc zke9Y|K0K|F9`fA2@NKlWhskZjA-ga2qp3lklOA20Hnuw58l0uoz*`+xa{W-{Z0a$2 ze|d()i?(ojkF-SkLzD`bKcAdE|a8aoBU+X_j&7%Fc^u>MKt#SCzUB#U-$m)0%}? zDCSKqL{D*G`?o)N%1oc{Q7u%JH!HpEV8RC=+a==;fBo!_z45~Q5mVk27i+HrGKI&K zPd@R!Z~q$W!Mp8=U&G$w?*?L~GWYzHA068>+fa9HWVX)nhn^T~NS$=mv!uD_W+mG_ z4d;SMFb8W+NNZCZVt>QKVW;=Ky>{1*yuk12j(l9ls}H||pQ@46IA4=EJo}Q_wtljq zD>brjBr8b0s%rN3blV4&OMrIBBhn&Zf9%*tn#s}2+Jkb(?umq&e!S;B^{FF$XWeH_ zz`3x9V_SYr2JQ;cR~%h|PO#OYCFP`<=d>_y_DfsQIkG5Ht&}Znj&-NnSJnLQCI@aM zD*~(?mICD&L74ofHpeYkbJ#$0&1U)LU;Y}zo>QO%E|fV<6#78zvW=C zz-gm)T`tzZ{Vwje8Y}`9dxFV?P0}7LHTyf*pkgOfb!-|8{5rtDY8GrYH9@g&WftIm zH=MQ_3@UaMm@C>eHCTwYu2}JRl!0j`xF~oGQUol}B&3=)5C2~z2w>LG3AGJ5feke( z|8ZH$x>NJNbRH59t`+*(qAI0Y`@D5KjuvQqK%2pEIYo5ofkvEEyAW$z{P^2>eW`Zx z)>AATS$)TBv&LG!TvzxI-kW@>nR{2Becz<>!sKUXiAhto$)Q;;w!6Xy6iS~pqi)4$ z3o2nmKX7mvL@8J9EEdFHl(<<q2CR|o~)i5W=D3^#i=MCL4D?rOK2M5w3V^q_8x z%C$zFAy@f>a#+Dv*FFdc*4HEWsF%ccU(&9O&xSDh1c1mgE4RSeRERMRU@er zw!<-r^8mO{4W?u2Bmha@HZ@Di6Zl6HJ*jVmcv{N9_7iwvf zGpLTT6C11Dr>&yTbLKfl6N%l$9~cu)--9`J*ke=}8gV}4+BIwQI z-M>3kavoD}ZzfWwY~}bg=H$;_JZygTeT|xpd+-eVql$C&VQOp_78jUzc)mViCio-U zRpy(W*#kxq{xqtDCD?6mGYSr+1rF=Qc*Up+j^~n()qT`|?z@XvY(gg9nDR-Pu@YG- z^R6wk&?^bg1Kfs;B$dh{_@{7X!{kP~fb+S%9cINN+w$|f6nDR&xfN(;9r{{bOFa5J zrra09fzMajlQ%g>QSAein5FMf}$_yv`pz|4~lem zIN@b9u1dg;*i}02)USPUKjgI%TdW21dtyqr#{8hkBj*4ftI*ITQt{h08KKh5rI{1> zc!eV&lj_mEd`!qk)Jas2ZCw>kc2v7a~!PFwypX_s>KSwZqKCG;LQ{CwvVcPB<|h)k=UGlx7~Ob zwF|X3?q!86W%_L!YCrS6unJ#=Yj!!R_rt&PJr6*@0pf2dBXUluDH zyC;lJlXKt#v)$~CNI{Z=pXm_qZdl=a-f8e2P;?opj%y5NHhx z+qLtd52=5>0@C35&X->zk9jd3R_MLIjY8#+vy{!F$?J+<^n%kg%0WN)sn{AIRW`)W@1eBBZq`c{@ZEMXVo zQgwwMYN(6cHUp&FO{|>nI92%@kO)%5;ZDds9H%y!OuFqusqh|8C3BB%&M7j}E9ULH zvO)=}n3-d$j9+vRX?jCKV(yoZ8N8kQ+InYSO#Kb-5$lgn+9c;#?zg#d>iYGG$c#6- zji=TQTs}`OP|ws3r#QG@*{LJBOj_-S{Q)gUD;ooc_8&e10282lw7X|SbKxLa=gkZ| zODs!oWAK4;L0;8n7S5jwcl*zeVSDd*8e?_RVRIfI2jfJwW@*}r6CF+7zY)?a`l6Zk z4BavNNoVXD-Em%OcbQ$T#M;jgXR%*s3o|Z#=xcEu7@d-Se!h~2+2w8vNiVNsTAKv* zWRkNp?((VY{AU18l2f4>PlX0FPtpJEmsKlgn{O|am@UudnD=@#&X!Nea)&}7c=_WK2N-w$}^qk)W&t}EzJ_Zz_SJpYJ3R1J}5VFXXEj;w_GDib z4R}Dx$n|sES{P3*0i%o(5!#_)M(1|Vd~1Bi?OfmS0~4fQ!QWGcs3&<5c5mOA5!mGI zEDlcEZ(rmhRrgy6a~SAY0CO#ocP><4D)C}9c*l5GV~3rxXF$?R{SG>w=ni4qpCq4j z-u7={jsEJfvw9{1?5^>bw~sYbPKo%$2Ua5#%;e(3?5Yb2e4NbjTXg$=T3jKUI9XLP4GzN zpBMdWv?v@jO*hZ{dt_+~u+exC@IdLWHUC;N5V~3-TTP84 zlw@u6a|k$Kn+(uTT%)elelRGo z&WiA}4Yq|F(tkz~EKJ%>C`@|B@JOJdXYpt&8!Bq-b!^X;wj*m6_A0rjCvAAB^+gAAmn{zsL7yEN6Pz6CkK2u13nHEEY)sDg zduLS-w12Bb7z7?2dF2ts`=-%4z>a5{jaF!n6#4}puVvTAylM56f%t7lb8Y=olBsD~ z#Ygy$(%JH)uZr*$&W>ff0hqE)y*f|K*rr4IP{-L%iw?GdFN9XE{Q$aG#g$I@3*{pz~+k)OI)&NG;m-wrrX z`0`c*%j2s=d53~9s#Yb(?mbG<%iNDX^yZ?DwcTl1!B^Db#Frj~_e?ynIl|IT_T5AG zb&RR3|88SVMqTb}Ux_-enCV?)ii~E>3|Flg5qq{c*U);0jud14MB$)2IZnJp8i@Ut zQ%X!(^`zaXjSC%@q?(zxV`3AUGRk{JW$W>@xa85!OHQRDvxcHdQa7!;)h`Oa zHXY+nIXku}lv#AK}R$_Mf%ZoL-5us||zF&Cjrrc!ossVlSt>;_}2R)6(HfP|z zX8spfZI;p!gz+^)FU9FO>ep^_?y6@lhuq~&4F2&R%V_!W>PVMlVQPnIzOSvfVn1@zEX9_;9`}EMKBKEWa;$zg>x-b$~l? zWST_l(yUPGP_qZ4103BF=DI!iI}PNRTk_-?PcT9V7f!kpu?EaxLe zZsb2u%(qu@%4K?YM-Ul_mNBxi$r}4U%ywj7(|GftMU3gCJ11MZ-wfY%Ki}$CP;}m5 ztUch>nZx~|WIMMfU%bvKBY44A0{7&pk*r335^(MQ_a5unYb0ZzlN6{N-(iMdSk<1c zwW>`q5&7oq*c)WF8egGqIDj{5&_$@&3Yr zR$7U9<&j4YT;92=-cIGJjI@fC5fXL=wJ98RQC2QHNYyP14V8b4G#eitzc~FfLTo%- z;A~nAz|OBO`lua=Bo|6AdWkdSY(2ms;TKxP?bcn(metDL`Iu>7q`)tFKQCv$KQdNt zeC}rKK%`0b3{@{)=vI;XB^!#yP3FN|^YrWE&#nx04B>KE#xDB> z(t4#N-m{9d_lZVXtKLWZ2Zx7FJ@(LYo810U3OH)%))_X_^w^g@b=XhzBX6m%@-^{2 z=MRm%iv9GiW7kvS!CN2wI+p^;$mdn*T=K}{GfCBoRa*GPp0vGJ%-I$p6!YGeiS}95 zAvZ>vGoh}RO0^Gs+QsHi7gGGP6`txj*Ga;h+`AAgvCnJgz`QEsDZ29a-Men;u{b~9 zS;>Pe1Qrgz!U#6kb_4jY>WK}X-)r61(sSu{{B*9j-N{o+mM@&HQf(XB z^$AO@>ziIAarRSK<<#q${q)iXH+hGQDozefoO<{ziZ;FQoBgEGA@`@QWF5DvBXW=S zqJo5GnV#*va_Q$q`ckFv_@WtNt9?t&H~2)$?uD8+IV1&QlPiwVBY zaboP1$p;kZEL+Gv@^j{<^G=x#EmDjCy(jfqw(thQLiH8P2>4SUH!xgFFwZu4gHWT) zf9(yTmK&U{?}LK>-QLFoD@PyyO)IwzhaCylE3}Li?`)HU^_ze7tJmdfh<|x7=mxJp z3kt^gQJ^`#$@T?{E7nZOKTVAVT~IKkvw86E?!r2_fnUc^388+lkHx0ZqQ4hBtQ-An zFxbjR6s&F7JotBa;rGD+q5B=!1ci`g*gSfjzW})lzmFyi23z7Fz@6DV_;+7=i@`V% z06Yo}=5Br;y*_#Wr>V3 zKJu{z!qFJ8H6)ng`yFm1D9?U1Ii~#oRwMF{9<9>q$(w)W(h^z~t>J3*yf+Grtw29p zc(m20L5~(kTQaJpm&`B)BekZzTg#yonm5dicEFSrh`!!!WOqD}M(#r|a8K9c_Ctle zhc5JQy8$aGTKHBe`Qn9;eqST;**gIH&U3|DUN&+`D6gPo^L$z6i!t19k3WjxOx*GG zI%(@WaUazdhuzHtW2?LO3XqjlJ_=(JMNRrmSAqb3E3wRcXx zAL4&JkgcrolIv%}HMTF0dgiRu!uZE}=z^~vk2du=e%ZNG-r$+?Sq?yC63j}B4EE+6 z9i0PCpjn*FHCPA8b2ylLuXbN2|9FHX{f$}kPOmrlhL-(VgSCggl`!utp}||2_z1HK z>=vLt&}-{4DnR@o(8q>G_@bJLq1g9X7?0rXwr=yZhv6}&Yt0gFbxm{aO z&6tuUPM~9`B^B}HLw+@YY#2M$eutcicS-E}p;&6`8`s@^-o-|cy;HJg)zJ?N{642Q z=y%-s)vhAVHy6q0S(VCYc8PllOxoD^nwC5GUH1rG5M#TToY0s_7I#oWR{74*o|nT@ zOBRtx3Wa$F)mEmuXYPuzImFrIvh~|vADp_H%6-HBsq>y>&J&DNEgV3C-z6#gp~wU0 zVSJT?h;wtu8H4fX&+@nJ-g9-^;*wprRZLuS)OmW&pq4j031F?wJqxZId44iELl5F+ zE-P`q3#S~0H4rU^;zoiH4fjRJMn}+P*ZTrAMu4bc)aD}BjX(M)F$>Jxf!V;tQTN-6kBq!p8bQX&fu)l zxdxo&$+v1r=PT&L;&w9UooJCGW<1LKf`?>5kp4rmHd9RX(?b=H)0S{RUF^%KP$L_O zR;$?uQu~IS*}GUzR#04|Up7hY>TlMa57W$b+E;vtlDyG}7P5C?D&k#)r z8jKZ8B<#sBD`OIt5utyjRm$V7o_DLmO)>m|obwy=23N}or_1LJjp*lJ02a+9U$Pa? z3f^~c%*@4oxI99g^R6bpJpGVuBkr58rN?8&YfFtL+MyxdOBUg+`6}kZE z-v(Sf#OBauIEaAVX}D8ZSSn*85vveCD?TKl#&cM^-_c`N50icK8`}kPsk998vl_-r z)(Uvx*Ij$Vehzp4^zNetsYzeZ#Q@#+TI+OINrH5(i5=TF^Z}ML)}%gU3%PbFCeJQh z%IL`&=_yN)xLR^KH|6<mRKka zmE6*onK{a~xINdSjxU146f8Q0+61pk{^f3eza=!5rR4?_7EBL-0`&i2!lIyNm*BJj z)B+jQl&Ix?#mjc&pXy{+W9?wmtu^`m`{2Kru!PdiwZVi=#v7(y9gP8x8iO74ephxV zLRZf9C8S#q25W9XvwhRx4Gj#TnE}W>*l0b2eJ}A_BneO(`V0OEZm>Be9<(PnPyL$_ zx&rrahC675ih_xZ&7*&_LbqBq4(uF6$gXS}3=zv8BHn}sr~i5XX9Ds4+lco+TLcGI zj7G4j(7JtHH3@KFtG+*(1WEhrZQJo(h{A)2uko_2#9(Ex$@9YzBG#X8+xR?pJ#T9ARFa5w z9DVqfa@QqZx3b4ETpRi`wJM~gUw%^?50vrK;dLQC1QD0&H(W(BvI?YQ@EKeF% z?-{qo@hj1PaO8d3K5!{CjLdet6|+!qL2WL<;IPMwbL6lSbS|~u4fCfFgF2?OGH32< z#)RhFKe#kgaOGm^xsP|m`D6U$N+yDDAbn6ihQ#@(@RZys35KXXvtS}8EwfmS54{#} zX%SsI;K-@h@0oY*5be%G!z@PqZaA_FjZfjDvbwB_xdORm9_|oI&k|6I&Z-mR5%V<4 z<-Bya{kFG{2e(cB@m@`*?c-smvIhmsqxV`Qir#5(OKLa?WYV;G+tHgb?~aps+L2sM z=@5RTOR~lxyz9-+>Nne?T4lAYk7FwA^z(1}iNXMq**PpsMk;ToWnh4*?_h*k7=1x5 zhoScUQ#um;a#CkQjmxTzzfz!f4F9kxMF3fMKc~sluD?d} z_zPXlGYY5X?yByvYOktRFTB9b>t7R(_hS*f1b5{XuWes+VJ3Ik^CQ3h4!a1;*J+l| zq+WAsfLI&VX}+*f3v0F#msgZ;B}LgnPYy{i*9k3fCcE|bz3Lgbha{&FcdIN9CALL4 zCvzG7OnsxRk{=_qSA|z!URvq+e8aFzwf+8YuKe*I;U$Z?IxL5t24bi@>#bE@VOwr! zU1wp{c8)LT8!|asJTk<3-tKGl_*8qu`CV-DQgs09zT@-(sXA;ILr>l$VHkY6;2Iwh zD^3|D&v3eA;N$s6Q%IG3C+)jg4t=coMBjy{DpPejqlix1@EJx#cICVoJ)t+pwCCBC zFq052an&A*;-kkz(qQ1~*E9FS{9$Ng^_a!KlS}9~Pi-8``II5Fz&cy_m2pzU^b*f4>i_QFG+KypQcbiZ^M+CSbwHpQBi?r2=L z`H6fIqECxo4|6M1RT!F!zr`fC>w1s6*qJ>adz)mYef!HPx92pevjCv`5#Hw_ulK}O zcOcDjB1kXY_+#~FajaPDB@PMKXz}^o5#Cv zIB*@cloA2P6=xHxKfF?Ivo%Zh_AP&$+``gy$XL$8g8fA|x5M*;R;F*0lwzLGc%~d{ zlWpa&ygLRn%MbLs9Vpc)U%|pOfIHvh+V?F5KnBr^k;@y9%NLV~544R|OW!#WYOB<1 z=F4|o?>S8f)1c{bAlGZCmMR`b!b>N=Y%E6kViMEwM3QQ66hnzXSzE$5i@dk(4%+4 z!n26=tRCQ8vJG3staLZq*e>o};Y+(KU!(kG_g-OrvJ|DgO$L`f>Zr&e^g|r%YBLpt z(Te2l)u4OeVjI7`5{VcIPlfpw6+^le-E6CuEc+pg7@hNZfL8jjS@WB7UZoda$8B(;(bgkC??z3+&l1YZB2@XjT z`trccRt00`%g1!upC2ZRW$D*96QmD9HgSvg!KSs9j=NaPryg$0$>q@g;%W7xkm?QG zzDUtfu3d_)#EVF;)GNra9j|o?{lZ3tv^V0hZY51grP3?7k72w&i#?b9=r=hgl@|)fBKJE*Apz4h|nbq{#9WMaF_NNy`107~<(WZCnh)j#qSOFcE0-(+3XpY z#LGh+e`171ZkqYN!PS_%dhTsEawf+3PEVjdf6_ihZbXgM^(jI6S+Y036UcyBuSET*E7xddIj^gV#{TMQAvVxN6;8;1*Gvd0ruLmrE@*VVFjsPr3GoTj#lrCwx?6 z4v9pFU9+gZ8me$QrAIeQzq6vF{7eleiTRCFVieqv>)S$+j5aQ!jbUWXtKud*`9h8F zygbia9b$cJ@TWM!^$>bBS>7LKbb}ME$5mS{h3v4p8eBYLduS%B{*Azi^I_ zII0Z0%x%xi!4EidbjytA6+1T$hthxi%pXTPc};74Y-KJp>NvNof58XosN-&-d5c&D zMA-5SW>9>bma<3EgB&%L^Lbs|xsB!p0)@l%F%x%jKlAJ#CL)SIs~F71+nw!9^m5Zy zDqd!aaxPKZfw0Sjkqfg2BsM)D=dyq^xHSZYnIVys@}y>C=GKsT z*x-r8S_JW)g%hQM$(`a>6RuY{PtGtqliyW%E^*6o;`KSBnS|zW{J?#Y>3g)z^iG^v zS3Gg?=*+uI*3=)`4C`s|<%66)u=!t%MUYjR+O$i}FGlLzxu9aRfK^ruRixlQ{7z8& zrS{vbc02T1S(Q3PQ4y7CiM4nwHg?fG`#J}!=QK`SjKYK2()0=|T+vwVP-=BT_q?z9 zw@h+V&Iw9ZFuqM`XFoM@eb0L4ujy_~%0OT6I!BtTKv)L-{84{1nJ^-UY0^QOpF8GY+=Kv;MVxa|dc{?*%G@%UO#d~JvV|DCJn zRqYP4+`D~goadq0N19w5BKpMgXK5bUr%Lml-L!60sw#OUs9eGrdr2_h)}qbFu4i>Z z?uOky2_fN-J&rxgY)`G!lNr38z^lpR*;7s89X5qU%fmW&agnAuTA8*3CgkkydFqb? z1-esmz9`doT=r=`Pl1SQ{npVR`PSHyXB_JhD{l+O%JKMx4}+f<-Wo+TWN5{$>j|^> z;FeL<@o*?h$BbT_86M(3b!m3qSk8E$^3~-A=b8gyOwb*6(MCMK`8tNL%Q}X4M^TCd zjm;mkiG`wYPelb6A#U_g=QLa=Hy&!pxW6|J8Tuu_|G5N~+o6-iE>x3Jh$SVDhyJoA z6q4!h�qtC`VYr_FmPfHGh{Foxz|=#rtw!MbYGA8zSHHDIc*~0vwnVQC^IG*B(T9 z^Q|U1SFyQ5CqIs@TgyogMpJbiFVp59)0vYnIW_W~=DRXBi@>$Lr`&uw=Y(Ptc>Hkj z>ibgpjB)VyNvz@(=7;9HE{meD-d;=0c9r*cpD~TFwLO(V?!NZs)rD{SBktW_WYv1N zg4D@tv%D}@FYa@NH2F^cbOP@Ko%`7zByn-gr%K$0gf$?_ZiP50d6F3VRUQi)&5zmq zI=+L&49(cmren3_)lB&*YsSyR_b7)LVm&RfH@W$Uo@ek`B5F0sudgYdGnFMd_-#g|s|H7K-cpScsJ<#FMpkj!|TI1inm7xk8`L z!*B@qMJh)T)bS-s)3oXHwHM+lbQz188b%yqZS<5{z4tHVB!2ohb9?Br)XeSZG1-g` z7md(IPwY6LNeW*)R>8PWp^fKOWmttcf+TYKZ_n2$)3jV8YdCqHX7WP#ErB;Z;Y1F_ zlt;}@H{WtXDj)nRCHS%9)N6u$`r?)2CrZz1E%h1Bzn-#%Tnde>JSdOoP(*cFK0Yw! z=5KaH+r5!gws;sxd}xKdcB+T|Rd!*~w|3l*%!lNQ4Gz5xLkeAcp_FZ%l)Fcf^17CC zl*+X`uS(Tx3(`w!ZQ{mH9@%Goh1q0Zh~QL%n4b57uutNu483V4rDJEaz@-Z-;d{Dj zQjeS|(>Su0U}6Hvif_~=xEaZ9B)D13v;EDof0(4 zx#hvK_h6s$wcZsso43KKGKB8T`_8C(GG9x^i>p1E+aIO)T0cVYbI*zUDu%kW{bQ&^ zoH);1{Vru|rniTz1a(qZy*&karC&pR{ucM*n<=w~_T!I(=E`Ge9^Ob~Q9&155|6X1By%ASA=cqVHaiCxxR=l{H!xT`hHJ(n{UB5gjtmyM8 zlT@^2X$%u}3gpf4m<@+q=UdzMh~CsxQfxACoxNSMtXgXv{ra;0T_(+1 z>XbP~D%K?VM%#~&f#=!J!JZUv_2adUpZlWv{AqBEBsUGKnLFZH!N@TYSGHP8Dd8KT zMBQo*oM-#s;MFExE7_Nx#YkQM)%-`MZz&9JdcleZ{5Vn>xlEg)@n4xsv&E?>fac*c@jC8OMSg;RTpJsj9 z;6CxxIHt^Jcu^vJ$~6B?4Ex$7<&E$6hg$hmqQ`a2p>y}u4r)9g_7;D;tbB&S;#sNZ z-jZutRs_2JT2?xA7kJF^)Yy+GFrIFGeOETUTCnW>=y=g7PZ0&53GptD1C3OL-ph6t zcxrDBsy-fmfNU0H%5y%>(0|nNwS7$yUeuYbtQpNv#$%I8zq-`9hx^g>thN?) zoRiCoOPLz%uY_yM#=CuA7YZCXS{Hxp;reQ+d*owUO8%Bxj5@l@0D<6sz{Qu?)yxYWu}Q>geGPepZDQhwN10 zR=tD5cNKnpyM(IlOWSd z?m?Se#HKGN?=?i0dC2&1_9H|DH^Bc=j_sD zG4u0&65gdg%f8%{mNX6FuKQ_^HNJZ_WA$tk*CQTV93xuab@BesqIHaCrJdR#NVKu} z|FLL&2M@R9#tHcD*ug8IBe!`aOk4g>c{mZEJr8cEiU8J40vx1@0OOdjYbp`omjj#& z{InCl%kk(xdjLS9Jxgsj59Vsnumk+Ve^F>b-~atR^Dm<9zxXEn^}6h?nz#5VwAY>A z58X-N6XV@{;Lc`m+mjFjJ{-jVBp2W1aO~GkZ&wZcu?YXco&*x;^8dYRbY@^rLR1o* zVu0HizcPa8t0TWdY}YXaxK)65IrM8@%mCfW{7$W3zUE0`UFGXONl7Ex>w;r_siU9P$ zs>D10rN~KTA|DR1#Lnl4Rynleyv)XOB-lUf#43 zW&9ojeH}gBRNxiTv-07)BpJ>V#zgb6cHhS$0hAAFS&Tl~PI$|uY%GR-`m*4n9o++| zA0}tB;&v9lWWOlMOzN|@+I~@UrmgKJ%5k|e-Fh!IwHhn_4cn$Qq44iEA0AbQetAnu z(uH$ARP+N^a`y^Z{3!|ZN5PxU?{r4S&mWVQ^sa&=j2KuseQ1^3W~^#fUa#4uG*f6X zYvIWC%s(Giv!`2?EkQXY^Bp{g-FD1TNh6q%vY5XkU;b^}s;4Lg=Mzf1{=U=Et}nT% zuTmBZbU0d?32)+Ry<9_mB%VALnK9p!)mu{Z^)sQ>H5App#_Q=O@s53^XU{4p&2vd6 zlfCvXfSRHrlr@>3?oTMpev_@Sl+%cukGPd~pf9 z4*FgRKI-zdoo?i(p$M6*6%Wj~LzOwDtUb3a|Z>mB#EM`P|gu7*5bxe}fhVZv%3sRl_?g9}?6;~;#Jx{D^ zy|uEkdHwDwMWy>Ar7BfX28TR+txnQ}Lzz7S!AGI#7CJIJ>@zz26}_Ku>e=@To7Nh5 zphy&n-IB0*Qtx*>DNH^nP~dMEktHcOtmxI49e`x@d=s@O-z({Ivh}!%Ux)_K8)Z?i#%Z7Sbc0w5LxR<~5hn zk)OWn12y!W6n-tyeEe811K&ca>}uZ$=<>$fwnC@AbK3LIfhBm6Ope4u5ap#>a|`PU=f*%xB*jsz#9T#}{G zqa5pVE6aOW9~*L9jBq$0Lor0VgnTz%Uw^qZsPb4A)Ek_B{`}1==hz&pzE=|5%ves?hUnY$l~9sk4EN9a;W91s*N@K~G&T+CQ9ISmlV^a~&t3 z@UANL@`%Naxqjh^r^)XJtR+2#j!a4?9HWw(Sh9+j)vgUDRY-Y9morzaeI(ko0C`&@ zFMr>CYDxmgQ#>3l0)F zmWK?CJZ2lF@i{Z&azCp&u66XsoH>Y5V`MYouqh|g+HA?!!5aau4$yAidQW!s`t8wy zCI7?wY*8KF&>Q7}2t};K(6yUB!odxxS8Md(A_wATy z-_kVQH%&9?^|C`>r%RapXbfDJKKq0}3MFPCWEmxStbUf#@qDNNyMp}rW|t=p3u09^ z5%-puKU=vE-+bLt*1?J0*4-HF(RaX~=PrK(lc0aUu9y~duwRoz?a-6E$)a6GYD`bG zld^5)J!BdstmT{U3^v{t2n?{*np|ZFT#44$|2gc$*z;~0_4^0(!!9zERny6~-uPrC zWyQS@)mm0-L$BXplOl8ItAlaAxM^j}MZuEtE72Qs*A{yWUh|)`+=GX6_6YYIM#0J( zq_TpC9$WT#;A%sfRulrJT0*D#5wvti2X4F(5;`r`D^etUT67A}O?YpZo%IRXfUJ`q zY)`8F$)Y|v9B*v##Cwjub9z8@cyZl&?~DAiHy@xVQ13|7$G)f+aFcMbG)!Q20C^b8{{PC(9Ca%oAs=HBj+^!FSVT?}YynANxLx=;Q5-rZMB>w(o* z-wv8bT^xR6Y9BV#3EgWQ{^D~>S|Y)GZ*2`Lj+b%az2xUOsf1Mhtt^*3KG2!{@T!*OSi zbBEtuL~YcHTzs)K>n!!)!D2LV^*fGC{TQaP*Vs1uNHwTkPKVlQ=-HxpjE@z%-7YXc zRuFx8f~wY?BZB>bs=Nhmd5}tENbCa9h}(Q_e~Gdnf8$2cOWw=VQl->)FNU`T7*_EJ z<%(T-Fz_yXvP$so&3?#%s=9*Zb=dwD)#3irempL(?byc$!egD(-45IjP0Y@n2+awS zl=0fn5FpMZlg0TifBYQ1-BTz$y1-HOi;&x0Va+GXR@&ibMpa$D)le|@-+8L5Zsqd| zEB&?*vawB)=F-W&X&s?a9IHk{DD?6kHD!VpdqZmGsmD&^Tv(P5Aqq$1L*nIT@ewpk zs`8A-d)J7hT5DOgMw`p>gm}~1*F;9!vWz*aWHQN;Xy%j1`g;a7CLA*>Em%ICemmjm zSM$ZsB?H-(id}V;_O{>BCyE0WC_8ZMDE=j9lgFCBh2}ETjCO5R)rNmqKm&?e=GzJI zmyURnbJcli@?I|?G#a=i8v#cXFi)eh$_x%6xTuL!&7U|L}oftE8L2{wBL{H z2`k$F{*t^|-^yHwi*jV;s~#oJThu*7+{qGk-BTee&Q{p|_|(vg?@4u+nDm0qx;C=u zeu=sgFQWbw*B`GhZM^6V`ODBxefNp-ZM4EE%#}~od{<6eOT(WFkx0hsT-6L3cs7z{ zeM?HQuU$V-D&SJw2JJ#HG3&5(!yFrsv{8ROlXo4joPAe zHkrJVH;u5EOdlR&Mpb>6*SxW~&e;I#XXJwY1im_-))d?J1&Bh%eJUegvH7NoED6c< za_(6{m08ae7^N-~+|=ip8}Vt%=`7w<4KrX1ty7$?AAEP;-TQ6o2daU(CBM?}!GZPH zYF=!_v5C-tOmzLwi@4#z;)j9UF7ay>)9@}>*hB}_;#rcSP<>&U<$iy_z6O5dsAyoKugjv|V_W;Y@#C!XF zrZ%+J^!#Q}*3PT@i#CH6wS-I42?yo(THaxfWlK3E;oEB&_CV23t~w^K$C4z_-rf7l z=RIjRR)|ilbs9?|tYQWezcZIV9-oXN&HZ-d*;g)~kC3Wai+bVHD@pe{+XTcfjYB%ujQLEU{r`(YJK%K=Fm~HOf_Y*z)c&I&;8(i?yE1RH z<7h9^f3S22c+9`oyv>aR)jYsc|He-v(Oy{o)@)2>2yo-Tnd2Y0apW%7FTZwrt7=hT znc)wb`@ddf-__-s6pX<%XR$|M4*SS2rSm&J0^xKr0kN?mMkl zAi;?FS8yYR#RSE`)r8;Ljh+<$K?Jc=0s;Bk`a7~?o_o_pI@XJX1t zYbry1PI6pQ!XJ6+-1a(j_Dz!97tR?xY~KAeZLeRfPLb@Vhn^h!-l6Uzmu2uGZ`~@H z*U^$UUW8SBd7eUZl4tcf&RMOm3v0M?eO#}LlD$vXUEY(_Nk;nT&Aa!C2Bewu`Cxud^{&r+3@<&)HJ%i9Z1^rNj2BYXKJk&#(C~4>|e+~ z-FuVe^_(ZiZ3~C-)aS=@!4OJc)m1!NA}pn z7lKsf!Po4k0&;|&WyI6cNld?r8qxm4sBuKuUj67rB z7!rOmp;!>*`JupqJfYfgeqCIEI@KHM@>~C+K&Mi9r-Zz=ThFauqZm&};pP5%UUJj9 zH)I*r1$jr)K8YcHN_hop+Achuq)4+=vE&#if9{m~mT>S^t!^{K*6eG+TkJg&<0H8g ziyBXSV7hRxv^%R~Y4COYu(iI+466I?%E@O$=TA6ShU84DG{rgO*!!FDw5>fqtok_b z!33T{*r3-isypRbE=dHJK~sk%G;OsfO1(XhL|NK2p3={D($MTMDZeKFLH=%>x{AkP z{jQhoeTI+s`WG)Bg!(OpPgW8pg;FOC#AfpEw`F9W6i}RrnkHANoh$B`n-%*KMgPTy zcs7fDjx0!2jId|?`JAXr_E%rFFdDk_dAc=Xf<$@EGJM?DDmCu(1=<4@W9l<8c1nry?b8SPv< zSz3;`-hA~ke{|#x+eGHuBe=&3+T)6kfHRlQr4z*`rRA$8>WHL-q_ptJmCTe~8Jbe) z>>~nmUoCn&oK47tJbG-Yd3%HIlN}_U5?b&(b(+h@NpB_M7%%_l7sCpWxyR-rma9|a z>kBs$ry3i4jNC4sCTirjVh$o|vvLegxe}Y<#I;VCW&1RdRfBTmMpMaYdakIhS<(9~ z?Dh6sMA56n6RW2PKS+LWkecPxkdQ7r?eF1luMtZT=&Zgj^9WqOh|MK@TVHsO^J#4J=2p7=kSgftP6H24NQHX?^}-Klx0ON)ZeFd$6a@=Ht1++ zE^_zNuP9iQE4n5m5~(C3TJv07Bh8&ghI_`vWx&7Jr>m98tW}T(Uq(5iNIG^6JE!AY z!c3Gmw3j^lwtXrz`>^WKRM_K&KpZ^%8n!?x8CDf`n}f<$cxwSV4kN_0J~*`AuBSe- zueKAbJ1F<7QxxMZ__c%%d7L|-s-H{4Z#z9r=#-xRr8e}Ym^Aj#T+0`UWFJ?V@(E_n7AdhuxQX|{$Mmz7@=y*>50!2hC|KjWCeTOIpb z%tV=)u9S-YxyFnJsyD_oqDCUEyg&Bgbfvg`o%Jy8w=sHT`833|;Lc#~8HeXDLs>G| z-wg3p$D2stMCPu$xrNU z_6}}ia?bW0=8+g9Zcdy<96lMkd3(hf@odt&gL0 zdb^sSh&!j%=*y26O9Wp!7+jvA)G^h-80MH|drre@mi9)9yyS!In&(7)!C9%x4%D}1 zO*EqP3LISHl#IFb`5^MSj>5q5l%VbBHfwH z2fzD%WsR{rM_d$az3G%lq*PI5+{^r0644TNRwnG3r%#$|mE0vB@{N7!PeRNIuPv>( z9?E)MKrfPhsDz?T`OTG!HtimX7ky0Vf=eYR4;p1SFdZv;FV)Zn3FyenPFAb4=(Rt@ z7f|m}*tlj-M@wOlY0WWy8bw)_QMc6~wgB)V@Z1NUNG{ouQp_#Jat%Q|x|6aaHI$=Z70+ zTxJetg(X;>F;SIJBqaT5^i5cU>~fYocV<4m>H#b)u_Kjd^s~g?nT`wXj}#Q}zsR0L znKkB4#Mb3%!Az35FVgNd$?aE9t@;kNFMr1lk$cVA722(m7^9mLd@0iPU~!51m)?f5 zsh9Vssx|j>>HDbS()he-`>a7ZkmqP<%t&#eT8FSW-`Vs~p({5wl<&+#Z~5*Ku2+7> zZ2zA04?&@_2I&`z6gTy(MvvZpekY;gN~htaeOSkVZ;w7Jtb?^8pJGP>Z1!f(k}I3N znso3>uRD1|B%y_mc+w(+exP)KvBGWHv_B{&{h+q!yUe<-^OqXLXS9O^-r4XHUn`oM z;}3Z^S9^xAeomJx(C-6O_eBt=F7?z&2f2ib4$tTEc7+9}(^g0B@)8yvvFSGK*R|4} zqJ1;kDUy}av{0SV)r@-2qsJFWZ1Sa!bG{@nm+90Ti1N{bh= zl+Pmfr5I#+GX>iCzd4ItlSt61bPg<~sdvB~hK{zVX=<4*0 z+T*-5aH_{%tG_5e8-6fFSo=Y5j+m%4)4=M7JjaD6(ZfaGg1=Q?#8T&WnD9%Y5xMs{ zm&x>nI-fLK$qA)1P&rC|^Ofdw&L%CBbe{$fuMyY^=_8lLH*Y@B=AO#`?q9ah`+6hD zB6=}v-%+JarFZmF1>G%J-k+j{Ar>(wAg?XH zyt!cBAoD)#si52EJni{>RKP&wMa`Bu(|+IOn}VgT$J2#ph9PkQCg<3J3qDk$K@h<+ z=6RFc!}2A)BVuwcEUaJooaZCzQOv585+`Unz4t$Hx-ddK_w2*w!?X~Ndu;8~1`o?N z(*`e{4|`EDlqPVk=CJmwUTO*2eNS72kyqSXc=r&@3F$b;_H;bNPm19nuK$RoqgMYh zKX&f9!`Z~tR~j|@XzyF%Nn5f{XL!$des}2H#4`PQDhMM-09V6)au0u&BcK_K-ExF& zzZ^P_4zB<1gyRpY;alFdkcxTF zLsSyjQ2^}}1uE{MJLDcl>^~dk_rv-7uWsov(EKkDgA~1d9sb!Zy(sYJ0K4801t0^C zLfL`PK~MtdFNgw71SAUiQ$^#CP5T{cJ8#P$fkM0;{}+ZPfs_b#+aMwd1Ou>RdT|NZ zAwAF{kpM`Q{A1I$(TRcC4lC=(U%i+h(R1TJN{e>cwS%QDJ2P*IP9%QU8i5>OEFph& zf%3=Pe=tM>#AG|B@87EZ+dWL63G&m_0&LOaPu2eIA||j#_;ca-OI|S%p!M+wOCm^M z@)DC7+Oquer{q815L<}}7`Uj~Xdd=Oc`kypqc8aO~BBlB%ivns*yESEY38QvNvv!%x*rhfD z)@LNJstK>#|9n4o`+6)WYHRG!0aHgm?#F^FJb#l}0h1YC{gPBQ6Q&G#(#v+cd6IpaOKDredjTqEJ1I=h>ICWC{S@pQ*jMrG4fpyD8l)zV7B(5&r82?hXx?Acs(ra|fPzxPAV0uhLnuYX zwa@jQZ;1R!QE^g3y3fAHnV=C3o8}gZt^s_f09$(9mNEX@oyg0VD?^f6lRLv!>gw{y zR4rvcv=kQ+sO||VsHXN>%iX_zc2Z6wQ_ZT*JJXB*3Y4cU_$FV>OiZB5*ke2_$yfZ8 z^!U@>45p#A1*JC)qtQ<&8ToHg_$hu^Jx7zni!*##kkZ@aPVEB9k1-nZcyVBl^U1jx zgZ<1S0a?qwA0IRs&?v8pOKk{YmpT`-+dcD>)<*iq-6dA7r{={Mj~a6lmB{T*pm%yG zp4*vlN`D z-bPmQIFg;t;Nyv|L+KRD*CC;(V;>UOn*wTwMhnPJh@E0otDz}i?)Ez&o>0x3o4XLd zGz!J3E$^$seNIQ7FDn%`*E_u2;4Hw)FtEUJMOpc=yv%5G%>0Qs!m$GW50pbo=ihlx z-;{e&K`OGIRGg<)F6=Zi3(VUaIdBj2zIpnTig7qVwC|9E= z71l#sNy-bB`#*7f8jWQPQ%YaOMn!p~e!9bv;7^WCPESw$+88(CxC7ILWzVKaLd$N( zFab|Kz9VafA8}(87EheyV_fDNCD$U?KU027mo1qpIy#~}ney(lQH`tw)(fe%#Q0yi zygk`Bf{iX+wIWmdqJFg}^f`27NyX&yJsk=Ca~GTQhPZBv4i|WO#!uF_-!2MuzR-PR zEd+b*@bxz{A7_jvH-`q)HzH@#?H(U!pc(a2@QXh8^qbq!shS;kUKC zF*htl9bE0|tF6iJOw4z(?mBLt-D*J;BOy@g<{5cks5-Yqr(yHrI0q%I3>3CH8;#F; z>H_n10zuJ{vQahc>k(wlLD|?QKJuipMw2H8&+?LV*uRTnk08q9y>iTz^3lCQK!vuPZck0=N!S$iC9o$%^R9f zqGnaFs{MY$R!={kby7r`;VlBq(tlkKVX=BU}fx6ovA4omXc#&tUL)K;@>!Dmu%dfvdAKD&A5rR_U>{RUb5OLdJEh|<6w&wDh4QsXO8sVmjWdIJwR#ZOx3*3{Mob2Zbl z@A2-V-3ZDf(P(}s|0G6WwMAWhLRH?Fe@|C`P1&N&$wSsbH`dgyiS5%()t@c?ELuLi zz;)TUI`+b|M{mT}g}H~c=kfPKU+*Soh&tDrSQxp5W>|7nVn^-?%TEiq|NZu<7ala{ zz7u)njNQJyWUyJLy-L+_BheeDh%Y;>K;n?#%lCe1{CZt2x`i+DG+Pi#MRM9Y$Ge6$ zLh&hDS_FK?tL`5vBWS2=`8Im$TDZyViy|Ja{_1d}T)uwK$H32JXG@P!#yytVOO?d+ zweol*jKsr6k>S`vLp``IUl6C>@R4Fzr(I5c{>(a|H@e-d6I#FcziVWpPa#2aw0^Z8 zy9GE2#oT|~eFTe++@?$Zw=5Fih`>1+FyjT&_v|8~z-l`%3XTLS*}{L($hLQ}_4ar4 z{mpngaND=N(_?)5fpc6mo$#Mjv4Q20-6ZDD%wiCrEn!=CzZ2CMn))|g>|MDrnZ77F4~nDQ!(Jf zg#KhD5BZxS_8)pIA}$2L_Pc>TV9oyDdJHVw0UhoiZT@Mj9=6W-cW7<(7zr!`qJHr) zh30(##rgNl=pO$$D=^lM+^MYpe~l)F6~TX+HUEV^{z>@4r1LJCcn9hHpUB{!Km*CQ zN#h@|_!;+e7O=A;yq%rAeVy&tEfA2ttv{G^p6m!^$3RzmM?*D5E13S?F6oE!0t+!f z?~92^2nwS^5q5EK=R%TQ6m|~@xOt?Y1iLt35dj-54ix;sDK(hrKtcj;oP%t@Vm=Bu zasd1_JGfI1k9%P2P!xnH;H-gN6sRzWvx~zv`N6e&pw7T9i3Iit*~P&zza)qOejwPz z!1@7L@gL3FX8kNAB+!X~p~9a}aOo&^3BZa=h=E)DKu#kH+9C(p9;`sAbJ692ArP=J4zJjM}TVKCWxWSg1-X60vZsQ5W@Ts76KhW2cp1R zCoo0;GQq+Tptk}VhAt8*1d0(A7lY%0J_`uJ_!0v*BH+apZnUT{kUoK&aKnJdPPl3b z&{%LM29yC*H9%_62_O{`g@Fs3$prBb8jCQ_0A2wWK{4p05G5rg}%6+&R)1<;iQS{1vBz{E&`IsrD&Ac4mOKq43=01sd} z0D}u46v-|L{DA-*0#6j6YoNVg9D=nC?%AT^Kx&LcoZ@FWQV z3<9jdLj_%eBrGyx$J8qfJGt9x0=gj*V7*0wmcvzvqv4Es#`Jw>zqX(OFb4((Cd9x8 z{VWQQy%h_gqjzORCqcrVHMT|&+`3)h#S{oU#{oGSfFEd*B=CpwV{UBACM?9j-JgH? z0V}otp6LI@G{D%{|E~>z{@nW=(A*Yn{}*Wf&-?P1`v2zz{kb3iS`PI8A@p|ubz6pI z0by2cBzhJ9xq|=tgn9qx5WReg!BgwUX7JA$;IA%{Tl>b}lKxzocc%Ce`?F75-d`{t zGJlu_`3G;n#_bB4_4-rM|m6J4u`sE^}6pz$IY?ra8mLGw z7>0M=E}J2wgA{MTfqKYKQ+eur|F`^GfNt-F*p9jHP6VoF!m?a6UfzJRkT;Su>?zU0oo$)fFZ#WT%-4>M`IG zd>|koSi+U`!S{=9W&3ct3Th$WoXp7B>|yiVtlO;E#9o;FPL3ek+!Pk-*|f2F2VvE* zi|dnd5gOtel6K;Dk|J>(=;f;AJx6@Mj**PvV6CVq$8L3OCtmS;#uPb~j z*3aM9(bj_`xKH5BJqmdtT9xH>x@FQwWv(oKNgvs5k#WSik#oy5%md?NS3i4v@I5HS ztWxYk=A200cH`2b^bI-=#ZW2I+pg^QM($;iay`Ts%NxtSt@|qT)DOTob;i5Z_O+hZtGB zKZRkb0BDEPX`asoq_AL97P900(%x z+k)BF#)q-DyqkT6bt-rG9)o#W5fX&1tp_+S0`xgX=~Bnm-`6#m-2!k#!Wc86LO`Pa z^97s$!R06_dVxgjV(bEdun+wow}W9NTN zFLo@p+w}iWKXcm<1J>DHotzwf9X;(Gfv7`79AOL2U_>Mkf}URf4vtQMsz%rX3SLy? zM~t{I!WJ-r;>aH{5Wot;76iMFkasybfJ;T97kwaDfeS>U*D&}6l!XMmHH;&4A>a+3 za=S71lS%_j1Ctqdw{qHMD}08Ee-~ZO8Br0KA46{ zY)pvBBZ=lqP+%}gfJ0(n=9rVgy9BAh@)V79NKz8HvpYT6ixeLLQIjE zg0>n77(>ae2LG**|2DxrVTKcv4$#Tq876ro0N;ZG7{_oBXqaJ2LPJL!LKm=A1>nK}1B%AdRu!0}=nw@s zEl~B27?>RdZ14`yZa>2)+}SBicSHf&;NQ=;t^WUv+?ffE#b6JEF-*gDLIzzRP(=XT z0fu40j3D&LfWL`@nGP60G*d2$h6p;{j@k9Yhn@w7lNCdc7ECA!90tNXGaO>bHW48- zRJSOvZ3+xcg#pe*6wL*r-{7cN8JAqkZtV|N7f%03b_219noSu!sXJ8t8C~5`>vX0AMiuFdah!6eBd)8hRL3 z8l>A|uy^!S91KDfnz2R8DbV#{^1`JI|t_meFU1+pIiZgeOHTv1A5$_1vR;nL9)4E+p$`>#I{Om0jFfQKm) zToi)gj1Dm|;EE6gs*waPHNa8`WJ%)abs3CNbay4t5-cEv1Pw$1Ac$fRg>E1UFy%m; zfCh!I5c&_CL=rtTU?DRo5dAC(mJ_f}0bvvh$d*6~^iTmcgY3e>k`Ou`{f3E?K+hFS z55#vo!+qb`FEKH&s-a88tO#4r0PbMvfn86cp@oUv%7>1HO+x}X0va|bF-ce~wv`oB zfO*0sffaXv>kFhu?v~5-dn)F(c400ng&#))c%o07E5k zGWePj*qVXy4LB?Kj25v0l`1$J_>L(MlXa_bv{(rC(YaMPoC^E^!w5E^trs!0bna)D zcR#^z5T^TZIiezBa2tL+AyH_dAQ~`mEV{esI5DuCz>5|TZ2`-n=o~*%VZMo=rCTCk z90ItbpW#%PZ zpD|zm4?_rDG^U`PA*M1|A_~Nk=nxYF;}hQSMd1+E-vZzVhcjST#hY*m@U#H494rc$@6cAU=~?ROsDG1h|i72e$Y@2>cMhAy_nF!@2PD z-$GcF4!**h1-O+7mOpe(AjaLw0k(fwJ}wNnH}D6Jhsy?HK@s3g35;SuRAZ{dRDgNj zP6iJGxH*^*lj7$$k*&hEtNL3L{}2!T)~Br=Z}l5)8oCeYqXRK?2)|=`D*>h>pye?= z1tr0Y2`rDtOfh(=2Lu#2)&L_A7^Q`~h!H!(M<6J$Q~>8$Fr?5aBmwz^S(D(>;b%#3 zPysV5upAp+)g%Eq2U>%MGZ0yW*^hq2gyMj$*hUvz5vCiM9`Edu7&sJw>0u12#L;3& z3>~`z$MEN^Oqf_O^a0h6?k5H@=mOBNLqDOv!>XSEMIZ+l#b|W>{Dy&%7&_fnC?o{N zqVATh-Cc*b80iU^Q^H8(w%`KXhZL7Y32nauHf#HlT^Ou_7+o;5NP~Qrl%>4k6E>k5 zFloH=Xc=?l4fFcSj(+yOuHOD$z8KygBQa1=HBmTfbVx}@(M%ZBqv~$!?8h#;Q#v33 zvfNwF-7leYJy(75UDUJdum2AD$9bKJWz^f!+`ysd<_@e-DZAOC7@V9k$ zwO8R|*U=H-ALNAyKm;O! z5W$F$0~m0qx`I9+(dgU7_JTv1pXapNC`V6ce-{{|62jYf0~iIH5Ni8%7e>WloksZI zA29l$&fZEMAawg&>ZfGj?+DdWKcc5=h(RbAQh!9K2#io+fCiW}_`iZsA<((MB2@7Q zO4ShR0IfP`gl?nL1fUZLjvRem{oD|CyD;m7aB@OGPOiV8)Fs5*#nBUB)@=*7|A<~~n_&Iw1ZMN#GYbqG4qmg?=<8Sj7n9IY=S70*$pck!~^4IwaJgfh8z6!w8bZf>6 z0m0&55bOwMnLC(T-rFz^^4bkk7=XKH%r;XbBO|>7roepP_8TnA5&>0W;Jp*3KpM2u zTg)#o{m6!mPX_^`iT?wj>gg!z8)yQcY8*N87nlOWivmbs%|RS!l8B>#1ntKPC=4d~ zjujAoRfe`93e3pwUIBMN6kZN~08|7n1qeeiOW+YOBjAYuR9VffY$G`u~CzcnbK!d+@K&gkiTEn!9%4?M+xh2=QYT z0c^C0sJP^x;0bsQaBbsn1+0G*>b-h2d!UmlZ_-SR>gQ-oV${ ze*h%(uT<4PtSAbIU-#fYlU61OQ!wOV0_(?)Z0ir!B0FDO$llG-A5AiW7x)=NG;I@N z_Ff(yw!f2Ojy?gl?g+gtyKBxIMcOswrV2 zBxqa1KTcH>AL1WLbz>0M<{Y$&_NN>iR8Q|D6RM@cr4UQ?Cs8L-=j0f)>dg0#eVE6d zj6=hA&2PxBZGB_&=y17Hk@Jk(^3Zar7f+~%te`my!=aL7((cUobQV5>3vWBd$8oVD zzYWNSZX#=3+@uP!QcHH1cJB{yhPz z+NZj9Dz8Tp`xsSFScG5?IoSeUKh{YJvJN~{^yX9oOA2we7p2d-(cKBV&olT@k=W;P zc;mzvCzn2sV3%UoW1Vlt4!tvpC15JFsAvE6Fu5c))JjnMvUNSE;>jb3FiL*Yo50+n zTCzGSdx+}RzFPYCcmys7ZbN?6^@AvE_FJu9N*7WxUgjQd+JonVV=l}5%>YG|%~gPl zB`dyS8s_;%D6XKIrJUv2r=WP5C-pMA=G4bIXWvheovy03-MFiVje~1(_}pN|M0-ZS zsj6Uw@J*aDs#}TEvpnuHmLX)>7t&d6jANc?v#MO$Y?`W_!BSwS;^RLEVPQSNix_#c z1j_szRvopeKfghLIPl%<=gIdZCE@NkQ=|AjKD()U>Cq!B zmbg;sS`v4{NV|V$>WLRE$B58N?I@`kGnRGP+ zl_9fr{Kl+(n?)%pev!B*Dxnp8EXxy7kFX4=sUu3*sorf48$49F@Z=8G#^@cA(g5tZ z4fCnd)D5GO3G3zdwYQ!n*c;MmH(qkO)?vMYBt1!J@s1uo8N8Wwc_a4aCR-h(w)({9 z8$@b2@9n#$)VdbCv6)MNQo8zR6B#^K7`x87DQ3x9j zP{$~1lj8K%4OY%)I>b1H{VqxaMbn#HFSC53hqXT7MMjlc4X{74ro&=DP^}!UYHYn9 zc;W%Dg|PY+ycjHYMzXO);uBmnS=sEh>B~NJ9<_6aCCT26 zoUW=kT!qEjzd+biRVxGyL$^w>BGjhKKk9MeynFSHz9hK@6tW?hgdh|YZ{LXZTc^nrBO02-CTuq-uB@Vgq?t_H{NZIe%y@S z3~s5lk?dji^J#m2RHi#e{NeXIN7G)!yM<}{UU6Mtl@=EqRxyihBo_=tS{OUcF;5wa zhg1-5zD(QyIi`hXFmtv%Ep##N4OeEF^~rL2;;NK>k4vO`@^DA-haEnzxeDE?TyzZl zHgjx^i2${~oIUW4@OaH46;$`Sjk9Wg(&+Ttujzv4Pqlk|o;pG-TVExR`Tnd}oG|&p zE4HCt7sa$TXj?PmKM+fgP)ZK9JRcXeZCk<3Jc>hjW7C!&$|?RLWj*kweVX`>)8qIgdE*=Kb1eg#M6(Zjo|RbXtmcX0~A3$G~MB-OP4Y zsY4pmq)S9rVaa1sQZB_dc;t4v`$Rk)Hp&LydGPn%`IPyWdi!^QK;HLk6K|LVaWkcJJ;mE-Kg2a_latpahrKRp zMv<89JyNXk`M!&cpo+yU{R8$)$s%2T7Aq_d?-2;J94~(Y+~ppbF8vy2drW1C&7C@( zIaRL7#CYti0}ZXst?Vafsy!9Y_Aq{BH@?zsi2G(j^@6<+0hT2PT0>p3G`qs>`9Qd2A4o zA6MInQfU)AJ7taNPio4O+8goV_MF#&QmOCD{D{!y^EgZO*ZVkVd+OiQ4mH&GG0e2` zS%j$P8B&UoT3ZIWhD0xt6;OX4hRDu_e>^IHpV^Bwz5XE_7l-TWE2q^*=TvbhGY1L#2^$_UAC}6nO?me=gR7DF!6$wt!_O+0IE`+GH*c(6 zPdRhYUF&IljKo`kk1du)bs|rMc~}moSX34tqwJ%QklQ$RL48sr^Gm$6)@(p>1BBXV zv2y>Jw^2(_yY#f{x;z2#F=GOOc#ranQDrq8`B{OiuOC~e<(Yh@c)54RWyF+HLkO?lN+sJD(hayp?JaPoGC}Y z-ykOu-`Mt{%!MZLctxJhF{3Jm+HWa(t)JfRB^(t$S!ngXweEaQ@IkowLlC8HGWMlz-|tRGVqJnDHOq?{hj_u{e%s4P#oS!sT~ z>v`L;vAg+0iR1alhvJ1k)yumb;?=z=95;FKk95nY-H#3 zuMB-sDr~S@<;vQZ^kxVt;dAL^mN(6+PVU=aL#zycQs10!DWwCnN{8%h7jzmErjkj; zzWBQB)o#68KIJSKfA2-N6&uMLhVMuk=TKKpauU`;wL@>`m%L-+Go5h6lTY24>Wa~a zbc9yu-!k*~zIXkU@K%<*txF=auk`z+9x}oVpG@dsTe{;bDjH2>d9+Z_OO5u|??VTN zVvcr;JDSkILXPf zIO6q@%!CrGv=&^P=145J{Q=VDTO&N%8M z4nFg}ZkvS$3G&&Se(!rPTj1$hGT3Cy_4Jb~2q8UYSp32sk*QtYBO9Mo;UN{m>*x27 zdNcq3Y46L!sqDJOpHRw>L`o%7GG#ae4xtQ1naU7~44E^}L!3wvk|IN=t1D(Alz8sB!VRl-g^){dh z#`A!-UUCtu_2&DZZ89rl7k5VnH(SSSt$6*Uru;1PF5UvvuWG&P5AZ>mH?B{=EVdrc z;4s5?`yNxyK(}RCt6dKabyvJVqou7~f30Lp`i;|vGfA5pMS@iu4YUQyHAeAy?iQ^z zjuzQ_QFS9PSvJ2j?m-V(={>;3=-nRlVo_R~UpK&UwwANh_Vepl`GD?j6UdOC6tc!6}way=U($f<-Nm^&h2lX zlz-vd@pOuilb|nWqCIEibWX_*(SpEn{AcRzs~&4zXV?v-xV7C#?UOjRtrlld+G^TR8v3lcHLdlz2w zeW*sG#k7w19qJIm?n$CQ5w$7Lq(2gW&8E8f=r)NSiyB#3Mp4b|XB}#Jn;&!EuxQnO zu_>Y^6V+T)k!IjB{+-p~_9IiJ(v-AqX71OUR5@o4oo_-_wLf4EL3NbhS*#oNxD`9R zpml!a!+30VesSWnh$rRk&yFxxy7C%+2#8&Lp?a#oeXpebYoS|3Z?C1^d=r-A;<5vO zGmgfm<*r^U?Oyv-5gImI`T6}AUz3+DrWlZ6yUcF(gw454_iSW)8l~M;)RilbZ!A_` zNBhs?uiu33y3wT3aNgtMaSfXJzK=RBbJW8ogu%Gzv)JynKWH;Q7>V=deS0~c=7Vw1 zqYt*!6*}lTCEs+_tn7_iNk@U#?cZkD%o)7n^D^%67^~v&?x3yB@?F-{_oqZk3o11` zKdaxkCN0rtV59yuG}feg!nXL_=Gj}{sXdq&h4!T;zc}iKZdxbuL#IVAQG28A3;dkS z)tdXF`o@x)uWn8ku;ultw6^*GJdC+4%+4NV!FnJIu*R0RwP~6eI_n?2__2AD?djroFFj8O zj7Pdpzb5%9Pj#$` zCWkr;TTcI@okkYQnOV)FMoRl{N^?!!r$6mE9M1kcht*f%lXcBM1?*SE50CTAZr@I0 z$yi_*q2`g`lD1%I!R$Hv;R2JFnt6IQE4ARGi@uvOb}(I%q{?B=ZMThVb2(*JWpn!` z-c&XOkH0MUez>{y@vbDAvA3VQ^;oAKy-~9B7>-k#x%4SsjWa95Np&!PAy8M`OKq=i zL`Au*-33QRKb2P3lBCh3Yg?rbOb>0JJlOqegA=uR;(={g?yu|`MX|D5Gw<5hUc4ia zkbLt?i<{f**Y(0eviHh=nhBmnohvNyt2GX0_+)|^#Gloa(CpRu+VR5cShD5N1^47h zg|RQsRqx*m)N1&}yyJT4zJ;fo*|tnwn!T9h!M%AKr?2TRF72M4q5$6$9Stuk<=z$A zl&KF4?k>Nm>$d*7O&uE(F8#@{|3OCCOxvbx)OqJZpJ+B3w^A16Z}+?(o*ZmuI9Zex z*0?qNgs;~5AbkF4Tf_BVfkw&Ou6gqm?UQG(l!2q-ha-C zervn3*72;n4%_Y0y1*xHeLY`y-ZHpx@0D}~hvG4-$@<8EPQ8=y;@yW%-_%?Xwqe_6-fiue>w1wP-0`KeEL*seppKAQx&O^qHw-eqE zcOBi2F+Fa0nd7JT!0l6eo3D6l?UL_#W4OL0V#C=zw*@Ce2X>4-O)b>(H_^{gnmvA> z-j~bm;Y?NZyc>{OoYXL5FJ8(T09oAv8)HQ`g$ zF2kP(U-k0jq&d>2-#T$+$E}U&Eg-P@8R3Zd+W1s=FKs$si#4)ytQ-qq~RU1 znEmx;pM&1S+)8B&=s64$K{N{QP-<3s-5OxE=VSWCr$^>5YMtz1OkZQIdnnytf2*JD z&v?6*vu4L;?nMbP1C2(>y>UFlJ1&plX?MGa{?y-c(RYpC-Vfq0jD(*4*8dUD!&yA= zd4Q=(=Fla}9D_wB%qyqco~l!rmdQMw9&R@4F1$wDrg*2Nij`S>-Z>!D;Jm&`Ypv@o?;om9XWu1T zVf}q>l?LYlVVUa!!&+a@QUWiEnZkh1UmIsr(A1v|5y6@fI zXxge@^wR69zs-KRqqjX9Yd!CGt(&;+Qp&4A|LTj@`%SKGLGi66NfNi>?;S04Rgr0V zt8vU?-&6iWniq`w>)a;Zhp(r9|K-HFt#LG4*-LVBCXzXWUnqSR5|=HIY733I?^sk= zP@6D{w{vKS@3twXU0Z6~*3IMf$RW-{`O{e~0SVbSmco_;gP#OgY*F31Pp8hEaGg$y z_bG~Jyx!|;FaJeh)GeM%}lxeet`WYKJ zA-`7{!xvvWFbbKf`OP-XOx(S#G-D%5?Otc?V%><(z8KG!bYXta4{MvbaJQSyr(**Q zI^Wd1yo)*l4qVmd#udck-Cz&!eAm9V5nm>hNgI6rvq>jwa1%*naKCx)@d; z&DJ-I3&qp_u<4wg8k}q|lVSX2OEW4Uk?A61>0xVf_CVr>N4M48v|sF-JYr`zl0x6V zK4OEjT#;dxU$647L%kPhIb2Q^qL0&OZ%JLZ`3aTvzAz5i{rG})Eye+Bw%T6pQqv3m z#=Ix-+qF`ax+zV)r0!?*ufHm24sS?v-B$5_Yd_!q51M$PAUz4ul*FNhDGlF$-k$xC zdPaHp$C2YT{M@MunVa+llH{mIK`g&WQ+}V+IefEmj?^KRxj^taN5zDFLzvJHRiXgzE>}Ow^Q5h zN#^F{SnoXLmQ8i{OJgQ<*6u2^4Q?&ZHQ6VlCmHdTt?B)67Jlpb;~zv!^b;4f88F)n zby+tS9>Tm$o}Rjzs}N~(efopl`^3KPUV~QahouE)Oa573war=mmHU_IhNyO{yBkrt z#%;HorxIcw^M$r+nLT~v7G-?cJW$1;*0w`c*I~;It(PvEZ~LaoXOg-+Pd(GVt#!{V z?%spmmiq^{e^a>jZX2F?3a@bb(Sy1Q7v8#>Guf?C;FH^G9c(5TX zF)DG<(OkK;^Y=8>?WBu!Co$!I<75@v(}UeZ+Ls{mxTZq+VAJ1-;6O{9lOraP9+$v8@9@(MIXpbuGpMw zr^nkRb}`y-%G=iG;hBx6dS!T?W#7(!vx{NEy5wM;p6{tJ-3i965iFyrUoI!7G}6{v zB@JG1rX5PZ?ZR-iR=m$cnI9iNt#QM{(ooLCPb)Q}_{7%|RjO}`8`U5Cza5#hTH7vp zIrHA8aO0n0<@`)6`!eNkbAL>cepjh-wf;|7)I*h0>1$J^fpiJFFXALC4ZBU#4woLT zSxm7LuDgS7_s))qdS=E|*4?z;Q01JXjmoK@7u#gZN0aOOho>(qa*6P&9KxHs$mH!k zQKoIZu+wVn;E$SZ^K;ujcrbi_-g>fpafT=9!shI~P5x>ZPKb@#@bvw>v2X>QbbiJ# zK{b&oFN|p)m5tA9bH?5B*!`CAT#s1WuS~DQqBcCxsCxa*;rPWAKDonpdt;Tm;}6Lt zT&Wu<(U4^sk(F)fuh|;+K8&q5Y$G01`|O5h!JQxp4Y_A2XSP(0(Mz_x5Up?9u|_ay zP52zH&B@Vb_lJo1$1L5si(wOFlNR@cqHCDYhs(6_a&8Yv#DOK(~Lp^%FSOeA@N4JI(XNo zUxz2GnbUSjT+*Rw@|3DrFLS4;@2u8YjaRLmYO%*FCLTRMQti5?Z%$qNWtE)pnER%w zN3YbV-6Jf882D1Jw0=DN=7*Not+U||c>CRBE7~M(TBAJwgw>pgE}OTHh`#w#Z7hFfL7 z_x1A&N<5}DJTg}k|0|F=IYCJBbXRTcT)-W*lWNDgC)iWBJ3Crr31h6ff zU*6#tYOl?5@PJXgZw87ZfwetjqW88|q@a)3)s60f)t8Ob_l$=0ZGY;q!+7?-8_#7g zbv%#s_6;5#V~g2r;jh?dJjXg;B{$sXd*#B+%}CYA(jL*9Uo9gw|Dvn3P8{UJ+k!Uw zb*px+?`llogTa?`p{U3d<05IT0}G5FGU{;I0+$ZuJKFc|j;1XsTkO`B@L%`*Sn%o7 zM_OA)${W!+Jw-!;rN%X4H%bRLec~Q{7QUYYzq_uraDXYaq2O6^R`FZMH+MOiLTw-Z zcBD6KT|c4buuJA*T8Hythrpmy(f&>5iA@_qv$S$80}X<>69aLf{oi?P?;G9WqcITV zID1QeKlP6M!INfgEW7FWIySZ5KJ%NuHM7QN=!n$2FO{3N@b+BQO}A){(kM&+Rx=UZ zQERS=zd+xahmG7~eXRTU9B1{r(`{$jzUhp~u*u8U*axevVc2o+V1kH8`==)@`VA(R zcg7zd38qcmq2-}3m%DCcFP(<4Ubb;|MMBd%9{F0f#MTHpRR`OA0h5sFQt!IFKSqlM zg0ziv~XSPnPta7A*auRcEx<(uHu`aWD88 za^fsah*Vt_?^};gvTU2^FZ+!Iqs#kzD;S?yOgJX$RIyw(Orm1f`8|Alx`f}HE;eA`?e#^z28OE_ zx6-x9zO!w!KdbH{CEN1Y!QlR+F|I|zUM4xK(nTwp-asrq?7KrqMLpWF8;dKj&Fog< zpD`X~+@xe)6+B*%k$h{5Db7MRfAXY{@F5F7DLjjZ0*g_W&{o$SAvStmqYjd8mo?em zLmi{b z17pX)BMko0`7z9!{l#elwYw^Lu12qW;i|&d_N5IHDOvKG@i%#W{hk?ICyQ1nHtys zIGNvOdr1;|vBYPfocj48PN9Oh#3RNRm^2src)yM0iUhm}G_b*UU%=Aw?n!>-WEZn{ zg3JF#=yknxrJmMj(O<4@+~`k#!c>ZVG_2;1ZlkA?L4%6(*rc+F^=zO<^d~;5&vsZ= z50-7m={G&*8xk7WI4;!Z)UIWnQDSElc+Xr*QDdB@KtQg@YFgjRN?2Q;BeUff)%Ot@ zi88NA8|+U_<5v6TF1)U@`c`_|Z2RrOu^V2q7@y6OzZsXQ=1*s@Jj0m~lE0z#+l%)2 zCX1QPo~pOQh9-h@yYkMeJyN(`YVrGCkYT;7Pxkt|;pG*Jd9Pa9vy$%}ocD1zz_qy4vYP#nW#PK@Y?0LOGdjb`)*C& z=4qDa%U}qS!d%vrdMx(j7~AnPBd^c4?|FBS#rEwJ1(h6T<)ZTI$xZLPRMgf)$r#26 zZWdq(H=j3unQffA>p@1%k?yB|swUqb8dkRMf2%z9Jm2%HZ+cMf!MC4n$TY}ju;mNIhnQMyAi3PUvm{O@`o1WZ#7D*KOUhfX zOvUyq`;cfz<;64sM#VcBB3;r+vQ%%s?vMEgLvOUx^#Tj?r${5}O9n~iJ+r@CzgX2V z#c*(14GGdcP3s+awp*~zdaLOhn*JOQj+oG$7qB`8;mN9%elhGb@0)7Q(y!}xe|sY( zLU5$*yy9~ooV~^_AFxg(II9KN{>DbKyRDH*VjK_M*l4cA;M- zYB|q-_MA@KCeoRZVa3Pk^UJS%HcRH-F~v3xt+QkN+c_&R9_LWM#s2Wl=<%nHfh3V@ zaz5l0-x1vFHvFQ~ZgX$9$wj+u*fD*UxOLjs!>}xFl}5c!&Ukx^JXgob7MGW>&g@qS zDB$AQw|?}cC5?4XYii57of~=gbfGE}_eCD*<1*p?6%`ui#H+GSb*u1I z{DXb_wEL%KoEYrdUpJh!9f-Oy@MULO_J>+#bo>vwT=UEatSNT5odePp`N})kcORn_ z3e)znbJw_UoP4?Ktp$5|(MyHTX-x>)#Y@8-K3jmt~4zU%EYim$tfey`hpKQpq{Dt6FWLEY2gChmvT+u%PaC{^eSK&Rh1>5;?xpt?_`$nT`LQ9x z=7lWn&2xf2Dwh?gO|1iIXDTQ9AM*OE-e;Pc7PkH1l&tP^q#oqk=(+L!OUF?9Zm;h+ z+Ky^`Znn~=f{Jd`Ooc~7WzJFYXP15}aJb7f%+Tx*OPgQI>|drhv1{zP@ZIoxgE_5> z46fLnCH&FU=bLlR-@vA!z1O~pd@^~t@_?bF&$#cA6S$(fN8&g;0} zu&}nE>>72=!R>ErH(~^jKEaB-9{x!`I#Hi1^Wq=;^ez;)vE&_>z?f*I_te*&rbEqa z*qny;sWwdH#;M;M_3J;fS8IZ%v^kPruPJMf3+r{P>{Zc<4u%FE=~tg<`O6zj-)!oS zJKDH~u5;j|Yokfq`(5Gm>!0`6s_Dp^h7NHm#)x`N%fI&LI?Ep!{NYA{c0&;FR7tgw ze)UhxlLL5h7X!aCI{p3ElOp`wJC2LK7dR{a;!lRk_{G+gmpd%0KYowoyu^Lat8ITF z|F?*CQ`FURsA)TjPRWHd)XWO~$W%+%8 zBbH?*%dtd6lvd6EHY4uo(SGda*h=rIc76kk*Ry=Nf4=M-npjK6Sl)-yFKx4GX^f@X z@G>-AqOdCH_f}`7Exv16pPYVs;QOC719VzZamR+%u;9i2n-ly8Rh3R0HGmI>g0sl0 z9k2x#D#5`}a77IFKRNdN_XD=ew{0ZBk;A_nuRQ`T*MQ43@YzkmH5%y340K^;={P?) z3JDHD!k1yd5nPH>kI)%M9r9ziD@Xf@M;pOK6G`Y@62d`8aKRWW@t0$d5+E<(Uk>5s z*I;A>@gQmZe{l#`Ur$T*@L`zOe>uzyj`YKaZ~;o-&Mcfo@h>+epd;p%D`#*4OIN+w zBmv5;@EQ!k%BguY?;)|8l_JUCa|Y-fsr3alyBcFyPEE;`}qX zqOfv2nZRnmc&zyH#T0PWR~))(MS0#-N*tUK|10CCJMDJ>#*@DJ^Pf&Xt0@?2YMlU# z*D?It@n`6giZpn*0R?WAfrk>nRchju3UC&h>`Da=u=HOpm64ny$AXG0%qP22Vdm=U ztO5X;kI&F!V+BegIGs4i&2Q9`BH={T_mURp-XyTTCc`Vkgftv;O8n=x{Ash zskoE@KmVIcdOEs>iU*YeKmUH|2Ry$?I8KfNegDVfGuCD+o{=Z%_5`h#y&`1*C z@b`bdWVEb|C@vYTDE_}+F(TfvRE2N599_O!xqR%O^8V4%1LPqX}-8j}CfS?@w`~x>TcX zZo2cM!sLocP@DtP&9RADpXy!pT}5#X%6dN<)vHt*Pc;;mxu+-G`BaqAGa`DXuC4{U z$aGpr=W_la|MZ)J{_4r^^2XM~cSkt)`gFzML)0t}eQ4e7&^=tO^N{H&9#gTW-(^N+ zw01F68Zjp=iVhC2En>Th9TWNY?2;ZVjFQr8-(}XfQ=L}OUM8%g zRpULQ@5hbbsJIRF;ughSKlQhy(6TwnHL~z8)rXn$KT{X(~Fg-c{`YcHts6T zU%;QV^|W)7PTsfn3>X@F?X!Sm-_vCfmGdaOe(te|Mv+6e z#=|dXU&bEHqf^&el!|K(>k7EDC-(K=ES=tCx>)pMgK7ciaECR~X?LqS*{Z5Ohtjdp z))^Qi3Ftq>8SIHy*T#OPl4JF~_tInQ{WH2z^>~$o{XZR~YzBL&>TuiDYEPRU_7Ldd zHe;0KO`=wLOidfP;i>%dMn@6%BizPXXy%$()1>(^s=oRwIuR52S>B8%*@65?w9KFP z(#U>{RzJt*lBR`YGHN=POq(DRIl+ao+2YPf7qWMAmT-@Bw#UxF%hr)fUQy4(ox7E3 z{sf)HZHd%M(XOez#2Z)}(xFdR;_x-@S`La@bNzEKRkgsrmX(zgBB)+u9;mG?Mn5MPF)Qo+q`P<#;hjDpm7m zV?;`LkJy3wEQ3{^W?Ln>GjuB#@iZF>Pba2m=4J_f0{MIpx2?&pWrIt5OHZY(JgWvR zX)DjFp}|tP{Hz+m0ua-6(sKmYS1h1)43aZ)9f;%FiL?s-w}{@W{}a9c??unoG0S`v z(fi9)(|;8`u@Bze&gy2?u;dcck*iXM3TACfaN(Sv+Ia?z7sP4qx!DH2PY zeAV-SSG`ST#8r=TcOV^i+gudR{^WJL0)16=sm;5OZH898CWgxNu3e!iHBQw|-!pm# zY9_31)md5de@n^V+98#Z^Dz3$U?HO;*q_01ILOJKIN9#n`GfPLTPy31eGfYXb zp16JSFvjoHOy@BC!_U5O{Cx`5;KL661wZYbUigqBVBOQ@xR{mVe6!;C^U9WTGx4of zsm~c$@NGG{H|7H``85{PTX-a3Rc!7u-L+P*mpRg@@6nd5m&akl5Nu(o$1u1hApG{z zEvnt6Gw0=UhW%Nfb?>v}LYkjSvRCC~rm%zldnf$&>3m#Kv2?+Y%Aifv=ii52_xx#B z;^*DDOC&Mi?a+m(`R)^YTtrk)w!{=V6)%O5|&F_$@$(3sxKII(kAt<{#HCcF)V(m z3B@xBhTf^{Km4sCW)cpAPv88l&_6NJeyMz&3wxQntuIZ=t69ZKJ->90SxLsA!OO-gMH7Q3&x*ve>il{zV)#Op80uU6x;gRZ+Ewg^3E7@d^a;4o1ehO zKhNs&P`L4EC^1=M>RJ9|Os z#JWrdkysR6r}@ce7WXp6i!MBTjJp-|QeiuNI7>fpv}}WFcPsIVHY=rtoG|jrD><~b z_N=Zhez9$G>knBYQ+Acpf<6p;GdHf4-TffYI+(sE)U)Rq%B*$pQT|DKu+X$dtG__W zZ?g_4eTJLqW9a+L`TLci)0v(Pg9^bo`ZY|`yID*R8+~PIq;AES4k}nh_Q}55d19^c z`6}-4;BR!^nrAp_eEN#}nmJfUd6J}hrReB)-=^U;n8rw@)3kEpmmB@74NvV15_pQ(nmTdR0@w7;)-#kJ#A6y`!rb zfFMS`qODd@Iv@oN1SQj{h$k74GWnnEKT_9gX=NdmNb>h6cJ2Y5pwZQIdeM_a@xz1 zFI-6~YfF=xNl#AS?`7=l6<5$dXOTG8*8ckV+wY?~KIX&}0c1p2fdT<{1p&^2Lrfk=h$cZkjn}ny%>3~|X4knKd zS=87oX|D^VnQx|Pc3m_LXsH~Cex%f;yLgbHk2=d$Z5|9RISBn$2l{NK1-j zLmK0?;D%3~U&N^pI^();15{MWmTlm-`lydK&A zPOcR0U(C-}pU*p>XL+(Mt>b#(CkakTRh&uw9f1#l4q1N&SaUvd?#Xi1Ef3B;sxr)V zHSIb2B9+(2Ev9I!fkBC)0dA*RWiQvI3(t=%1U*-{WvZRt97_K+EiL4fv6oB9p0#$; zf|NrcF5Rl&fpxTKK3yB+`|qM_GD)E#tAk@|)K6SDCi!Ui9b@{$ky% zlBS7|yK!3+rzIjFbwbNsJ6pLIxyn7zTzTQN`@9K)Y~Q#K)V+`{~I^fBRJvS0@#uGD7fCqwCmakQo*i}|o#C|J6oZ+q8fAd+uZj-+4 z5kk@k_;p@{JX7o|ph@>%mR4BkY3je4_4zkuW2o^ckQd~?B@MZwiKLCfWg&GnF(kHw zl(Zt~96=eu_8>LqSKBcGuWpf0wwj_Miya9Oc1#L|-rMCy#XIg+y|sJi{Lfa5(&(cf zfi|)Mn1H5$?IQuDj@Bi1GoE*MN&Dq1`GIT8YWR^7c*kU-Rk-zD^ zARRIN*mk2s)NcFV1Ck+mf0QGcDu1Qt+M7sme*YbdztPd2@^r)|a9v-eZ-bDZGe`38 z!LHgzb4A;8<7b*0(U;CBl)f%}q*NsAP;o}yKFYb=>hQ4#)E&Bie!SGp+ppgIQDJVg z{E-kyQH5L5Y!F4QNql|Od}wo)vy1A6t+a=C8|^%!{sgCL{xL$w=EViq4UBS=-+aUR zBv1z9myY4*tXoZ>6!`woZWCE3T9*gi7~%gp|RUUH==lMov0o|-tN7lEcXHv*S@%?a5q$e z5lx-56gV2+{+aJR&$@y!2Gry_X&dHy8o@VBqH^$5V|N4(s%viBfEs${mVUE!-EZwT z7^_Vz)*IydgPxe}WLY<&)MB{ZZcB3d&EqXy>^Lvw(9k_QUpOGpVWkp3-x=;n(9F? z$>t~KG-=Q%p=x1of46n!sJvSGjA6->a&}Fcq<_ToO{-4Tm;s$7CFsJ?_fK$(zSOi# z=F`ISLce-LQ-AB2PHEHN*9_I?($GPQ3Xv=a?1BHca*fy zL>mORN}PpaM}{oYMGQrN=gmnZn>-XzWO`;z9u*!us{G&5hTL&R(njI&V2QL5NFy7UTwwb2xcG&jr(o9fcm=bq5dR6M}1EYPJ?2p^Y@m zCh+_u6kyYaPOU8cA4UPdP9R3 zp`p&uAQ=w)fAG{I=%2h-IQ#IwkP3LWI;l%(Ye`8UFyr4E124ZVaf-lWfE-LV_&*^H zGFfvl7s;E`@^H5Y?;nGafP-pWOG6I@L&0$@h)m)bNsy#qB6&y}>|*NLg6IFZ;Kz zBY=K_4ih025IDCOdG8=s2|O1RKrR5Xp zAMh9urGBm+u#1rNcmJntG(~_`6kfONKE;vOB{{gdy z0$unkIl3TF4p&eBNsU%V*Boa15-o&w3CORofjlM&a}+j60N+5ni1Az+O}M2cbmR6v zShHnW&%)a(a8=l1AQ=qe;1l#L38lmtO_&HUI-nY1A}sZ{${Z$hqIB^v-st7wTMD8O zX9CFTzQSgTT?iOJ&RKY0fxAFUv@vCj>wyWEg-ll!a}_~_QWBB zh6p}Z2?`N-u!JFj2TPq0<(O29tQ^L~gE!s@Q*G%O8yGwI=qqTGsJ2L0K=f&0DwarE znGxiNm0`{<9W@255;;j=#}W;icxY)_B4VW>=qouhmInwSUc|{km{?0JCoqGED?A~H z>|dG?7@}stgM5n^vMoI*O$0E}1eBdli|DwUxw;F2Y#y+t;1W1(W9AIro)!{C0U%+D zF9RVM!H^;r6GaPxBTf+aRtjN2Ay}rDKzK`6QLG?HN(2=HXM)vj9c{68^59G)U&5f2jXLA|UD!E#gL5cO!-DnJkk^98*c`5<=`s6)V+95Il{3i<+T zVUjO+z-5rq349`0-+v_(U`a+OREb<%Argp6dlgDSI#akCatt8-2c{bQiC_eicLQ?X z0o(}IJ=qtSQYb+Waefl{22wl%hLX(pl@HL|K#++Cnx1t0U?fQ)AoCM~PSTZn6*@sW zC#W0B0R^f#}calFJlSPe1TpFE+PV9NM=BeD&(kuBxBGA*eWEgffs>iABx}+lYRjoAmTxq z0%Tht^%nR*@Mr%*Ba9%CMu;y^9WoMOd)mnyPssJY$~b}?m{2d2q<|F5&<6@qMBurD zA_NS`C;`d;pc=5vgZO}lNueBI2}1Ai0RM$X7(r4Rp*rL=LdgHiFiMC*`X5C|oq|9N z(*AH(8FJkt0gWS?Xb+J{4nc$o*R;r*0(lBS4T3R-_yB0fqX`z`UuXmvlItCi%nm{0 zYDN$lUSJqOwlRuXM9?gFg2_yV7)Tikst_zovJW6<8~CsktP^ksp*{!z(n1m%CjcPx zg5cr&g(i?35YV(#7>}T4HJgV}6zZC2w-F!^Ux^A3hCs87xL&{jXmSyJ+ocYnDNXR7 zmjHNZE)!#FD@*xumjpZ|5LiN57dfxsDzNk~&2tz4&z7a>4FezxhG2}t7~m~MAgIGeXvbS%2{H+Bz4gUbRpXZiSAOXAW0uVl@f-K*cZII zz!x6C^j`r21i)P-SRbtb0h5X5M-sA$9!<6JYH}P|BpEK&%5J zvP=Y2n4AcRd!$2whkPA34|8IDzyw&xmeeld3qXA-SWQ|7&Sy*V7EFruA0Z1?>2;n_4x+qX1sX$KJYIXs!aexGh=#CL#VBVm`@k@${2m~BQ z6XrPAQb+J8qKVEXp&Yz4q6xvFm2yyn1k;z40%FSmh2<7kbSRu86hpQ_Ok4|zT|p=j zqI{Ii64s!Vl4Kkq3%L{Hzy0{Tqk4~== zJ4D(T$vQwVKZ$CWd{)AiG;vlgHTAxV3y&d$^*|a^igg&)I*_}#3QwVyK-OWBnZ86k zQut3UR|Y&L`MpFZ-h$8)skU8eX=xz<*`Q$^gIV$g2)&Z17qT!0We-aokm^T}9S_7G zK%QAb44T9}g}n+Qmk0+Wl*~Z3ODAh6R0OuYR(83F+b>|8mqIdxNrfQ>e4+9h;0~9V zo|`4HFx-f`nYpFCo2;yuriTOG?W8#Pr6Z;+FOMSrFa{WjX*pXuD!`hHi(pj|!#ObL zNp;LoS0@XP(;!`#fQGG_uB8E&vxOD-AMpCH;-Y9AxE8J8;cnyPDj-M*gMt6+c$nLP zWk^5}FQx=#KLa1t&7i7Yw&0L4xWjJg>S}2vD5hrVl%N(!%0sesOVrP!7=ipNi&6BCBTytlpok0T1dOOSmjID6K@y&L1BB^7273t{o%r%UkZM^11Z=Ii z2r!VA6WSPr{&6|NFg&;>FNuN_E(s8ZK}#DM4EzDb56NJlCKMweg`q(}0SY6L!l2*K z#svZfb}7LgC>aoimV~AgDGX{5+8HE;f#0xijDUgkiimQcJ;?1KEeG1eLAMHF7@oie z=r;yh24Ua|3`;=?a7Up~7->1sAjlX@4g=pMpg1vUIq+Kw(&wZwyf|=UAr(Xl1Pw|; zh6yPQ{FZ{`00F~6I*1I0k`jkjAyOD<56U-B1_OUcgY0i${K3Da$pC(##Gybb84Rpd zkS#$90}X<7=j1S`F0vdX8PH}4Xvf2!}vPVURpfjxkCC@|nrXfvJk*4AdUv zQ^5QuVLwQOj;JgSOm`#NW!31V`A_9T$X45~}2 zF$wY~js`hOkqrU_5gmgD!80?6a-c30BO*fv2r(2+BZC2gp>qVJFwhgV zD2bf?07j`Dv1p(QDcgg#43^s?@E-#*wvzzi&sg|K<#Ivj2Nt|yfnYzN7%ZTab)g6m zSzU1~hGNc$W2J!oNl{Rmbb`RGfXYxz6VV_JDPjOaX(-e}S`KJi@TLm_2B~xcr@0H3 ztet|bKogQR2$us5Qq;v!t}i%o6y%#h|A<@=2lgCtRs$IP3<07XNV`f=SCVXU!7by+ zvcu9$TrUSH4 zXbSO!>_040_k-7OB%lBRIVF-yDwAZgf}IWuy#NLdinJl>0_9Hu!vSBFp#PR9Axcsj zisB+FQz|52WKzgGi0?x|81lL}X$m_Lg~O~MxMcr9Mh=CI4-PX@%xV-+dK8r4Bq$a! z6iyP_Lq#A190>#r53D-M35UZ$!DO<^AcjpLC1CfN!mt1!qHM2l6^J%+&Hyoij2D>y z%LRe5rZBI-5l-lo2m%>N3X=;Bp6EhS2cobPCLDl4(cz``mh3+upcIxYfKljgAc+)y zA+YNxj7&7h_)UQ{n$k8vD!>t@91;vL zZYes(5F;ui_8-t6(AnT+)<8cn7-@|}3pzbW-mwI5 zi6~)2wyq#3LFpd=ADS}$2Y{<|EJd*#VD$ z=y# zz6F3Sh429V%j}0F0x?0rL0IID6>#3A;VowJPNXG}wj(G9EEOdCrGe%mfkFHSA!9fe zPu3uaU?T$upd8S9B$WwuDT9o_?m@=?ps+MjnSp1fP!Ro4zbjk-uTn3sPDEB?z(I6` z9D~YOq;Chokn#>-63CSjz=*yD!Txi1HM6z1bfuG(sDKq$l9E!yDx;L84@%&qrIZhXpc8N#4*~^>JtWWd gAB%9v8*_o(UN`6vKD>hoa0Avo9UtF8twVJG50L-byZ`_I delta 66751 zcmeFYWpG^Ek|iu=W@ct4i<#MCW@e@%W@cHkn3>sPi!52p%(BoT3oID<-S>L>_BRt> zOvn4#6;TnDS$pTH%-nfaR-9aelSsK4hybOsxFjPh6DJ&H?|tDb94jX$2@8p%i47b- zKMAwEwS&2_i?xrr842sVM8d|($;-t>!Yoaq`~I1SmxP^_n?#?4S&8KR3le4t5)NK& z77h|-MG{ta5@z-HH5}aU>jeej%pJ`B7$y0yFW|o_{$29BCtUx@6LoJVa}s7PRdb7X zKg=C0-Ky!A%rfF6y8P_y9A+%07UssREWDg7+?=MSES%itCcNhCChR6W ztR{l|JiP22+{Wx2<`yi*X6DB1W;~`G7QAfie?$`#4i-WEKaqEJb1^rzhx5uYH8I<^ zG%_*bfu&e}1A!g>OfrBat{xo81&|c5;!9giLZVIhunb(0M4E$8gd(%Euy!hdXASi6 z$i^ZBJ8uei8Kvgqf`UAv!&m=F=RfWK-xB-(kjMY(|Nq0*f6LPUx&FT;jQ?#T{}ccJ zcIf^s8~-Q%|CTWRw~hSUSVA$Nm7E+G2X#gjf9nl z^G{R3#>w^{XwAmP`*$=mGJ40d?PGHzVLljY~E9rQT| z1z0GkLrex#`5SQe=1XV(S;4Wh@s{o*^{p;=0~716a?n3(_a7{6PFclYv_z}#Si3AI>t1~%>_}D;Au0RIK7=DqeoW))>RounPgM%dtW_*d?F|TlxiDDD}~Z|ig7PG@ZMSd?_7#GI=GoT zxV@*7>rd+5BdPG-o*0Wd{%Jz~XgGN}nRws3FCGpyCT@;D{m6S)^S*i z@XOzrysc3D_cC+-oy^|TaVz~IrCMu-Mu+E8iICGzxOi}+S zgB8)LzxRsEr!~hf~b(2^_7ve*f~;yXZ@A z3s62<^kfOJ|IVHiuB#X%wMT4%pzX`AC0Nf9Mh&d~#Knya0<(Vr0p`tui{sW`i!yC! zArIw`vJQz568#)Q0uQa-F1WhSphxU&!$Kqc6%lOv5Yh=g(g{8qBnX6vMEOBT97hj= z#Bv9z6{_4BOx;L15uPf9VPXs(skOz`AK*U}0zfA;7(g71jYWP54S>$ZlkhB$gTW~` zIkkZE3g%jwgMuFSRrir-*1gI0Ggus*oKOsWJ=odlx86A(fDKubt*-~?@2m9yu}_4D zxj<@!dUn9cci4e>YGPNEhYGAETs~0hLYf#}7z9HC!iTK@0{b0+6AP zX{x|Da)hUa#wr1{Wl&%es&8`a#j~@w5TFJVdxL4MZw%(;(-!{Hr>q-@hyw;@MN{t5 z>C@Q=&XB*@8-;~}y=>YY)&+@UV*DumP~-w17~G82x7*O)yXx8y9P(LDpGUthaf{LA zMe&Bm+`GJmVPXaq!ri_9SwD{ez(H#CA?U(h%Huu-3G){Ae+A<*DsP#*q^jP?X}`@k zvgA%|TsKUWLO&GZ(oLifB!+>2gS!D0{tl8@mPWXAdE05Am@kyG-aDB<`0&Wnp98{` zpb+@n$CW=qEKiIL?}^ffc>#5O^Vqi6i%P}K-G^s)09OyO5?qk^W(IHoA$G4r2^q-H zzF@I|39EZhAOP_CYI#k=Y>R`jdx!jz@cV!*F^yGu&iHrJUavnJ8;epPGBh)BU`RwB zJ_!Q_1j;@%*z@a$m{Q1xI=aBOP&zStBaj`T82_AE0M6s>^fl{k6%nfsI8u!eh{l32 zdX?H677oZ?*u%bmYnlb#U>@JlA1)c*xZ~a&Mc^DzMu{B`g`VGt+rr3kcJHuvRGJ)W z6P`3f1ROwbfeNU6Ax{lNGei&9XT!S2s;_%Q(qi_cKmzJnA;>QXZQEe|nzL7oZ+RYH zw&b8d#Y8@_VE??-gzW1_gm_K3bZ)99ATmVCkdk=a0TW>Q4iE#h$SuLzZ<{Pc0ic19 z1P(i4uznb5$U;JW5x<==YeaGZE1bTsuwfm7;UGk7H{i>C+nK$N>WRpGCD&`W)M#K% z6>o8&kf3^40C|tp;hyM+SJNG+Yp`~^H%MY!kGJ52F_O370T8AUV0#bAssB~x_1Mpk z?CszX)QR^6UJxh^e}jhuY1exTPpI5=X$c8=g#~i3GOD1@3_%lW+F!uEiO9QOd7m4U z9Z{Z@o&bfVaYQ7*&+v8nZSVLk6iPNESVF7#&FxUB#Nau5y1Cx&>F?{6xW#?&eIP1(qvR44!TA| zQX>*SJIs0Vz*>h7jLW|1gdYr9!|g{zhCoEBrQ*l#GO~5#RxQ$@E#_)AV{rX0Q>Mju zjMB(4gt$17-QS!wV;wG9cIuAOr%l%DwV}G@%Cx4rl$PfEEkC(zOb|b1&7b>yH5sek zbOcWC`?}+gh1SWK=IDJBYilRa$H}}R3POpnedj_kdRPx7D|s47$=YObdJ51`eWwUC zoE(%_Tm=`s158v$K{56inii7!2Q?zPDcL4wA~glFz9$@%v0*wKQ}$rf2F#2{pZrz; zWvAKiFXKk5$01VYqMz!KOuCutAdHeP90Z80KAk*rmMB)e<^C|g>~DAzu`>IR3FfPn zI`GX&>t?nE*SrcON`1G$BVffkA^n50R4>#FvxD;f%g6#-jH_+F{#H&xZ*L~~DmGEV z-pqRH8cVXW?rYG}s0ghXHB2UV|GfG@w1@}jOx;Mrl%>{?mpj#%vuI7%c#^sgss%8n zZv9z%OV5K>UXK;-xSsa^Fm+v%WIU%XsD-wf#!fzOXA?7Bw^ z$*Yira?8ofPHMN@_lBS5JV2+0+SBr^Gtb+ryKPVLRUeE`HRih9T9 zQ#0RQs1<9AEO=w6=gu_<$P)q+I#PhiYC;2wm4vGgez!O*FSi<0B>8Oa zSW#i&?I!5q3o*4|&RNFHNfiW3d|?^*39Y}k8-E~ZGhj{Pl_t#{NFtDLW^4t1J~@%S z^f=7AbyZ9dFkYx;a)jjFfaFYI5o}ZhpJ>N1&g!>!@dsRwEBQLEdpJowxZe#0oq~#A zrVnNJOoY-BFg>azFN8asuX4YBTcB_adJZ$W7tX-2!RG>ihvDW~up9km(qQ zg;cy=F`n{%7mX#m8@az+#P(xHxAd_lzugqjiu+iDdA5g?A#Cx?oeWWW#s;Ar$Kr*x zh)3{r}1MwWXzX=Zg3U;SCp2`f0M zQ?xmiFD5V8H2(h0A#f%D#1o#2q^h~<9>jSXpj-%@90rY0a#gJQTG>Vt-%W4P7a*O2 zwm+EcBTT}(Z>m`AaD*ZC424s~K9$vI>Qb4xYH}m*LG1ejTZ8)1B30NqJwrHNl7e&e zR!qdfG!!nqoFy8!*>WX3`R)?j)gifHoD@$j$E4N^v zPUg6GS)GFR39lGNrb90vN3YT0dLo}nxv=Yv{Fx5vjlgAl9Ae=BEMcJgDbRPugW;5{ z`Oth62EjD8G3*H2BYAW9v1CmuXp&=Pg&QIDrOu5xtC_hFn)bX5#l^U3ox{dlS!m%m z4_D}=3>n^ioLdGy4$dWp?Bb&(Aan914LPG+H*O} zJau;Gq~YF9Iv+GE+ozJd$@nE|;mhC+V%6b(+!C1pn%~{1+ebL<;c{%|+6DJZ@C$Kg zg^z2@P11^eiD&5VrtIh{z;zeIU%+C$rxMtJ;TU6YR3bswf#0kKAqXW1J@s`v^Bao8 zqU{LOTb2yeFxabEV|TUZQyQTXFaW`4ZyQ++Z}j}QxA;-^^`*qkX5zXzu}zcjr5-11 zPUy?P6)AmqLPGSnM%kC0$SAfKxW})m$d`)H{XbSj?2S6YH`=3bjeI2iyF#s53+jC@ z+`6&pG5Jy&i)?S)(jr@)no{QT2ihv<)Vr`&bG^UZ7I-e+60T}RI(&ovx{bkd}CC=~^j=|(l zB{1;!AQFRpI_u>p5+hjL~W@+#eW@~mcNAAiKlM)>f>pw`$ebvj$! zcZyMN(m4Qz<_l^6p-{Xy&QYXXvYP!icL+1hIe|Lg8Wg6c>&(D?bjZ(p`;kgc+~iF! z0F160xl2{h=WtT`t?_9!k874M*U;&H+$W4T^drR)9+s&+k%}@&jYpL=E*jx5sDB1Z zfbw~Cvek8U1C0xa$u}N;L^_$8p*5w3{WhvkW($cU+X*8T3v(@9C0ar4p0I5ySq&tc z6UwoIp2?FFI1jQRQ(WWg)mSlCCH^WmK-p`yIAYmzl87d~_RS)*cXr>za!+R<9qDVd z=Iq3aJaiQ5IvuVS!*c3^xMSH@sr=+?#fAoatnUzzD#Huh5%LUq6AxHcd@lyw<8>Cc z&ADkTEjkMTIg2n%DmQ_XIq{Q(ym-GVXpbn=BRI$L34NLCV*$ivMV~0!^L=#}08Uzw zdpeCo+v)I_C++eH8Lh@U|7d7DLr$u6ayl#lORH|2Lb~<_o*=hI(_7l(!<_>Me=OzQ z>#CFe1XYRlleFV7qE+5SQ~1&&#)h^jwL=57g3RZv`bj|gI5Js(;KrdN6zw6O0Hh`1 zwzAV8M^decJpfjZKfkbwwUnS7h&9~IrP!}6n#?f!HKsa01}D2+Kdy$ z*(6#&D<3AR^ukuy$Yxf8N;2OWp>}CHtSZURkqn1wx!X~83QoS82t5M$t6WOLb8ne` z<21|(srWRH{cw2%p1(ff=z7q)v*;_=#}E2B?eNOFmwft-_e45LTH)wVQzU12Yk5W& zlyU3dI+ee;w<9S-1hGaM1SlV(oQ5r{FC)ZWNHu65&*f%DFiZkAkC~^~_ z*|(SJbh~8GT8v1P%(&X}Zio$PKjWWO82s)jz00SluY)?u?XHg~K~k-f>qY4Fd>*zN zRtEPsg-SU0EcR^j90H?!IF&O^Ki2rk(V5;<0s$`Tpa=XYvz^X>1d7Vm<09?-3Z_H* zusJsH8VQO|W=rsQ2h3WZcSoQ~b4`mT%5#Gd{gGpMuV0%vH)3i<%^m_yIXAGMG`A_bh6 zm9?zM6Rbjw~qR(2FW9ZlWIqEwdc z4mf<4m~-5yWa`nParhwsk1#%L$@%eTQL!&oREBUgbI+9PZV^8F7rJ)5$S*7i(c_fS1yQk?cN>%#7J&HF$?=GOOA!37y&uf9 z4S;CB3sQb9WrKP_;EF3j>aLuEPYDTWWz`iFJ$6Cif$@8|A-WsYq3WDd=ws6w{uyKs z<$^)A3%Z@tFt1Hlt7oWDxP2S03GK@KgqikDQu$3ireC-F z(mRR>tT0;(Jt{h|)#PMBdhpt+y=(8pLjZU1K*3Sh;RT~XCak@*+HiEE5qIi(J9e;6 z&V+2Z4qRFbx-efyqQ66Yp6np?^cX2WU4dKEb^udy9#G6~$lv&9xc_7qk z!j3d7t9Nz-$I1;2sDK0Mw4l(exW99~~weK!-feMK|hdMNew& zVUQkUUFSP3@|}C5VPgv{(gy?agF4NP)LcRq_?`GF_(kybtHt)zF{=hUiTjj58?IEr zn)sg*e-B-Q=;oe_NcybJ_i%ud@HR zRu9x-hQr(@<|pW!e`Nd=U1DKuVstt5 zP{j-=nF&Y;Mf$a(;j`Fh8S#}MI#bGg?X8ngtvtn)%j1Hj*)&$`SOr3?BD@NGa_#0u{_)(+kVrIvuTGfw5x1~Ke*c$X}auttD@km@3 zl#QbFYA&ZQ9jMc|`fNxh9k%1T{wk;g=IDi7|0-7j+lcR7-D+s-f%t{!4KCA@{o;1UT3lc-KSG*+39&Sj-3_XXm(f$Nf&_x%F`uo8uF-8 zfR+V(+-t?-`@!Xr%QteJ>kgNU3i_rZ2`!{+tMs3Y_P z4}@7~S#Y96awL1JQc^9_P;{v^-7^5wErKb=kJ^i<8FmrsmFRBK1(N|X%q*%hxT>;Z{RoYntJwNLTZZiA@4Qt+#JMRe0hON*NXgo*}AdWv;fy@RO16b_nt{W%U=&r%eA;P<`3$d+_jTo(2U%|tuWUIBZ1di#ks((JEpDMI6#wykQ@aR^Fi-RMyT*HAC_Pn>15i*KJht9=#&*4);ahc3gnCD zUfBEhIFGttd?+dPUmG$eQfM)V)2E}Wuktuu%UiSk{lu_6LLaD`rNX!$i@VtKGms12 zN-hg2Y2?GTw2jS0E}}7fH`PYqkIhpk8x=m^UCt|+(IaKLA(d&jeyK?umQ^jkWSevMGT{$$#Vbb$d7x7(?T~u`5rxW9N2QGd3ad!@ zHdF?7NKA8E>d~@F-Fn4Vu{#=i6E6k(xGj%xbuYqg4tAA(I5U*FMN2Z&cs1R+tvyUr z+7Rq8+FHx#W?>eE0_S^-x-kTsbU#XFD%rnvd~2=!J@Neg`3FA9-&$J~fkw zB);Wl2lG79HDA?rAe`iyBko%Ec?vpaf&=w*FsA8 zl~vGVDl>L>XEQ5FDdN~qv#0?;19iX9U3f~~rYticRtTkn^l^86i^Uh+uAc)x5XzI0(j^@$x_pNg6~DD1xa07WC>o1*n1IPt7C z9bDImi9YM^-tOUhPpq(WInQpRLS&Y2m83}HUreFq zaLZGIhPlM4zD6#grWvhsTVJ!T%^ET0^$7d7j9Lc`72e~s%uv-5qP2n9XAh@l#PmBj ziEd?t#pEpJ%%Pl35LoXNMPfUUQ=FkY##`WXsFJ3@0guyEJmV*m+)-TC9h++Q=J+2a zaBI_0(Uy_TI39XL7vDRV)ig|UR*1MMfIP(ha(x_fTF~;FJi{1Cx}>f0u}$7rhK}mf z*wVZzbAypSL7%3BIe>Y{fMRjnY+H?L+9yg19rdlMqg6e6J@#?6&8=sCTrXJ(S*&W57D=%|x;3e>kIttd8DqHP- zD4fMp5hYM5kj&DWt2k8uSy}s?%fonoRBR}8{Ff@-HX6q z*k(RO-9x%>eL*Sbxa@vWpURdu!jdUdnbT`Ym5WqtfabC&HlkHHp#1ZqyI(Qc4(P-O zeyG`Dk`}exF0`Tw{Pw<|@~Y-uLq zgJZeQ_7*E#{tVt(=}%jVbqr#NPE>`yfb#lcmNFiypS!lf3V{iNl4ba&mRwKJz>8m^81rlv2)(FeFc3?=J*2Q+{gKF+ zb?^;tWNVSqY$e{Cw3oN$Gs+Ox#x?jAQS_G%Q~YiFRiFO2(+btF+e*K zv;HhY6%#OyRgrRl&V)Bq6^9|JPzvY4F^@M~-PZTfMBrMSvzdKhe&U-G_Zt!eH(V2( zj&-ru=C_O;s)?A9GG#~fl(3M`0Hin}mYM6b{Do<3k&d^&B%=WQ(xoU0C!YHnC+pK( zl;a-3`nZDoq%n_5e`jbp=l099W}S)fCq4#EI5cyS->XFg<}`XU;sHn8ek$y$cQvGh znQd%?FZiEQp4NI{^s--KdK-Ni)o9~F5%Ho>G=@}kJzf*57;tL@!?xO`f!9%*`xM;- zzU=WP+PZU~hh0?mrRG$#bE#~nI(hdTb=Hrs(wUsw8UJY!rn~)cJ26W0Dgd^O-sP+V zRK{DNT$ubNb6%x((0*2<#cU-;0Az(PND66DFZFJ4==RI)0r!HeS50i@&n0X1`KiGV zEArvr?QmrGwBhW$Bcc_-0OwVfrU1TiX^G-gn&RyP|BG%uE|h4E&9H@88n|S=qKm_p zFbE_icKMLmVNRac38lPlN{WZwBLlX)MOy1CRUdqFOGY~L5?qQh@B;j750IS-~FK@J><#5g<;05L+U7OdffpQDxjP zZVa&`X?EfkkjY4`#q#wkwZTWfVwBk?=unP(Aq>q&@g@1C8#c*OS1NV!-1bTLi}4#+ zh?U$J?QiZ%_}@GeEQHnV*i6+*_lxUlF9OvJxVyfcxGXXqKV;;9P5<7$r(zT0W9id) zJ>sO*)pb5m^|EmeK;|JT9;m{B7JK42gWgJ6a(~4dj%Y4E6;QWr6rWNCo65tOb8^?l zOHiFwxFbhgM{o_yyQY|bnH7fUCYQtF7%Agt8y`Dx`h92aWo%wc7{J0DyWo(R083OC zRcTJnY$PvreoaQZx)Icuv@$=SHc_1`TX$!P57n1=*pLtn#D+p~!~}pCmW-|+eEBWY zl#p9HktQwnk=WREe`O`(SEJ@jiK<~bzq$y#7IKBk+^JZ)uT>J4@KSrL2o0#(7L`(g z7I(AYV=EqJCBF0;7lMIFxI!po+`6-}7tfs&m^mjVgVV-0!%5x`=;^NRmqg|G-oQ66Lwx=4Us82gsyiaZPKQyCsGoa&>D>4XW^0Y5z333ipDMJ; zPBpa$tOu6g(QHK3`;`4`!GNIclKyhvDv%qH1)`((0Mt=fM3t5sTTx^&Jge`IO40{$ zjo*}+^l-7p9^b6cih!RYON$k>@=<>o$vEF}&4l&yshB1*D49qQQ$O<*VJROD(DWO7 zpM=SeE|;3pBGTTJ91t-k{`JjGq?OJ{;3fTxgEk>g|3dMj0LhRTjQz+5Y!rcM-9 zQW`VDDxt`X|0bK9eTqp{!hrn1_z7OUibI~^kq+2}G2X)9%=mo}I`qlkX0BF~RS@fY z^7uHba`87(^rBLC{i!{(%+K31>}=w2Hw{v3n{C}K6+YAZq~iUiw2y)TCn8-WHnIkxN13$_3b@w7VuA~`AH`4!>8x8WgZB{B@R{ z6QdMy&D-V6olX5S9hNHN8Xib)#ZN*R-iY>`FNg=AgJ@SD>|Xn6ke z9o3ANg?Es{>_t5~ZJIHjuQyF8o?ZX&5iImeR8Ys(uc97_&0Kdv)_J|U8f-Gaq1$XJk^Aa8y!Ec0fbO4oM9w+aiu>ZtH{cw#Umji#HQ zt~-?3S?NFddE{N?#fitj!RgD5onGKdu@g_JIu&NDY~(v9fCbf{%9yL@k7}?-w;-32 ztCWUa;Q4%QOaWKI<$fkhAyQZ80+r)1PTl$aqm2uX*m^g(lttF?Jd^|?VBq^(Il8S3 z>C8kowu6U;NxOrUcv16In}8sH?o)yjA*H6e3X%QJScI?H99JI|^`d4%AhlB3b;im$ej@l}EVFx;COFLYK zFG{A4t=Poo8af6@alSsOfOk#rM5XF%y#4&JKYc#WNAg3oOu3?zqU#?X5l#*lvQU3i^wL4zR}jMSh91oqO7oDbz{d)QPaE(_!Yfrs&T zcKD3vo<|9J+V>2N^XkNQGd=4 zKFA{--Z#_z!e1|Ebw}g9WD%N3oP1`0jJ5fUw$Xgf#=qzV0E=%04;xvVT(ol;FTdgC zR6jeLR`0f2bLUpD@X(O`&4;if5Otd zOblyxSXH7&p`|C`!M}$2*#17uC;G1`y+6Zz+}!_?*hqLe-luK^y%P=4)H? z2jinfWaauZ^Y?$66C~m0VENk)`;Yw&yDS*pNA%qhvgbi2A1H$0NLOH!c+Ohs7HsMg zGaV?T3n>Vm?oY4C+wFWo;_^nO*nb&a1a#X`SK9ruKofNNW%G2nj%ktny57>ip}&Z6 zlu{94IGkEaVELQ+*DUAPMT9dzaKkLS6OV|&TMc`})mn1#p2B{!YE`WsxeH(isJ>R9 zyEF=Ejucc<6Rfw{@aTN^oSRr&R*37M>{VFORCpO0B0pNkT6E9iqV_rt`5^(F9Y2or zQQrfb)ehM@D6zUadWX$r>+2X16qcjVQ-2QET>)&m9G8Df%FDDrPZ%=Lf7?N?pt*2) z_9~h6c_nx4msNi7l~IdH3GZ_F;_teRvHBmLP<5$o35zU0hJ)2H6|A2;G&3DEd$Wzk z9-|p={4g^+pBiHW37Qvi7AA2Fot$~Ma5)5N^hv= zQK>a@u9s)YDz+MeeE+`(D94;KX)KkKWFdaeIWv4 z5L#&1PoXz`cL>Ygi4}W1$ZR1RB1`#Bqhtl0V7YTTBhH&%hdgf3q_$qD-?JmP4HN6u zug26)cx-p2jjnfor3E|SKa~{QP6TEUX@TQG&wP3o6-!OYXb=JCLGJYOe4aRx@&67z zoYx2nWa#RF)5(_!D)HTT&)ViufBwy1vVKI1DX3I&>n&@!^ue{)?_O?XE!6Y&=>+qr zwh6h=*KR{Bg1WYH@=m}Z8>?!4L|}h!3rcdz`zJDeV!6(rDoBZzhX0R|?tf9P{(sMC z`2XjOD#Jv)zW6-DFs%^@?;Ty;#0q3~5i_4A)vneMeoe^vva05LA^TxGz#)ESLw_^< zD8&BBa8aWb&q7J*ti<+P%lQi6-#V_Tupz{AT}gX?G>%k#DKp%&q>yh!?D1yt7CSCR zJ#Xb(d+)yy%hT@Y-s_ojS(;VVkd1dE7qE7r-ZqJjo{;0Jsm1kik!X7dBaWGG*-N># zD!Z~P(pXnICe-Ys{;!OrHzwo@mU^!j=aX~eg|k&FLCZZ&Zts&WCFonbf+`8pl&&yO%}^)J&+6q>wvoEPs7%GeD(Hv824OJ?;twmF zw5$!Ccs|j~S0co<3nRw?L_QuHUw`0Xl>f*tfA0FhwCc@2$QD1WdX1r_N(kpAzYL-T z!yJUZ8U$Vz>Yw-RdjO-GEUCW|TT_~Vyk_7=GJ#bUh zN?At?QRG2m5nGyYo%Y8W$+fJ|8lgKMN^|jYGJS@(hukEge3IoLoTy-((VUVgAH%|n zNeqv(Tz17m#w(C|$O7U8qurjm=5yS3`CDK*X4v*6M@YGS&|HdA67fHWuUlcd6uFPa zl~!9F&tPBX=0yPfKc*i09@MzDr+%*S<6dzpNd`=&M2Y22Q0$LUkSb9jdb82(kIBC) z81~1G6RAx_bLFwF&8`!vn-IL&Bu3SS#d6tDuT#eo%j{gfu)lq~?{(`|@RF*lGw{CP zyC-=XSrl^f+C3RI*xEDo9in@?LVo<&7Bu6E`U2kNS`O;^7b^TGX62s|Z?S*f`QOQe ztlWS2mH*%+z7q>MS-IZXlz)kToAKu2c%Kvh&z!zCdmK=G;dfa0Ph7{pjgP;>!XMV+ z|7HRHUtz&c7qfYlo}v^V77Zh~`Q`WB1Lfagp;oHW&cg!Z9TqIUPnIxF(7bdAKze#= ztLep;UgXBnD)3I^<}KT-RtE6EKzFYHg)7d)L`0^$VT_m(wZ}=hkmPAr5lTv zRGF?lMmPM15T#6S*>YB29fXw8 z&S3-|QiLtiJN50*d3FfuV)of-^gGnPEb*bJzNqwlY`nr8zXHpNC}cz#N%x=VyOzf( ziHe>X(J6{KWoUxjvQO*f(?^mDeh%vUu09gt1U*mEfJtJR)Svq!X_^)4q~t~Zzo9nlSDPn$ILRu_9BFFM?j33J;kxt6dW=X2`E z{nyn55uQDbZj+(kKztl3P5x9FHP?S;KK;dOQ8KYnbF=?fL6LB9@pAu*YNSZ=&Wd4{ zw03cIduLW~{;g8Zb#-0w+mHf;dV>@0culVYvVXV@dz|R$P*p!7cA?}| zcsfM}aVa^tu&F8j;eFyI?ka-&+-G(Rauq_^I2^|bkdfJQ4p`RIOr|C zBCogDL@jW9SQGY56HD+!ozkuOR>^fN&Tbb>lc#J1FpM1LT3W`*#Aj6c*}_F4)( z*V8_pFGQ?;UGV)yzc)2f^|2+B^)>&Ar=whvn5fZ{a|N^>rrtpbSLoB$q=ua#djP<% z&7j$ASFv}*DzI{U+x*mASR$)5H4+0SS+IEPC#bDJ@8;vPE)ND;z`Kf3#Hxr<<}yPCJ>Nrq^XENr|P_dgpRL(JBx{&jQ5N%=8eP zuN8?2`D?JG7Oin5ejSKxG`yauN)>NCCyxn3T|lX$ku|X2Hg8}lB>KmX@GHS5s2QJu z>_Ikio%6n4a@-W{WOX5lMRuSt0`fK#$|mEGT$@Q9=?O)!hIt5>tM+&ic7+7^fM2(_ zLQ>Qbb>`>JQcHV~6j(n22nyuNgRdqoU7wg~@U=gfkOiMb=WTp3^zW%LqN6dMe|-LG z=fc`s?L4gU-@@XD*G`dw7WISKkLYSmM zBEM)*liHF(IS9qfOh42Fnv|Czag!;L$u4~$7(7LRDPx;1v&`5N2xe`N+00y18|exU zSw7U88qr*6tUB+%`WDh=IYv6<|A4=sqKRB#yC2wEK>SMW7_VqZ`SA4%5?+sTF6pg* zFTdrH<2&aAMVnSb0O8!&yl zNGWi9yGbpeCBy7B3*LiUj=pK?2jBEvfxFE9$Emz4YcEfgh8r5-gaBu}cw1|U$nSl3 z){k4DeYU513qU%&nOmKZa>T?zG*!krvRAQ5Cj5+3$@*wdx+s!BHZ;^5v>ct}+?cXn zLb|1|m*5EclC%!af&=gf<)x?~9mRaJ0cC)E5%J<1@X`?CH+g2=D_i&)tvb#E_i(6K zj+#r|(m94F9mn?AIr z!y8>-pUvf*NDf_W)yhsnBB0)`96$cTCkGu zVOWU6DbPnX5gu4F8(A9u70P_5AMe0@(t~41YzokhL$UwS3g|QO}2$dQ6SWzq$8^^vAlQ~7*3o7b08Z`oL?Uo z=rG+O`7dudw|v}V41{rnK)>^1S^{mHhfxk!d1TQ2_iF06LH6tq$=UhaUu+e3?x?6DFs1&kw$FU zv|pO7aQlc12jRt`Ez(2V;>5qWRxpm|$o1jZBVeIgPZy_ems3DY?<)J)U16OPRhMZ+ zVgZa;NeQ5jePa}4;O7$(&Ak{zpmG!s!&}gJFq{aiMr3X=^YD_e-a4b&;*6ka28sR* zXndla3tc+q3Vn)|O!$o9&(3o?vBe&kD!u!RGF9o&)H~iHWMDkly`Bl$L6@rEwgEkI9}f242ityi#6)um@wDoFFPrnj3H58! zkx%d4h}B#|vB2rgnz%^+0r5)YF$351shhcW>aLmd6OL38o8ZuhT8)Doqs|^u(n-Zp zUFs;YXL51rH+R;bM?Z?`N-ziq<|WABB?ijg{O5kfG(N^mW4R=xxF}PIV9fHa(-Z*V zdBH*u()t zBaISxz3g%hcUeX&_Qtg*+PyChw(+dDaUW!p7*Ykwg5a;zT7@2uTHymWDdUMR5WEtqG23!c9v)4pfnfTIa`--cPVLe^ zNUvnMK@%j+WL_x))BP zevvDKNkf=DR)8z^q2m?|60|@7YCF_*98aaG$(@T@#N5x;?oy=6(45{m7N65pvvh)E zFpj_^1e~z3DiVWblA?X>1gA#&oz&`>U)n*m=h)G*b+#a>?|?@Gpg$H9UOqP29$5ClwQ@BV{?V-aL~t+Ss;_IrrGI`>H_qpjr#4mI9Mn%t*G13VJwA zU71>|YrI(k#tx^7wcL?jfpu}=$53cSyb{|0YqLXzBP!d)?j$(D#GsD5kdna@$(_bl zZ&5BuT?}J=wWRpb(*Qg`#2#x`Jl~Q>xfIp5a=z@xZG@C++G%zVg+pZ-+GJ4;DR=+u zvTzjWA@{hrEU42v#GSm*oA1ss%;`Pw^&PUc3m1@P83Z%mVM^XD~vDna}H(J=R0^8XCqnV4X@lDrTFdPq6^tdp46G(KdXVD4n_f z7isSlX6cr+?WV0tS2`(zb2ewr%cOy}JKi>+9b8+W++(&G{b8 zcg&*^5l=jE2MBJbnhKuI4YS$J(%g?Pga*54=V;6g!||r(t-tmySrg{Mlbn*SeBT2f zFwaGuib=keUneq?Ks#H6(nVUr+-%66h?aL;1k-R$g+t^yc#LD~z$_EoMuc+Cvnd8; zN&Dw^>uF&p*(NhOSc-{Y$*N~a06c%RxQ|@aX3?w9j@G+Z8}x~o*VIPj>$&gxRX$w( zO71O6hXOgXqF7kcMC6=F$ME#cBcH&|4L>%D9GO>asj5bsm=($^j%7i0SE@mImoYQ$ z?O0M*#r5ReOEN^mYAld`AIQadnin2YUb!O)kmpH0MLFue0nw>#F~#J=0&Mb@kg6NIS5++98a^X(u$MZcx+9q?UeKSKjXvab){FuX@5pEv;Bn^z2-TNt;;o&$7l z{h{!c#>u<;ET3I`QoPDHgtl}{|$R}`JwOuXr1l)Q%y;!0d2;_Zdvc_O+$bdRyQA?3;PbI`UtP$+G_|Ocz`UxI`UkytHB$iYZHSWvcblXpBC`b6>IoYKM!>KH!G0v_HyjI%MTgnC;@w!vp@1+~(|F;TrPmmHK)k>x~@I#k}05VVv_nT^N4gWQZ$!uf*s5i$%HJ>GPk zbtc^c(D6LTPx*-{vftW%$@v(`+Kb)EXtB1n@zz5I@9Er_<|kP;_`%(rT4pbSOE7_Mmv)CYx>%gXw5om> zRm2htE>|ciYucPPRv}_uXCvv09l-dRRK~#%hjT zNSxp@gYC$go9Z=fmgwasKQ}m+lIhDF-4Te^igs3AoPrn>;dw9ZKOC_wdTDt_dwaG6 zw0aX5KuI{gmbW-h(E3)auQnXX&!Bj4vAFbQAdkv2kB+OB2t7D)oQpCOZfJhNNpvVt zMMpTfh%kKnJc$)9$ifRm6OMh`ZO#=f#G5@QJXYNJc_BC60AC=oj;10{HBS^N$-^`A zG`Gvilb?2(CY5vVg+UTGcBxMvxG`l3m^xS8Abrqspt!@?R0^?0)T(9)!)?*GK;?2I zBued(U0RY|k;i4yCHjWorSIQ)h<}uHzEzQ9foq?;q%`|8t-6$S7}cdZ%1mPrK{YtU zV38AIK0=FSSkQniUlqBvwTn7*9iE&x9<{ZMaFsP0n;!9tUg)jb6|?z|lQIzCax z<2+zMWP9+7kS{Kq+`f*`cvmW<9|>VNwU6C(*N!Y-*YB1+4!Z2|n#wN^69#$^U`l>` zYsfByQ>!=3j!uMdX>9UM@W%tGG52J5zRzK*6mt6{DOx)nDbt*R6!cVLJVoxHrGy+u zzbl4fbFX-JIye-M01Mvh*0nWZ20aAEK=@7yA1ZlUp{q(r zO(zVupL-$Q+V3!JdwLphoDeDca||axd}%AaCn!blfnnVu(YA@%(O@_PbhZLe+I>{S z?NHtXudw81MENN=k$iJ86;K35wDskfrP z@0)I+c_m0Go*=d1hl7B2=Eenrt=)&<-Q%bE1VC7@zzX2iXS z#)+#X`gKu=-YWxhDS>Oo1F+Dbq$ml#5V|gRgM-ru5r%e(DjgK$%V%;zqbX*#m4o%m*q|PU{^sGs3A3<9cPT z3=Tmi@@y^#-N#Qqo-b4(a`Hyhb_ySh{cm0=Rp`hgWGj<*>F@CD^I+MY%&4Q=bxxjI1=(NFiO_?>ytSq`cwG)0tRVebD#s)6-hF03|)GzEwY_S}4~{q?c|@sYJeu)`j_(!h3fGDJVq}p_f#?Y@7*! zV#s=UY3_%#tUt@nHv`jUm*Kc&BIcDRL{@VKyd2xsC>Ux=pPBsFeQb?aa^&XN3Zf@4 zLYu=;D9!(p=J9Lh%XURaqbFW9`c#(JD^tS^bA2v$VkQ^l8UV{UZpkQ!+4fy;K64#% z=dh&KG{%gJQMY=@0^xi%beJe{53O1Fdg(R#pE#idYy+QErK1z!%fc2r{$JrL_L z2w#bHT@n9)kj{OU{O$%hdY>0h?T5z;q8o?t=!ZT_jr?BNv6&Nlz%Gcm0Z3LIQRUZL z_zW3>Xv3YT3P5qTcxy?Nz-57RskhMm)80W8e1(X+i#=kE2U{wavJ)jXgj0I%^@$0P9j zLPiKtV*G}O1dq$?IBxWY_6L_P^F2&f^{qf6s$6-%}gw)32bs< z$Tb0dOSYV`;r27<$*q4_&dtjV519}p7Pbs|0k9A+$LagiKY3E-8%R%@HNmL8uMnzP z&O{6AsvLl1SjJNwZgZ0P;a2QfO-IWBe}EPfL#7AF^s#IpkEc0%HwwfVJ7!o-c(dlhj&VURb0U(FG`x_=kguE_7Wl3Iu)-tGcL*4T z1XRD2)6#3QZi;#W^CQfcdYeY}9{?G9_)Q+5h))FffqR1aw`oJ#uDjhYw>3Ne7(3D2 z4dE!zY4$BWf?QGbvAMu<(at*43~}LwHh906a`vJ1ofeAehOsS%IVw0JFyqzYkk-E_hA! z2!jvHv&HtQtrH#FPQY)2->~I`MYixI5LO%&h}VpfsenSZOu5BWB?zITkgpmiLxG38 zThQji8rp_+pB&`K*rJ0H?ug%JI{JeB_$fQ1#Ys5YQi$H3=th~1~Q94ZTC2*_SR z6Lvf-Tf!uxr6_z1s#xyFJ%;b!6+Z{v6v0KGFh`X+f2Exjq^IOAWFa^Ix(;ki{e3xa z2X?5=$eP7LdxlK8^`(kQ#ZN5Fa=H7M`%0LY@}JeF4XH;nnO@a4 z0IlUsA=shDtew8k+q&wrGSN?7Gs`J-yIjr88AiDZ3zdBegL=n>PPU`L z55!anTKqo}0ja;F0Rq<6Hoxg_8o$|Xf8zX@e#623z`8Lp{?P*dzJ9uW@qzdN7KQ%9{=*-?WY-C$^bf#|!T4KLSc7MueGg)bcy9QPP7 zh03Zi3KjQ{%}(Z!Q*=Ou#-l4`OoTx=aqN6v<9y`X+2U?(t$(#%UA(PLK7@lzASxz? zK2Y%$lg(9&IRazo_j{&%!U1$p+=RqdF?Li>CqL`ZO@c!+&p=DB8L)?-_!;KDp%90$ zplt_xDyYc8(ZOImkGyYf*Hyes+%hwbuY%B`5Z4DiGEXYt`|Xq?F$6K4igRa)jyG4ll)yuFD+BPJ(ZI&M?iute=!p{xVPVUexP3N^X@MPc#G+3=A z3KRa+a+A>Gb6Stq zvYtXu5}Yr2Kpnt_3lVaf7kZoZR5<6CD)njg+yZmOQ*QkRmm*3(JBDlx&z9mWGhG^>c6`{z$&`q zI{KQhhX8$vkZdhOZ*A$~NC=M0x1I$OPGmR;sWxR3`3M+)^W8zWuF_~>`XNfk+=(^N z(T7P)>IyKo$%em|a7ix+rKpTdUb+xNut?`)vJZA<tZ}SAINy|^Gqy3&wbbMc0JzVeFN~aq@oleX zL#h~6cVMU|3nB7GU<AdcJaQ=427f^Id{12FPGCD6P0^s-E^G|Oh-M=q7=6_ybEPq~L zOe}vwEB|_F{gZ6^zd>RCMvc-F|8evE*IVn41OBV)@;4;&@6^R}Rmu1aR-}$sm9bbm zBl;Z{+lXD05>02LL*v7r;Kn&HQ>cDUq@BF$C+N{%7wlM=)b;h ztE8c|yc*KkCN(Yx7N>{!kha;#2+ADPjEg_9-{sv%D`BL)J#COW%`QbROB1j4g%6MC zlrc+%@Q7ll=@g@D=gt*Vt=D$Y!6J9KeiOupbAJA$=I{* zx+bu_i!lmZ7|f}dTqU;EtizKuW~BSh24#?~fc|8SWj_)?g?^AeSrvjmPQ^$HI6$G~ zQ37r&)T>*5_HD%=e16}7z0{w`wHNUN?tu+}k$cQ60Z=O^&)9n$uD`33OSxA=1Z2y z&jerDdVuIW3wrm@n8sCHoNq=*ERAD|p!7^hHKb^lQ27GUAG8Hpi^O3Wm8wJbum@kW zRGq)kC4-q%(XK3jin&$0H(5`AE-NGTT2X1#*arQaSvsbS*WA%$nMq200o}=wm7aDv zDp41{Q^fh8_>9eo4kK4zJI^&!)&^^=CWw9UwUW8UA&*|tLF6i@gwaE+CfHR zzNMl{j2`PHcTpE}UZ^+|<&OkoDC~?Ph;~JSVRy}A-_Xj9859XEO8-xs*pvP!+IiXA z4|TwN{MW?zU3zWc3>+m%QlM89p%#!usS`g9WPe)64rLaM;W7q+Ad3VFZqi3g?`gE> z8o~Pm0tS9;vpD9VmsenPJ_y~+mh$i*XTQb?M_yj16fZ2a3G6jZEBQS?SE=@3S#+S` zs}4jX%a)(?0Fv7yGS}TOg0~s1fOv`{cSqUjbqc~>9$I?e%v zXTe>m=H&VxZRgHAQ`MNfPFPDUICZC2s?Y4BF$V)pNe6q<=&p~pexKB^yqj8=X?c}U zr#b^o(=LJ*;O()!Y`dhv?JCf&f)?>Bc)RU;K zNx!{A%U;PEVt3HEyp6!_esWI7k=qFv`J(dmB|f$fTL}@xqNT%1;+>v_87=g!dc7B| zT)elftCN_9H#Vs+E-Cd_7AuI&?^(8O$tldkvv}zvc?!lplayHd2P9}iNYE-CXv4yBiHYs+N$)j$E3`lgOL3hfIKZs@8+}ZvrdOvDZV zOGejJ7A*M`x_h@$U|cN)g1vq@1SgyKE{wYHJF2jlmI*dpxHD>Fb>ok|I(>fFeV;~s zEG+KOH(F4|6H{24-+Zi zpZhuf9VGt=_x@{8{IeeT-|(S-i;Mp#2L2-?BK{Ez|6eVqOiaK3&0jsI?Z2ILtE}kW zla;co!LnRmBs9e@e&DY#o7KjXt}l5>aH5C=&_q$lkrX#=yx)2tAh;&ROz%zwv3dI^y$^m{S%-5`2QnY~1 zHj;1+b#6dwuTUY4lThIq$&( zy2s~3GZY52WCT5Xv~4wVoSh(3)0%FlIC9zOei&ryuaH%}oExnnrEToJd;4EK^rkCx z4Q3sAC3>}`HMt`vRRE3e!Os_I-i3D^6OEEzz#&~#@};#emh6oZ;Ix_N;DWoIJDxkl zTow>5KP?AldsbGZI|jsUj^=Y3?j75_Rv#d+SGf4PXayaS5GT}fS%7csilt-FAt4)k z_AM1~SmO|01kcd}Dxsi?8UZ*nekU7%$CQ(kH}~MUXAFb_KL8?l7wo9^?uyL`%KLBw zADZQiyP(o#r<;^g-bH?TYpG81YB@>Hydu-_-7{m7fEtTZ%PopU)xDNpc80oFn$_({ zD7d<0N|y>f@GnO;;DZj3`X325*$6D+DfC0sBPPiTbgBxgPM>*gxh7$V8^e&LvZY+_ zy2L@|kig zOQp|vY83P))6WoLr}mV&wr4Ztbd%)h57$0m>nrH+5GJC$z=R-@YPGV)=qhcJG&}Hl z(g2g1qO+ZoX~4m_?vb;|0-i8Iv8kT^6>sb#=cK?_V8DR}YW4SXa1BSKdBt1Y;Sb#8 zlllI|4T0@8J4+QGLOmQN%;4aloIF^w5Eng6+U8Kd${(%-S@d6>q=!Mp8HqB$&27)y zvBnB0j_)CnkW}g3i$-_ym=4aM>)o9M=rUfrAh2cK-S@eX7uqGRQEbEKKWWqAG6cp4 z5M048k^%klk>mOnnZn=5%S-*ETz*n0a`B%fn_%z(PBquo>8{XvWe)W*MaPbgDs`KL zF9-eehDQ|WDesb&QGvdQP@@iG0(HGkc?5oeD&p3_ag69+w8_$Xd-$!K?}P6E^B(3> zFG0AcK?g%|C@?-#*}@f;cYp=MEhI{(qlARS^aC|;$LX&;l=mYW!>^X zn5lfaUB87f(S^5aHIJ{`6Q2l+th@LMj{)K(K{n`_5$Qka#&+_L3t1R(3J;Dja2G!ZJoPw^x;&B@LjZYH;qaneQ zm?#8o{Gmm;S(OU~v)g@>zT4$1knObrvoNJir-DU&&E+1TmGubz>^r$Q%nbpg9Uh!H z0gwUp1p=a+5h*FQ^ks}sl$nrnm zbE;9|Z)uS^_zfQKY9>A{_y{>JVVUSS(fTLewiq-VISutmft@|S!wUt2MgoL`Su&5V zAVU!(ts_;3WP3?WovE10a7M;R00w4qL~&n}WR3Zpzu(@P28!hUlIE*G!OEx8GdzGU z|KQ61r4d0mo?kTFnlcm$cEw+7TQI38RpjdYq>?$wjrQS&{p-1~Kok{dl}0=nrE*bh z%G4%*6Lb4Z%<|f!=UL^?8ZSVzw2RtG!aVN#UA`|qlgw3G*tHPu7tfho^RzG0`~WB0 zinZ4>bL49g`!YyqUl`Yrp$P-S7|g+2VMeI;&V0a&?5@%@N282PvkDJO1|!8gxoU>u z)sO7x+*Vep5C;27M0jR!4Mw}Gp4!C{`1%TTttfF29t={km^ef2ARquO*$!?loij5* zU`y5rpTJ{;(5%&ifIs%7nyW43id(oY=XkFR$5SYar{ZkcFirD|f`x-3P?k@l%Z@cj zK=W%yehGEuW8moA3HRmJ(e=I*91AP=OkFQWKIa(^_o81CxkoEqP zwCaykJ$WyIGUPS#(^!B$n@mO&hIhEf$u17R>z9 zcxatWS=Ij+K1oSDtBe`jc=~<)dbviU20FTa-%8YZJh19=*rXEg5D7mgV%0R8wthJ< z{F5l3BuCc1E`~{p`wQ(%n%HVA8WvQYc@JWvS){AsCk<(r>(4gZilFB05mM21FpdJ!1pLcL#IPZ=&&Jy zFW|FvG;oJOic;1I-b2<~UQ}j*_T`!3;8Hasge!)Vz$OM=3Wrs$F*~v;DN+`YAJL-t2XzD z;NqSM$kEq(*`M6Bz^8iYQ`yIpC*M9^8XUD5Yx~JvH7#dPYern|$JgJa*<#05i!rHs zpci;F)3b{?(~gI&vK6qmGy}&{Op$G8d6VQnyC7;k+pYnE9BI38dGGd>1ZBkt4s7;K z-3B|sP2tvLg^jU0y3rw4ULYbFE|ObZVT{#vFHEK_1l&L;m@BGPu71>eg)yzJ`7O)= zORGrjC1=M|5#xea4T#HJZ+_0(hU?$fmz~KhCbfg6idc22SA|^t2D~QjSIE{WY`2=Z zz-j$bF}VSW5o$PO>*XK=G8yz0=P}ea-MsBh0aj#C%G9}yi)?Sly z1#%vxUrsIrGuB$C==|q1MTo9VYMZ`goY#@pUD6OFAnAI1hQM{_CGZ6*T@U%Vyd!m#p-TlHTRY2Fvg@EiMf zqvHbZhj+I^YIuh5RO9&4jc#SKFLPw_6ifkx!sJlH@{Wa5y1D9o+M_LkN08*_*Lrme z^vF6;G?>ALb2+c@)LhW~c}+7DXWMqd(HVGLo|v8(B*;!8!n7;Ap%Y$cea>$PSc29u#v_Kn`3iI6$ncrYml{g7HgNWCQn!y zf+@Hwxh+9R$d^Dd?6)mTa*$(Yzic$&FHBzs9!2EVFof-RQJaB&Lvj6cfnk5>iuDFO zw77~SqwQEPq~7&cGAnke>^f8?3^Ih@r5XYxlDbj4^INh3)6j_8({5Ybqx3L*?AT$k zdUmepUe|+gf|qbQ?+%!?7uN)XQqFqUIdL+o&Qw(m_Fy)X?|Fz$UO|r=dy!! z;>S=^iqV<20R*h?eef3Jjgu656sE;=Uz)GVM)I}zBcI0#<~X;ZNb-0Dnmuzhad-7P z0V_^FZqYf`8MKu3cxhgE4*wPvh>8H+PzGORaq(W8%`$y_U9Hz*4an!wFNIrb#onH~ zFDDE*eu*mMscT2`0V5~VZ8e_O+|*xY_C!%DZW)Q{W*?cL982AuUv7W%Po`%O6WZx7 zw`kFVO6qIc)TE~E(u|gt;U5|E{oPLc;&_aHyCjp_+5yuc;U31To;ymqW%dd<-&S?6 z^9T>yzZyXOkVvPU4iRnzMHk(%8G`9$VsFUW<(W9I2A0}we8F$>nFS`#itnn2Be+!JfJ0g zp5?t@Z|bN+c(+3O;?*qWI0x3Oz(vmE)9s0(TXn(dBj?DP3pbCcZJmc(@d)|t!VH^j%1CoDhxTeN2p?k5=#=I|$Ql};Jjf7X3}8Eg7u zz)#>`{gU4!<;?Vq|9WC1_^n6&UN88cvx98(|3j#3Oe_Mi#D@Kg$K;Rqe*M>P2{`Eg zK2P45sRx<>3iv~e{g+1jf0{-Ne@e6et6KBVw$*=^U;pv`f1C=!{V%`EzuHC2f1ikK zS5dP;VL5ZlmNpQORtF*(z}F# z2O0lhvqUMWLzAdPdBLC~ZmS8#K-OOFg@wKh!>^B2k)*+S`Rp$JmG{&o2(*Bjw6L{} zbQqBumYh4CAI>cszhfX^u5dghia78;Puy0j6L z5S0)3@j`w)n@<;~jUUaq8+b2DAs5P-k7bkyWWwll8wHH*UpPz0m2?)@?p{AAx9yi( zhiV=P+8pcBqL#Sq*?%_gd+I9n8r5fLj#6zF%Q6H(&zRcX*b@vCow zgQ#WR2y9@1MczUR6t>dh95Ucx{4z8$7PuW2vtQyoFgB(V8$B!SIGSR;u>OExivbOm z@v5Msog(D5puFN{rOB+;eudh2zNkI26OPZq82p*OIg7cLs`_Ar_GK#17VIQyL-#w9 zNGdM8u~RTp4`wt`i02Fo+Fym-D!P^)FKKzZwxU32Ccc(C{=e5AQwbY=Z@To zP{&WPpQ;N#FP!~5GP!s7_*i7+c$tJ&NLtZLxI2HX2=^i&x2QH{J871tZNNfTS%04G z4?m0NQ1fBSTl3^e3{C>_u+y_N2*u-{kw_{K;uu>lx!O&(#@Xqn$(+WHXg$nMa#Q!` z#Ix955BT=Hf%rChT>{Fm_(>F0WbfGDI_jtWa+ z0^{0_Phcf|QocW0+93;J?;WiF+y3)Br#)PJ9( z{=Jm$?@Q0xziLIVvYx$r2l}?N3+5ebIQxQFHCi79IccN^16$#*^(8Y+R5PtMpFNz4 zhS}>M&R95D-697j2+5;%3f-E1y*^5Y#Tz)R4afIZ=8nryqmpKBjfgN}Rf^s9qkv5j zzGMBk22g6axG|+tn&@-5@x3NGn;jsN*KW;AyLt|0 zSeg!AepFFIi!@oHjgvqusD%3tk9q8zB9g8&v`h!86xgq9@|o^YDn_=fM z$5Wtt>x)1um8Ith@QtoeSI)0i9Dql-%=nW@lm)ixF`JQAc1MG-i;_)C)k^Cu%@=dg zP|po)%(1z7Ut(UC)|fQ^_?J2ExuG2N?JiU3l?RvvQM9K5&6qIljERyRiu<}?(rmQ%ukC#W*yMnL63hrgtl zK@)XOWG5FUtr@v#k!=-wviXH%)!YCs0_lW^tbnYftf;KOR9iMzx;8Fql6a_sTIL>7Ur?iN%hYfV4S z#)k~`pp3MzTg-AjbTKMdfIt`w9nusqhBjYD9@@9Mf*fK)0a@WGWXnbOQZ&+gvC(#I z;1OtZs1`zUq9P%h@SLP|j`}h1O72zg%4ajeN~ppU2zZ@9ocho;Xf z<@%xR?rm)%uV<+1b#)^j%G&pC1ObsikY~ir3E2t#*m>u#XzBsSVM@GKt)A_)B_z2x zPDHvT>t)3na%_qxz<>)}wzhFD^$?F;d+GH_^@NKH>0!W?VpZy0)~C?KB*I8@}6 zUb}^(wN*78aNl-s!Ixlycz^C5HrAP_Oue|k%mq+9ZCtxZtR@5UP8uq;MrCb`$uA>q zu9=q^FEt2_)QGGlPkhhs2Fzj3e$_=Ijwp~fLqzkSjy$Dt2Ljl#F-|g3-*hLBF5Wvo zx*MCLUlNkO4}(J(;qkv8q|0yO)lWv7jhK=WPVqxY|Ilsl_-eyjPGR?==Dhd&O9>re zCr@QX$GINshDOJ^ljZPnqC`r7A%lIDi7YcL)6sm;k^hiW{N!!mB^fL(018nl!%YG- zh&N-U&TVN&129C->ZbgIVrv`Ztel{QCP_&d{c|5oKJV+nYD?SobtR#zq?SJM(ih{v z*>qO^aQ^6?LNai|Yswg_$IHcNkyf{MbsNslo{zVmcR-!qFg?ucCV^1`Fr1P7oc^$` zAu<^w@^C+tD)~lD@yBw88SJv|Wy8nKy|yT^w8uD+JOIw(o35Vt0Bn^-j7;YNOUcKb$m)HVoXfdT_%?P-Kxp~d9QcrtpCN~GuTN=j~D!mk<}0Aly&o02Jr*xNE()-@S3Adqv+yQ=!d8&(jz#|D=|SFZ%}u9 z>uA>Z|40JqLskLKTe7cXHD{;xF>IN9JEBr0Y;(8T`b^@=5`2j<^orNNp;# z6|;pG6$YBZKF4Yse$N?C*b(#wWzbVp(Dj1)d^hXFxWu&AyU%1oC z>8jM!f1oygF}TVMyJ1$>e47^+0fqTNB${!sLX;DNOwAaFBI?AylRli#>dKqSSiHlm z7%odASl=zN%S9#S$&?D<#5?R5^Mv-{@U7HaONoTUW)m`Hfv^@q&cm@peJ}xL6B6!& z)<>y)zeJ4%L-68DrKH?dK>-E&>T0%?7^U+)Q*kmcrw=uDF48ZKbWXCauh~xv{|i2f zF5?gusnwR!{1dBUsVs1aI!B+Z0tDs{+ghrN<(;xIr}egCRB3Np-VHx$iUJYF(lcO{=c@QD^+pi#>ydj~ z!l_?qV)od8kngNY+oq|0HKVtbKz(vc0TWH(MumsMMDl|HxS=omeG^wzctJdNPNK7M z9<{vvycK))OtKz%|Llj%~+Jw=E2Kd>(|7Es_aL?_qY0fTZ=;?}=r#dr%A>G)0-o z_u%hYmIu%w9}&5FA;pIYyxU#MQsjrxKcGa2IN=q%*MwlyY;dzRhDzgl^ERB^ub zkMyta9Xq#9&iW%_eJ|LCy($jzFV%G+#3wVoQ%Pm}sNF}v=gX#+yNhY!JDJ82tA!Y= zjRZpy29#=#dWc=|@{r4(8V0pqdrP(PuARRnQg#$Vm@ZyB!B17}Dna6IdYPAxD9voc zxF2E8i1(~dGyJ#}K@BYRotig7#7wkGKEE(3-g||%nZJg=PM*wudq|_C{HaSbqQ%%# zUQFTAR?iLF0?q2(Ebe^#fUudbx?=BdVyx0O2KaucK?Q`!LSv14gE5 zxtS09FDnc&?uxk-qLxu;BtJS$90z9P7u_~Hhf>`kWXIiZ=3?}@Qo9nQA&( zA7tka?u#9{3NI}$UV30P-dyaroi1bFZ-9{)cE3_?s?<1by(F1$ZSniFq>8mA)0`M{ z6)@&1!I_kIbqHL){8r$7wVNDw5h%ud)(1M<(}Lj(cJ(S)>~_IAzBo|nB~!>5ZDE&V z@oy`@`)&{-Mak4&)mbtgwfsqQ9FhxZkxyjZXt3uP00iLk z8IiLW3>+jxUCROU=yD8Tci{Z4YbaAKfN-uh0EA7Q$sx%3-N)*evZI)2hM;q5l&KgW z1+E>!@xIqFzk8#j&|_%b#kb%O!Bl!jI0RNC+UZ_UpfzayK8ocK?{6BF?@l>7G+(D} zc*@JlDovh9I&S;am-IqvtqFQ3=z{>reQ9|H8AI}saU-Wfaeynk<2NX^-`q(u3oU1z z28jWf1_==!jq3hdG;BYP(FcB9f+)b_u~!;~P5ji1Sa$&L=zIw;zq^a}PC}D1-_(kX z5N;h4PG|*_#jjkvB8OnaZm_Y~&02&39J#Pf7vg*nQ`h7(O`YTO!CczU&rCqpX=3KG zbzvoi!dLT>Sw;a_hjl?^L)+>qL3JZ1K``is%`=zKMinYi&zkL4RlEcU z`%gR$U|f<0WJIsdj^oY)ZBF6*h5aS(n-|&sH^_yV>db$nEq~-2|0#+7U)pc}OWMNl zuT17Y{5OA`_Im^SpT~l7|4&Wyw<-1S^MB)NlD`c&f0$Br*I-#ta$Wt0q!7*Gc;iT! zoS2*0dFhd@94NzJNS1exvoC3ZMiQhnV#`W+FoCX@v}iiKx+65dhOmnH*GU3M5$U7S4Y7O?r{UYA~2@z>5%mkB+8TejU@XkQY5y~U06f*kUBAC z1=o+ZK>PRCZH?4~=#hL*D4StSC%;Mu zfWnxlG{>#WqfaYH1i2UH1!X+{F7V^1Cx;pkHxPgg)PG719T9xm7Cd8>2kf|A_?kKN z@_WyxA4I(370roklCkY+riU8f7E;6qn?PB>M5-y#ZW%StTd3@+3E!mlAhZT9rfbnH zyWQ)45ThV9Ac*J_-VV(a{wyVMed`zV;&6ImS244i{J;nL`}n;N**=ull2Z!2_bB_F zVcxf_l;GmhmJIEFE2pDQ1bJJ&4_8hcJqQt%bw!` zssnebdqE+cuKM-fT|q|l$v*c)Q2R4;7B74)DtGnv@Tn(OfrhUv2)f&8mj^GbMcxhU zF3d04H82WWTTTi(j6%H#PQ1p2!Y1j7ueh}s8Bqc{XW-tNPgGao+N3i2YkMj@#O?tc zhrb2Uabb^~=%y4ro^`z=8|8HzCki2uSiKG3msd;5M@l- zq9$;!ot}ords^*M>0z?<)TIcG%jDpF9)*phCW*xYbj4`%jjN zQXJ=NuyWxv!w@Mqb{oN+SkZ5$e5|2ohk2fHYz_f8++FLi^8p_aMnIGD!MJQqLvx^YggDX|<9@SCk>=ky^u&N_g;u`;EGh^dpjBzAm?s3f1#=I&3!;Sh@Qf0`RuCrx zD%Q1KiA7*7ee>nT^WKY@M&QM?UvA43-YAQxi3M-l)&Hd$ARvv-(4&Oz)0Fmx)gpcb z)Av&l7i&NuxYz(s7dQZJG3Whkb*(YdW^@8hb_@sk?lt1YrZc9YHsNl}~Ya0tRc3)l^RZJ=&kS4Ve2?agEhnLi3!pxXW#~mfO z9AWTp7jU1Xu0Phr8$Nt_eMj59W%;OE`Hn#61UafY0H>uMSJFetHR%5`;XTKp6)66GH3| z^OEa!y32M&hv<3AiP3nzwg^9S7T?~!k3qqvB=JkViD5lg@_|0(l46QC)(8-t45;fY zCp;5V1Dq&v@QVSJ#__q7-#O5=nYY@a!5HihrLSw)*p{jY1rwP%yd9^jHzntJI-cb- z<@()9S2{<`s>wxt9T;fRN?Rj)Ex2c0u{?lA1(w~VnV)mngMj)SJ4{cOS+slQyzScT zww|-A&s46^6YR*fNl^M=$Q5d|jJ}jpE5*vFSh0c_vlh2EN3KAIlFb7fWJ6O?CbVH} z8<#?D6If{BV-=cYfDdR~=8USIjW?BOc<3k(KIPoHhssn%7d%?wY%Y#Iir!Qt1H6eh z1sk$3(*!17+dje11a_}mWZ9(?8EC93;alBgK7k7Bw%>?@BZ59@9x9smDB+FOOiv1r@ExVyW%H|`qT zA-KB-cWE4g6Wk%VyF0<%-3jgxB~BDgr$AwU4s!-vObOj6`~>MCfEB0(~7%R zy(Vx}>DlSp_Z9nGg&|5brX9?_)ELWA8o%q8T9U>?L*AMhyVm}`s8cudH>PTOQ#Lx| z-;M(XtbNyWqz)SWimZO0E_`*Im>S+m%j>9U!+o_i-XV229wvt0C!%3GQU@e(hSa36 zoQ7PZBT7lG9ChLwVH|)7jaCU;jLBp-^@u>!pP144vOz zI-pD;hP(3RVNva-jng=^LUi~<7u62>DTg{>j&x=bQn^ROS0BL>+!Q@Ad^Ws?6{;4^5=mI7|^1HEp{oNA67z@1JfLm4HX7eaHycQ_IElX zFM1EI?w(GDFrlMKb~=?<$P0^u;>3X}o>rcB=ib|PhKMfatT7$XvRYoLL#s5^SLzz? zlROT~Rt#anp~JnqX)#I9oN3Jhn@+V#4RU=^!K-V#!<$k7LUD3x-sh$AYGAbnwVhh6 zxiXHBJ^hj#z2m5+De9+P%0YdSJ5dv8g~jECyQkAb1Nmr#I44E%Zv(r{eRx1C?`KkI zCMNjPebFf`JW09y3_@y~+0J>( zp%J{-6SerQ2?lpb!E_7pe&a9x)f@iJHH`FlWpkX(;*iA~OvjdsrgjD`o~%`Egbw(Z z&Kg9F-!}!nFDI}jfOPy+%;5l{v$QzpAcsb!%@$9k$E1OP8#paa%69&$;89p?&Z*L9 zE}l~mXPCB_z4)D9BuX_tR(H`cEn7e7=!Chax_F@pn===m6necaXDUAYnIJo=VqOl4 zIr7Z2G(}?3(Fv-%oStFu-c8!-?bCbHPp}hup{nZp7HP~F`S}20oiHq5{7oK~R{wkn zDLbb%Ou@9IYttME$dtpq5Sj(1(M=`34qm;}*3MHeQ5R3Uom~dB*^9YtfYIH34s-cw zlFt+jf!deT@9q;xnBxn9&&y%LIhN1Kl)w6GRfaZXRaUa4p3L<24IN~s#q^gIG{YjKL)x0xtAoRHRjJ;ZfLn@?_Js2+B;-EwgvDUZz7Fe*QAR?Bj)7CMI!K_uc(Bw~mJaom+9B7`%M z>MnWk7EZaUCq@7-1>=aHQBO7MbVKUF$K;~TqmWx%2H_$IR`=^R@RtsldF?k&b+L_} zfsWM&9OuBZ1!If7xuNgH=wQ=7B|s#&`UrSkJ*Wg!Li+gQGEufa+_-3Yn>XUlnT+g! z=K@LZ_RYVb<^uco3gpfbV9o1oKzf^Oo*GW^CBuwr^Yf|Ndvxe_(VMaK%dTK>^~?PC zrk{S4$`P;#-1_6y0mG?~J)6A?mYSI0yy#Z=#V{nWVy{MSqf1j7>u_s0@f{nz`VZKs#&o>pH&u#O+&O9X5Q%p<9yZCxQ;v!IC>ta( z$6t-&)`eXoLT~cPdKR7Z8QtmBl7zarVO?W<^;yt zvIJHO#YId>K}iJVh$KqF5iKq?C&Giy$dhgtR!nR-J$LuWVDwSZtQ5fmRs~>Mi}5os zmT6?Rz_r?BRF9dbowtylo`vdD0+z$q<#%zFHZg$C>x@O`Sl2y>%l;`Tbr=JoLOA-l zliL#j#Qzzqb(MF8}JUrubdjXe*86iUzqt9b+=)T~=CPWoke1kaSO6mJ6 z`@#P~h51iP4(G?7hJX2av$L`NYv;qCcHSIp9|DU%brQ|HKA@1835Q^a34zJzfB1s~ zxIf4{|B43yaDU9O{}+xAfQ<$4&xUogFg*AkD+22u^%7p7 zwJXaq=XZQ21d#=BHZ1-3&&XSA)avqr)Hc@XnDjT-SLkRJ1{m;|{En~XWLQqp;DU0L z?l*PX*dIQ%lMJ+DF0PW~AzWWIf5LAe&1Z&WD}pl@=eaojq$sSfD4hI>Q)4M>S zoy*sz_}n6sqI%RmvP(Cip;?@%DBU_zhL*;O>YlP;e6hG-HO0Of;*;xpc|#apb{X z{hgVd!7bapG%tdPY}J@Md&jP#z^Y^6DUO+<8IL{I!dr~{tejCC&Oe=qjDRpDuAeAP zyQsyHOVoUE1~&p$WjUPjkW|Kzg>fRCKkW?ZT&Ql{BCX+gEoEnxZC62+yI($}maLg=AaMRVh&2r^ zlgV2Us+Oc0OP7`xWrLE;4ke(k-|1z8^=wB@Wr=(jK;O?3*}x(G9hp7u1z!6lOq^E? zk>SV{tGdL&L`oaJwOKZGe1unJ?HGI%3wy*jonHwNn3fahiimkUSHYOk2KVVz5qFF7 z(3Fy7vA{1}+L7yc!Dp4sE~g87$zp5rW0wYcrDI8F_-|b~^D(vxjXXo`VD$2DWNGW! zzWm|R&`%&&m6eW)Y{)1Tt)~2N(px}$78#`QYg3c%T++g<%s04~+ZnKH#yp{CsuDYS zd)(ImAY?E|3cXKcxXdTJ`1-u=h__kwlxeM*HM8Q0m>-mf+2HgaE0t7YDs66A*ftb{h)Ixb($lGzcZzuDZ!JRL_81 zWE#XLaSR+w`gF9~20Nn~^@h^OB$;_hTzj=>jepFke@1LCt(W%b5y|x;(_N45fkV>; zf=Hgl==Fl5?e&qW{#OrT4*Dt+n;3-F_fs!=AJ(a>|wJj%-biNfhCKjHiz`PmbvzzCyfuB^j^DG-`CE)Dri zWh6+D5k4=be0Pe*;;ld2Hs&u?=u`JPa`Q+zzl1%PY6*Dh(Srn4ePUl%qcNHla_EjG zWdO+lOK8t|GIU>|`mKzDxO&LXEgIJgIS<^UxMvNdo&)I*z>}~5Gm_|E!#fuq#vX@G zlDa~k9y^+Iy_Ij={QO0KJYhG7{N4rO;e+~&(+4%s{<8Jv>3jXOJBP9fPQ_k*?H7Yu z;I@fxRBB66v`fcT+s9Aqm=P8})gplUSg8s9Tv+&897jcK8wKY`4TV)%wEl^?3KRGp z{Rg`;eT{4+FA3lWCskBqKqO+&?V19&?Qx1N1izyG5frbT+PhUBM^suwft)&ylNxys zk5agZGiE--ecdDVH~+~fdOxsvVa@%py0G+&7P&Fhd4eH798Nn>{Z7x{$NLxvuL~Ry zyVxy?$VM>t){5+snCpq8xouKLFgU>LX+MU%mT1AvpJP5{toBtuGCAT?2o-sBE#M~dL)&h~_cod~w1-tzcoHz6{GpS-D?iT0Iw1!6I#JAri1Z?AG^Zx5(f&VMk6l5Qp7Sbb|WH=Q( zIA&-Ot@125ob>un*egcaWLP_3(ShydN<9+$d7_?wObE)JHEe2ix*5xC8wLUl%Q?$Y z8(DfdjQ=s6cRyOoh?=NcmbB4;`&z%1%EwA$TMK?CyICHNA^Mtf&sbfy!4|d865*pg zcyK&76V@?W*@RzHez`3xb=1@M<>UPd!buS!DJ}_76;8QZ=X#f zteuAntvn^s7Z(&w68$aRE_*a_u8oOFzUnh%;%#3W+1JC*Z6eySpPNdHmnnATo?vbE z7JQ`M8C7xCQWUmpPNGZqfHk}<8?|=?hxzxY@HxhVChU)VGEqAl#f-x2+C1rApjv`^CK6c}TqxC+mQL^{5 zU8GAsRs#8E3$>4~$)DGPR1#QoU*-&B9CzVR<@90wfaFJ>Pjy_|R;SN< zFqTMWJeK8NnSJ{AZ=dnNHbQv0P1r#2P+_s8^@7Z^1L2vBPoA6>Z4g|vYyH06vzTrAvOG>jG?eMV@obD8=n_h|x zZMwc1p};-vFmY}P)^c7p$P}mXE@b?kvyk>0sCGX!Q(+oE?9J-hnuc_id$Xw57bmv8 z6znKWC)0mzwzU|Y=j>H?6NHcTcq_&i1uws6C4eah55Zxv01otB|9B^g;|4`Uts}V_ zvLGXsyM4_8g{O%S{xXGCsX9uu&8rY&AIY%RH-S2pdErVXOC*#-94>H{G{B7l0>EBIVq@)|3SsN!o(j`>4-?UQ<+-5#W zTBO*F7*_Tcq(y+?T|0eT^nExNpZ>HyK=I5ij6^?A54dK(j51Z})f`2~9ye6feQY8$ z*-+I#y<}f=gnjqLluMY#e)Oq4Mxd7_Jz(XFJ#i^jwte2h)6M3qu{>8ALIeb58)oEo zPk=MhkDAu0hCPY71$F(7+QeB%BAR=qD}>GET|+G8_da!qux>huE%s)69V}(T&w(0# zM}cT156{oGGp0ibJ199{ubZHFc*>W`~@zQ&l--UTzrRL!EKEp}t2`$bxU z9##X`BOM!4)n@Q#0=c~Pmc_<;b|0SYvPS3JAxxqO!op$W5BbC-4qGGezg&<7$5SRk zjqNG)xDI2_5fQ&?q)&Q7&9$EIIk8|!V~?F6`v46fAG}O>HDu*V58fPU?qCnFIBxQ2 zCyt%H?PU21QCU>>G}6N%wFyWSj20c^W1aF)`+uhL*h5{;oNYq|1VKTL4HHx%Tex*U ze2$q}`cY43{YsDN`dSg;1&uTond#Bwp&p&otzwb`${@;pE&uLzW~KJ0BmZrq^&R{TBL5~qBoH0` zk5SftLK=<_>g~TE2p1dAzmEQdG~8@F|E#l{E44w_NZ|ixR`vI&3jcp1k#PNoRNPf9 z#Zd(u3+T1{?=5voFthht)j46?ezwS;*!H+SKE*pHLvz*zTySyr(7a^C2hJTs-rcy>$N{|!P@3y1 z&#r5jJ{ZONcI<~fy$9m8;kK&2yt1?=^@O<}7L2rlGDDG;OePj%5TH!HAYr3Qh9_WU z4sLRw7LvE0xcL?Yx6drIcUdNs!r}M9@Hl>*IYsVqu%zAuDF3$D$ZB=w5W|x+qV)vR zQ6WHRAzSOoj_V92oI@mV69qO^J9n)d27c_STPmRX1|jsQb!BmNyKQlma(M+_^oRql z@JJNg#{l8Atx{M+Y1FIuNYs+nb1yJFZsLQ?3aueBl?yP9SJC~Etx`4gxan~L_yr~V zi_FKKOyWK8iH<+pYn0zy8V5Gx>FPm7q#E%}WtQ!r=4<^Yc^=X)n+ibKAWxb$S#pgT zJ-F5gbBi7}L0Rzn2v~G*bY?50Z7l40W^1aj(P^p6mY1Mi{SI z)+6ciTxtPWuUW_G1cktC7rb8uuG0xrh*noKhnoU|$h>oxuHf%k=F>Sl#IEN38dQ=zI*%($s-e?=mbW# zKD56>=lA}ciX=WUv+pwNs2yDIg=gc*k<UxOB#Q9DM7|mLg@$d2|riW>2_QWju$8;Tq434azbg;3pkD~?LCIaFS|o?pw`EORwp?o&M&|od#rCq`I!|~A zMZ^!lE_5Ucgg5BVDh<-%t)|5o-Z^{-UU`tpvH#b}6C}!X&U__g5cw+z|e4;cbNq_cR zX6~8rglLvRab0J@MWc+;hbX}ABZf$9=AnvuRw}V25?_4}fHR}FgCG5w_{iH5zG1S) zT7m!pn-AEnKoCVU7Fhf+7Z1md5`Ks5A`nYL5lAoxyTU71W(QZ9mL&v##h$VCNG#*9 zktp>e_JEzhD6swTN+<^#g-;a&EMOd{q2M0MN_7D2TqZaJZNGR;Y zB#GZLQRc4jThKfE=iTS+?Z*&1!mwrSv>h= zF&@U8d#0WC$ur)y4LddTY53~Mlm3FI3B#cV`1<4r4RJmh&}UT)aSwYY{-~yQJfg)W z+P}JlCP4a0N(Tuv0=?yK3FGgzZrJtkTgpK?`k@PR#hFShF1Sb~wdsTfKMZ z`K|NfN$l0od*gC$h#RJ|MTry1X(Vs`&;k5fw}cMKO|qBR)vwZSp8g)pI4$qtd8Ll; zS3ruYTGpe)de{m>|BC8lh~9W>x6?19l4&j+APET5c{>(REY%+sy^p9jR=BU5#Sym1 zF0z0u9?NZbkO8M9J=Y&cTnt@Y&nK{8TkSnH#lYK}+yXI`5YU_@PZ~M6jt*BB%}Pie z15cAj8L2CJ<8-bk0w)kMvX{eW1;R)QnM@j#zG`%1PuMPsTVSCDY8G&Kh4B#L6k1&Q z_<1?9Dd-B=Z+s9mkOudP>4%+gK&a~=l6zIr1n~F*E&H6fENcY~Pd}4=M5bo?;#tA7uaZAc&%cwox(;}rWJ{aRLa&9>5b!ZMWzlSY=H~B0L_rCWD_PzM)w{2^O z2hexlu-Jo(in*&mDX#*vTurCjDs3vQ57oCw&Ii%^7oLa+v8D;Cyj?UM!Zjm6W>WzidQu1yd1oHq0D#~H`B^5?l$>qd*^TE5;-ey;)|2y3MT-UiwILfs2rzES? zy%Ra$H31wDAKkfA=0;K6J_JiR<2`s?z5Q_NbYH;B(qH5^nygMoxBOU0r;UUQ)-2uk zwUm7_+H<{c?MW+UX$3X{_H%ihU3=5mO2O96bdCGvlXmePFMUWrR_XmWEM_giW{6y< z{I?hR z^Vhn>uL~>t^Dc$O;-y^pI+s5R=tzwVel_Rb4q`rgepMgPSf)x{YI7N3t0W0(V6d_w zA@DX-@DXsA?QTlh(&dB#b=_S~bpsm+6-Zg8Bd{+no?p4VTpFRv(#p=TVpZfUT*)Z{ z>IDm}9kkkb)3sNzSxg$+L}6;1oJY26dfHTwJvw+9veH>%s#yx*54u@z6e;OJdgMwk$9f>IghDlLVRa}sOy*>Xt(vv7n$n_ zk`6xvG2(dVP{N)!ps@UeYPc-%rQS3Fs9Pj!`9n_(uG%&Y`?=ioWBr7y(KL^(`4)Sk zTU@9}enXnAScu8_HVnFLP)5?1MY7(3Mt8iO=~z+|Co+%6=`2!$1p+MsVR$6e`ti$J z(*co0RdFx1WPWHV>B$3_!L$)9yC<+J_*oL)nXg>1(O=_W9xPT==Gmux+J;YdE6ozV z>t>pj;al*Sp4sC8OBA^ShW;9>D^L!N8uDzRV*KV=CfX@p?a8PMAhD~o@bUh-H);SM zL@KxZBL0f(8DkZC3o~O&R(YyHh8=4BzDk+c=<5O{jOOg-uRxb+eic{JquN zrB+~rGGuvRqs02FVRVp)GN2Q%i(D8YMXiWTUz3MMhOG2L9e) zlRpb}gdPBgRSt*CH_h>I3}{5Y%xw_rD%xPwmeLwYC*7=l5L#|ARwn#mnEX)0%7$xz zUC-}IB)COxCM(o|8Jy;(_jFmeFnSi9w$lbY-Fn(@a~cF74yT-r;v!2c5rzuQCP~({ z)?%=(<3j-0qM1yJ_JcMi&sB|>+wOi<0TR;u;F!27)4q7NaxJ^J01pPDSS+K?kA0T8 zC#}w}HSVPjE}`^{qM0rX%sj)MXFzzjTfPVaSbWyqgP4$Eq%qiIz|sTV?KnBq6fUt5 zQHLPFRL&cw#K_^sp~Hyq7E&V2&*B=iYG~15b1A}MHd4Gy>?Y+f->#Oz9!HWRl|61j zW)#-ew=zI#N}I|ifO=;DEcYf>_bym-3A*hfNu(){@$8!V4L=*zi_G{PI*wl$l+(IX>vBD>&0N!bq z;K480E%FpWZcZ$SpN~zUPr-tsq)T*z-)E9S*M069k}%9YfUDrKJAv~@mJv6|I=hp2 z6W*?gN;x+co?d#G#O2y9AF|5;@CE2qNfOtxBf@Yz;qR+b(!1eE5It+&q z*||d!H=n_QttPdDdCOIPiJs~?fNOo*=$_6&^@Ml8av+g?yZ44|ZCU;zxKZVGeS&0- zTb&o7Ccdai^AN+?M5E-6DGQ?I*ByI=3V=kP@STOsVo*COMXBRb;y%uLlC+;5;V+F; zG|VEZc^{u!(`QXS3TbnSTF+WUzD@tE5}^dwJxhpy^4svD9vF+KRW*WY zJ+(F3=p(pjYS~#xFbQwxTL3q2eE*)1jUj;Ck%oSDkjb0awio3cd?lwP_P>zRA9L3~ zSVw>RU<26x>PM)M0NDOpHw0k&zeU7Xn*m*83tq>7DF zNZL>W1*5Tv$q_hoSx-mJgaZxT!CyJrW+r-j+d-hWI(j6}v6qqK3rxeQf&49-kln!q*AvxCTdviR-MT2E;*av`=B}&e8Sqr%9Z6v#CH}b2SftDg-b+WBQep4ebeO3Nm27oS;&Bo@$~SfX|~f(vWjoU#|w)s zYH#Ck`A$kG$|A!8?v*~Nr^cmqYo;b>>I(lWn>87X3P`>F=+az1IygOLx29gSp}rqF zdL{A|Wd7_&`?&6r=W#jExG%E86V^SkFRKSA?tTK$MPs92bpKM}<+Yc+>VBhg4bnyO z#X)2jrj{ZUJ`jyGD%UzBiJ+!$TDO$Au`hA#s;TV33ckRSqQNIytq*NyS^mk7TuKu& z*8Zhz2Z+Ih&h^+t=cW}ctb`VlvsEU(pxeCg)-{}%&Tu#{oj;gy%DQ2}m&2V4fjS@n zPpQ<_J#K{S3+9-pv@1c+LA!g&0OqdF`3`rCX@*AL8>2o?6X#fg*+QCU>?hKfQp6^S zdlOCsOYz%-^tSF!T0SJSneb^=5Rp2#Z`_lYN;1F1#S>(kS~ zC3E6LV3=bJr3{U&t#Zvqx6&(rk<>JI4G><_qW3(qd&-#g#w_=ECzKk6Dxy$WS}xk{ zZFhk&LVULTJ`iMCzmzaKv_=4BIg5V5UuHtjB;D5KVc(DfQFe|$N@8_sEq^l4Uf@d* z#3JI^TObupj0t?L{CHn{j1UK}UnM9NGzj?6_8p`l)=2!&=iy?Flx)ZdwE1=ocHp_Lebf}&-SWtgAgQ2W#GI%-HQc6(kU-R2FwvjTFuM!J{Rsn7nsVU${t^GW=)507Urwk0Nrzsdwo0LZC$}F9O_}qWK0uUin=HLZ%`S2r%ecC?u&U zibn4zeDsLEybnD67#VO&;>L+g*1n%Z5L7!_1pm3Z?(GMpK%moW{FC>9Z>J(3@b_Gp)2zEodeAJ z)@CSTYTcZn7R?ohf*L8=y$B;cX0fwF5(?Qr1@`9MV%AWh#nd2H0Oz4Ic55r{J+*NV z^%nS?VeWp|B8m`I8+dj*SPjPDQPAD>pcvy}4Ryru^6G z58!sq>^scbc}v+n9a!q`O_*j92eCzvWQCmid|v(HJ3<53Zl6qE_s9^Dl4z`lS$5fW353aPjJw4!{_+yiF zh3T|WvPr&EW)bj5Ww_Up#0krw4 zNrtB9&8$9tK$8o<6`z)48AyK1uLR8ek%%4B0Y`xmIc*>UM(KrLm!DmDdRffOyx4Pw zDh771!FzLYqbfJ6jPw*}=P4ho%SKT#1ytnm@uBL3WVU>QGK|QM@HsopuBWW*nY>GUiFX`j#Ay#fhpdAC5p*cNS z;ZZ|s?QcRL;<|KkU!_KzQ9=1dPHT}~cQni{(Z2w9RbNl2i5y`c4OseUJxvmOc-z6) z;v!nB(EDp3Q`^@$6pz`|VoBzHKJTWcx9GcC)|4`$sOpduv5Ux2tKX`*w_4p7vE3#U z_h6eK0D?i_<7kXrG}q8@z>4e^*$X9ON9hU?jJ{Ja##aMz`{-5f?}0g}cSC=&0f}U5|J~8s?IL?l2BHRK-vw zfTN_t0tCsUT391?Dw08H19xd^w+MAlxO;&bMdxq>L-c@Yo^{WN9Ba(8(T0mc0Ll}` z9DrXQB%B-1yWsb}(q@VnCcCb5;;Sd6GFAW&^^3{HIl^S%S9g)6!`S z)X_WJ3bsz3!Y_l-^J%{Ku0Se7P2%s3e7dWr^;fe*2{0DqtLCGFNAo~Z|L{S@HL(CD zexe1%x~3b)M9d@r2$hqyLAj@>^6m;&lkpm$RO97ap;?F+0T3$$(K0J$a?+&C57rti z;JzbC5f8eZ=}8BthD%0CVgbR#>$SW=YJw%Yuh_McRdj=(ni-HK!njKtk~Nz ztVwG_Ns!RUl?6p9UAoQ`0gd6IbCkefWZ9jX%;2c_sC*6-NFxk_1ECV66Q#yL7Q(X7 z-4`CS>ECnLAa=0P`UA%4{aP$qa^w*`k&4OZQz7ZW`K;3BOCp5h>L>+4E6(3PYhwh} z0DdlCb&Nii2!5aX#iYR>P=jf{8(1+TCz$KRWq`#jx{cF?V-{&u@)Vx=%8>&^E*0$? z8K7d9I|;7srCp1i!d!NLB!FY7Mbuv5r*TOuoFmC(tb*p3wd5`YV2qjN5LAfn?52Q}l`(ueRVpE5HQ+^f=U0P$q#Vdg}RZ5{Q`r&h1t_FntQMJnFkdmgfx zNyr?+^O;GkRSi1?vea=A-VlHabI-6WQ~E5(sVUP(A{0f^nUMVh>@+1}RoCZ~YK`VQ zU7ZBJug)*Or=?k{N#vjpXdOA*GG}+R3K?ze=Z9T(5p@MS98)L5#<4fSD?YMd2wLme z+Q`R`4LdW4w`HIbp&Z=0mi(ojm%f>^s2jR-zKDB+!1{J3!qExr)DXyc*Ukyuf*E`U z;;tfx^#;RgO^U3&35!xmqJ({k2uYU8LPr(Yz7i#X7po+f>?_?0hd543roCq)7h&?V z>y#Hv-=+rYIsCP&Kw(WxKzQraYP*ue1iC|2g*D;#*nQJ?dvSimF3jMQ9xjxKK|sWw zDVR?^u6NSpN4kT|Ov7fE2+6B;L%R}E;a(kt8F4h)>xS!wrcFT=1jm5;TS)l#aP}Y8 za-1yxGpc1J;rj5H|DU6O41HPH|GBcK8DknWfH=Vu0^zUe(jN>`p8w#x+N*#MeK1J> z3idZx`Zu6s{V##zKV>9;W4izE!0qp4NX-BCf?x&wrx^s88A%GW3pnO~^IiWv3j=U* z{PS`yOgd1k z-B<_7*H47RlJXEBS#;rk^Ijr)&?mYo^5f*;_Q`9Z=7ATp>KIvPBZW8F7uRLM1T#DE z1zzQB6*QMuz(LWmcEJkD>(9_lkm1AX4s-^@q+yn6DKXjKitFpneOr*1h_-@lyt2T( zS(b8q1mld*A{a&Sgq<+9>j*#{zlp=~OANfMjQ6pKw&?{q@1lWUS1L8PD11Hj5l^IR zM}(SvGRvwSJ%L6Yi#W0J#2+4hpV6v#0k-LC%!Q&7ffkLYf6<5Nu~nPxPYkcGm@K!I zXz*-JHN$K_?FUB#lH?{IP%FIDt>pZi))}#KwvHVe(7?DC?0boNl05ysV16AB@7r0W zEdT|+qgejJiSky48aOa$1AnZxv`J*v(75={%9Td&;z?r2`w(ma2``Y(psIW%4@`lU zTA(t{!^lRd8TU{-m_XiKw~36obz#&H)#=Zk>EXBa+2OiLiRlV9UF@&y!zmGme@8h- zYX}u!k*+8(P_kF(%q+7Zt18t{u0B{Z)@_De{3-r!e(4y6Zx;3jMBn^)#~H`fGFtOZ zQwd@dkeoqOdmErYA-L1K9^R&yr>ZAIM~-N6QWnZILgTJ0#dY@@=os%qU`W4}LBlxu zxnTOo6U4@Z{V-jomLv$u5C@)rbG$jQa{}=25nV`=`8TTUOhtUtB@n{@7Qq+8%)XO) z5ZC$X=$Tm1(H_dAx+hl6w$uL0!E6FU)9{y_Xi(@(#Y@i^Gb{pP9;b+Ms`hf=P1?CIiBX^60xomR!5m69g%zRA2qha7^4N4{iPZ||g zW;YR0uk=%ljbYQe|H^X~B9Q$vpp&l&Y8nbXU~P#U*B~O9C_$#R?)EDfGA-nVm=~ z?5&0C#{VnrB>UBg;Y)$Mzaq2bo@s?b`dRR|2Pb7!Av?A?tE9qmPL%xteXP!jOk|-$ zc_+T09u8M#n7GF!^9Kv()f?zhr7Gzb*Q|KukoHp)=;j>~`Yzysnqja3)>uUkbX@IQ z-KWuU$mxm;fEXqg<32EnK6c}%zflpKmf;aUZSDd<5?x!}@IYd7@O*eDorUcJh^4KpRsBPuDzQMh3a+@XgxFt2 z_cb21c42^Az8*~6j}L?0$;a5-X;FJtj75 zJ0iU*HtM`guy_{NcaP||IHgJr&lvGi}>Zp@dLo52%hIQ_V7vDKUzpc_&EtJ?nr5&w1h_pZeQRgu&?KLL8raM_$Xz zUOI-8nQz3!S}N;!^MypIx#DIq7uF2awn@U9`qNI^(q=N_c-fxjgIl+*@Db-JU-0o( zZ4HXYiW+_8_F`Q5rhF&2&8{vO&jzi;ftgsFD|($?XvrnR+X+;rRMt0Mq%luQJED&|FB_IQLb%9#~f{Rij zvEe5UpnujFA{{^MHqR3PJ~=&r5PC3xAW_0b)f4_5mOpqH7*{G`3Bk4Ft<0K$16cp$ z(o0rbLT6(=#_q(c z{1Am`nJ0?Hv&0Jx&>r}~0zVkxcXxzZnLSOMikSKiU+MRH?%t7L?NG9ixDz$8HYV#O zLQVY+*_5pc9PCR4FE2vkMC$Ph$9*=uO~xDM2T2idu*=5l69joCH9rfosbUF&Hsmo4 z1ZN3MCMdUr8;_xX1b`sha!MD*b^t)I_f7X;FM=9rIsgr=#JkbH4$8Wxb`x9n%zw$j zr^B$|NjL`hPfZj6SYtQ~}XBe0d(S(k37 z_l*jq_o;>q z%LdYsX_cf^l?DdYBuQ_1LDK=0^@K?}3C0;9uh#p#tmr8VY*l(i)G!Yi_RoF6axHH< z&T=jOB)4wq(;dws=vrH){CLuS$-6HwgKd@d+ca>Vp9v%lCrVSaCf1Xk*q|qo?B%Sh zs!Gq2HrlmWYsSr*fI(b|7QI)Yi#uS1{yT_c_31O+D-}7l_)?sw7UJ*f=C2h}j%Le1 z?&zKq+IvpqF5zw$`Q)I*pSiw6!83h|fyzxQVLCCOn%Zc%Xf`0B=gVUXfy$#v)EwBs z-N$wrLQQzsz^v}jxmX(xUN>W+wf+c9!b%AW9G}+0K*RwtcRVgufnb)yy#%P<*xd}` z{^njziSN2dHm4vJ-*!n3W0pHTpKlO>WLMVh(nsepaCzU~OC=(PJu`z`^k@C+ik64? zg<(W6Mtm`*O4E_@#4Lv(G?3`7@_id&+`i=p1(CK;FeMn;CL}!BT~#6mQSthQ6L$#sYotZFG}%wOH-}FtLK_!#ZEQoEHYPCaC93 z9PA>CWd z%DtkC(_-d4zX_HISwT9KFP(H6TXjjg64P^XP}+;wHq^mq(fSo$ong)Yzux2j$-MnT zRLBZo{TGLql?1@a&GRp;_b0joa058G{=o^XkDY>+MEKA&{tLDJl?MKZ{6Qk*_@6xS z5&0ug{Oj-^kvRZ>eSnR_hxa(d}X|Rfz1f!gDR{Q1XGn+ZPHB zh$!npP+`JX+bABbZ0-jO+g@AMEwm#0v>d%JrFn9xVGC)QWj zH#cA&t&l9KsXuc`=y8@Y$?PG)zk(x(G=T`j;rquD_e_nAh4n(jIlJD51fCNbf~ey+ zbw~IwO{zB;VzXdetuCxAz}xPD#Qpk+*r%3PtA%$nELyE)KE_ z8QAxRQXkaT2EvZ<%M7ED{SFk)iT}w7gg%^6Snu}~H^c$y8#H2FK--H2p909)*6_;7 z*6`ET(H7JE;bk~}P%N?`d{Y2ug~+`j3=nDx=%O01+zJ-_&a~#$_B!_@HoS){1aeA@ zxB}8d(k^7FXkj5wkB#E5Hn}h`fesZ2#;$88q68znv@ig7gFHG4d3N^vzWWxb6rlAw zBLZdoF3GdmEo+D@UV}WWj*sw6-k6niOZ@e%3_tL#D-voL>+JaEf&VC$2*_l83WRgK zf4xiqEY=bU=;wtl>kK!G9S78yW&v-*g_!EA1fP)&$<;eh1DOPs0U#+LLd5=xN>Vr^R3vDRz@t?U z(L55EGv?m+vm3>?lz3qh;yA=X;LZ2)=ug2WUb3G(<3x8{AX$fg6%>LFkrBT%f4yKs z_qoaqsvNk09ViqO&*s9q`&`mEe{dZ-YR1;7h6ZB;wB-+A8nlhytxC=76D%=*6`_9b z5*&QNIOb@LjHa5^I)h|zHsg`Y@YTICnN*H0^Df2cLNf?7)AA-}-F*022yCSfN6Wy4 z7fHTm!?MiC$FpB0t=pb|TG&`*6?M^nq*}y9Zw(O>@hn;U(lAr_S`SjR6UtFAWU|O7 zyC5C!3yk$)=2fqtIk>^S)T2$c3UoBvRPy;E?AMR}dQtofp|jmT|4q z!v}DjmyZ%JL22&kG~fFD;Sx9syfCdNeCLU>Q@uW+R2VKj^xbsyPqpq4Hs}ifTza@RCsgxqC{A zs`iuIjadCD4T^6g0LYm=OBu5Ei9xpP6_vaG0B;gY+?fl5`2{Tj!^s5)a~QmE`U?;iGk~w&im+bzv0fV_lx+lA$k;iUE{Wulv1` z&HQWX_=RfEgmItAeal-eJZRV$%F>jE|AnL5$KODq`%y__3&@p7B~E8>zI5l|(=~U? zJ#gz%Lm{o&1S5P0^jYh(vY?L|BTN_KaubG?C> zvjT-%Ws}r~GS_;Ms=vjwFh6)zQKf6wd7V~}PiD+ZHOL)Ww@P@Z7RBnKq>xH(dZpx> zy%ri4{O6wZd0-1;0qn>!c1~8)+c&nXhsnN{o{B33IZhw*U1&@Vb`N_GKO+?W@f@nt zgF!UXjvq?XrcfRI1plYKuZoJJTerpC-66PJBMreVxVr^w+}%4k!Gi{OcXxMphd|H} z+?`zhV|(v&{)hW?UheJD^-!y-zO{OdZ?3B9@y$8MW)y143jC}-edU-_@a~I)G0^IhuOYC}xEUBpokFYiY{l!xB5#cF6>@RU##)2V+P^zX=M|`U;>a9lCI0jo z3>{m75W!;DUO!v(Q7d97X1{C5L){BAKr+czNmz{v7m=JW$9u|HwrI`b;$wJEb11Sv z7cXA_EQCYV_TsE!n$=RacO1j3)DZAge0)z{ocyW167t3aPQVom?a{vo`$fGru+K}A?H}7s+93b$XY8>`bogtTH@ZHu>E){ zeVkD{6yHjGTBONEATE}{cXLEaN5y&Wfc@B-Z;PoUhYhwa#mjdQ9|(D0>jZuU>rVbG zT-_YQDW>#iTdqHu&3rPb0Z&KBpN63#p1FvznXRx5)}`{BePS`}AI6BeV@UB9dORzm z5Q8bG9aPbOe(`25MJ}Ca3P*6()59*<{ah{|LVJdJH;{+kkY&c!WWyx!24@)8?#4U2 zTBfq4LF4@NE8glc>#=?{nW8?*UQH3C}wBP zBp>IXMLCatQ4fh!Q{8ilMb+L}ErGA}67KN%RQM-kp%bZ605k0f)8$OQ!wZFPidaXQ zK7Y>n68`T<@xpI<)l#CIVBzNH!H~H9kTbqi)wBi?LkIP0fPhe&Nl=CV^VbPzk2TeW z!Pz07Rd#X2cEM<%6q>Km9=YE)1zE)ryn(eWb|`5V0u8dSrEJkATYGJL=iR4r;-%9k zO8RFsn|s_+*l(O)uCehNAn^d~tX*RIeI%ZiaEbRKX45ENL|pk6z@Ha3^tAGX;&|;= zk4e!BBL~7cZKbVy*R@!=NS5n;slF*W_9#JWXJ}454o=X)An?FAY75Kx6|-YnkN7qa z$P#2Z|GLF0^d11>nsZrd>S=botRp&m00bgSg4OZ2Rr=!~V0W4EY^o}1oiT%Cj){T; zCw&}XY9dSZa~H$!!F(_VpI@Eky+59l*zoHpQ|)|;=VRh7~NIE`w!3iBL*j~r+{Leg-dp)(-gmO?zL88)Q6 zg#IP?qv?F9+lBVDeD5mys^wsoj~TWOT^TthGbPeBl!}Vp8(4Ff31i7v8Lk=2X>lf# zH*I3R!48Uvb}_!Jb9;glIAR{Z1W3fOGYIuTSuS8stDPco#>Ec$fKKD> z3Um%1rv8#C2cFocb2@h$U*89~DwTv+ik(W`Tlk*AC~=-029*Uf7(pADxg3Ldn+AO| z0`di6@JFqE$Y)ptc-ha4Sl2K~y0)x#NOaLkjd)jsCZ|LVEP!<6aa8NmwB9q_$xS?1 z)Y~llm)`6oK}_1S%#^BXp<2rm2lK3CVdir7So9qD4zYVP63JD|dG2qe-}lr73eLQg z#%J%7+C0Wc2rKgn`2^1r!SG2oq{lv%J3l*zz~Mz5J^jCoc*fgnQ9+q+(PIojC@mnf8^;M*@Js&46g~`!5HubU9X>Ey-L$0!Hv-=UzChpc-tA+nOUQc zf{Dp(POtS9?&B>Y80l$z#=-4*v9Q3Xa?kOd(EzwwjNo=P>JAS{hW3?m$z^1wq=X1jf^LbF0n-r~vL(>U z(xwz&@bOb`Ea*4zDdH7z)*!*l9+gYiE4V)Y!5H~4lgV6xM4fhz!+c}E78Xy{2}WGu zXp&Lt*bS%S?rRc4`^gGKY{WIsF5%RJa~a4ZdZ<6;X^`_d*+a$cC(@Y~Z6JGOn1uM)MnOjc2w>Q_cs)siq6T##!sBiD{TB=L6-_QFrPIo z9PaMTajH*BLv+nM1VO^3@!3U72@Ksp0;SgpE4goRW+uAu3?R!_b1&aJG#xK$xd>T3 za9OjRc&(-tR&-yWZO5E?D@ z4cFwO;uvo$g%K6d>*pqbwl2*|iI1F2LV}W)Q*Ne+12O>TFr4Vy&l7ODk?dfntz7#I z&!_F94`ApH2?2i8`tO6|`#6-67amU#O|+TrTK6-2;@hpQXChSlDq;6Jm(_q9m2c$g zB@^-St%=?z3NP`ni(gkzE2Vbjx&TTt)X5~8NY#c>*&@cP#mrmjaYJ5GyI#r&DWg^4 z@r!GFLy9Ig!jk?K8cX0moKi*k0@l-F&P|GmUPFk+*tJS)*oT`MSo)lXK^k152xT$h^%MXyFjddv7ODX&Y4jRQEK0|jOu;j> zYyZd+jMrsqm*woeu_Vd$rxcPGS=f?wQ3@oJszR=OGe*^{NsZ2lBX{SK-i!TamUGWd zNgbHNIxR%fNf6#px`=`M{)YLyj+M(NZG<@K6{petFfetf^WK(elwzWrbl1O4U z6D`cB)yVBwPj?t>$xHqVaWd1HtTQr55(cw_u}ghF*bB^n;O|P=o~4=KG7fy`aSgPSIro15kcdR3 zm`B<`$*zsXrOEue{O|jI8F(jZRITj!x+sMou9Qj(tC{;5KG3y7k_qS7B(z3})#i?& zK^C3=nC?DbFinfth=<-qxbi-bueEOugeGlc&kM}!NvSY;(8U_? zkkq_MVUDx!qT+g7mKGt}cp@wQ=Fi8|LCCgb;Ud2arlN5sw5a1pKj)}>*=mF39Jk-o z`yS912V3!jODY&iZGa?r>F2sXdis7fJ22p0QLj9fh=m|!>KweA+trfyik<9)=DRg$qCM6!o{(uKu4-{$`XBX9*3LY#lBVTpI^leroE5E+>7ig!pm;n z7gII`^Kl>j5dWU|CF4{5&LkU2z3minq#bV5Pnot%-l&^*7xfdVLE~b@ay`;AJj%=w zqRH?17RT=nl8Vy?O%e+JUmH{-x74Ic>(5w@aHTj*X64o1wuj?_a*5JOv#iYAgBOi( z_82HAzr3oO-ZAsQWt!2Udxr5VIB#Xbmy-Wix~jqO$t%?^qU z0{1pX`pR)LGfC=YT8>!@v!yQ$V9yRG-@SI_irEkG`FvJ^0Bo1LY<}kiu&0e}y#ibRl#v$A63kYvT?T4CXEk5Su8H(HcA4cFG8GbuJKLL^ zV`4TfICiPIN!(iQ>D3;GT zUy9p#sc?CICeYiedf+liK1q}w9>i~<;tK6mEl;C6g@M%Za3R|KNdpOaS1DBr58gD) z$Wb!>MV}}S7t6Q$+6Mgt-sT|lQ$ZNxF!p@WUf;DLn*x;5WXfx4$SZ9w8Bm@q8nNl&CWu%G-OtS?NqdbA}I^+9HeIH zj9HI(LY(EjUwQ2_esrmG+R&xi?O7rry&dVdP2(igF4X94qH1ozfCH;pgVZB+{&;9$ zGJ$hTh#61~uLhC|%>?Ho~Ru8ogM0D*F^N&^#R8tj2nl>W3b zT9gpTbN7`15YTI`hQfaiyns;L9C{gA?>EI0A1%@<{OBVqX~vs^jEYS?rVW~!0P3*f*utWoz$h=+oudy68i5D7JZD-I>{z>~mFB>?$ij zdeE&`zucp6O6j&H*=+9;zL}d$dbh>ks@07#ugou~$j2PXQwZ+(VIKzHoEMokz@Z_4OaGmip3;5zsboQTQF&4~G7_Egdfg%SFa-9NE zqr9y7grlXc-;dhdOlCs6H~9PX#iN)bi=e2Y`!c((BX87RYb@feo;6bKfgWoCMe#Cp zsy=NhCeW!Ao7GZBE%-MfE8TocCPL{PQ(}gj3oJFO#xJT_KgzV(i@Y#fT<_W4bOtRd zJ*8BVJ8R~Ia|N8Kg*hHUoS*}3iOiPS9 zYQTN`qvu@7uyflU$2p_uDoY1_?m!sarOLZNjSGQKC88o z?eY?n`ax(@TIlaMR+L6iY90u$++zwxx=XxbEtzgQ7ZNHskzjDv7Sje*5SgiQidx~Le#H=7 zG0!JhRYWj4bMUvk6)l~Qq?>c7EpL2lgb8Bo5HIP)@&|R%4!Dtk;L2ZCA6~WOBKUlr zZDuU8H>JNMk&%1_-RJ2pZ+NbKd9s#)SE@&-NbBEgyXIY+P(W7glPy?mQzYW7APzSdDXxs2IW>J(iD4$4Y!u0#m(((Z_>6z=A@(N*=+Z~^B-p=u4-Fb5n zKXt-_7;TqWz2=~7y3)YSN;J9pT|5>?_LntF|MpENt%_~}%RRO4T&GqTlp2Jx1e)wS zO)D~B=ocL_Je#sam zLM-S|JKcwMb6Opy#GgBU^Gk)(nj0TZiWZe(=^;|=w!AI4K0PqbvQ#f26%BMfxAfd9n5 z5D0<$S~A2rB6o}m?61~(jQZ@AC2kb)Y6H1H%G^?sn_`*3GJ?zcah123C;O<&vVeb( zHN`}%pjl&)A_v^1Z^h(}$Rak_jI{y$0aqFAlGH2#`n`+bVdR2zk4LcjWf+^}}vO7%T>+#95y zm_#QWrjGX~T%`j;cLkcJIJvrZt!-Vw!U@uH2gnlE1M5|8Ynv_nXW|6IdmMI^+p?Kg zv7f^>>GN{&qghKPmblfi6JY7?JfEzF>Ra04KOO#O|m$_jATB6a(1$>JOdFYk+loI)V(l6qL;2zsTd|`I9`HyzU;|>oAZB^t&J`T z27Y*4P`J!Ag(HgzfB52BT@$fgz6DTeSB{8KwV!ftN7l*T?r6~*c}^M`^?zG0q3z$I zPHZc7D5&J98thy6o!OzDmQrF4^n0*8_1`@LR30tq54)Z;n3TyPns-o)Jn0!O(fg%R z{`~Wh(O9iISQ@KYY(xhSPNCBkr+M5>7a90&!6ySBb8Q*!3=!X7jK&e1Z5^KUrTG)8 znlGae0}nd8U*R%EqIDzBH1nDA5$25U^&D@9?1$eEIX5>e`Hmc+Xbjj;utZ1O8gIRU z)pdHQylr7=I_z_8h(?)vOkY$6A@ z&S+aI6G$ho=$IAOA7PHo>r^$zX5uK_bjZ$~F%`|i!=w?rtuU5sY!ULwAIjU)QNk$r zXK|UnbI}t1T$BKAUh4Lu?PGY2$Ocmw=p=Ex=%)Lt!q!&`BM_Zt*vW0#$57N8P<1f~ zIPdLXd)MCD8@tI$fCHB#D_U*8PoZQlkj+OcK8=#w^AD^_)%pnf13qYLyYB;F-oykOP-Mvbmt`+-e8KhZ5k$ z^W?b~l~-JXCQJv4j=VHIXX-z21O?jh6&KSVvSYMIyipedGh}QWM=-W}B$$S-A-VWT zNLq?)keq~4GO7`29KCfNKCEo(D=wqCWu0ey&3pL}Zr?1^x*zYrCN`BT4ij?W6l9En*{Zy$-By~wHR?$#a49Ilo9uhZMC^%EJg@=lgsPX4NI_GQPv6nu_rTzP=$of_v2!PmJd3-*u*k@<6Okmz-3_=LVaq0hL1sP7kZttXK|JNV4%c_Is9Oh z@pOr@I}xiEm6H7cP1ju+K$iiJxx)-2u#zDEg~lmY35c$*l%gUEjRR(=bR#*8wulk; zs0qgZ+3^+O4fb)2G+UrBrk*+mhL+s^GsCZ@+|ROYLny!xC|e(H)whEe05Eg~fVmm$ zTg;wRiGi;@Uz7LniSjahb1f#mqNB=4xm{rISo}(eQ}jrQNKniu#uDO{ecC9QIh0cH zwO8MB$v4R+PI}m&-**A$(=#D?tDkRP0tV{k84{^@iqT?aDb)n0SC(+OD* z0!JfT)2HDi7oYC?=J9!59u@P8mgCd#z` z1{VL#A^e~DE&pl|p7PFb32T;Bf)<5B;rgrC=kMx+-oeEFe+Q?BGNUEe)&NzijGSuZ*;5NF2VlMFZ@|it$E=#mB^0T# z5fR<(XL*@n$jrNz%rvn9G>f@k!h*RkqArMld89x``IXz&lanQ}cQM3UYHX({^r2F6 zHuk9(;GpO0S7GO?bt2C`FN_aUVm(2EqcGZ7_)o#tJ+~+;UU8+nKy=O^P0{5X#}Ufh zHmIz5-C?I6o(Jz@;~#81F{jeQw?D^KZ(NS5{RZ0XNE==Kyh{qSCw?j{u$lOhLZ%H( z1V5YhA|{@gkWwoO4McDA^mutsZNDk_q3k+HaEhKjG^2I#I|km9|F6t{7AV{}WWW_t zD!uWNwOIb>((QXMH@qI~ar5*W_pquSJQ#_2Ws%-LB&^{fnd}CN}Z+8n$YTD}p z{qnj*`ERoQzwB}U-DU2dr91z7-u@-si4Xp#G*w0jdwagU#?WsP#F&iA9zI`H>fek_ zxs}#BU5Z(tvZ%gW$ZDU9smo3`b@fPT-hVVoe@kghstjG6$M!i|bGGww&#t`EP`gJ* z>#!bsZLNd;?Gsmf?uJbP4BPBp70$n!Sq_YCI`rOJo(zY6(Y*J8e*(K_wo)nu{Rt=u5Le%X81c?Qm&x zm`1%#{$ob;=gpncb0LA@Fw2(>ji&N+4|lNKBunM=@a~U5xZPQkUdDD8(#*Hd=|P}f z2W~?S2l7wj1=FyXjNtXLteEsU_w<8JDVHgHTSJ`45uQb`4a4cm8Jy+s9N&o8xl78l z@s&YN(Ez#6J$an!-`fHi^-0uv2WJH^Srr@ET7YTA+VTLd9kJFxDRja1tQ}8OlphJ7c7kN0IL+@PTMev^gm!?Mgka)rO zR4}mBU3)NqeP1DYk&xW0qzfy2By*7v6O`?W^-?guO;;I|&E+K>6x(Ha{~+)n557aa zQpySt>UZ#aSMAIud|Z52?aWSdF@HH&otTV0%*UZkSM{K6;(#KZc zNg*97vLO#{^MJvpU*kWH`DJ&5S1(ixWD?@C6P%zNB&PO}jch9|dW`(BIGY;0w|NL$ z-E9(d{o&1m$7cU-P#dbG>U+qN6}pC|@XTu8t^SLkU2O}jU4l8cZYyDjUR85_-~?_k zxir|t|9x!X5K`nY$XH(0WZGSQQsb~y*_`Z5wFGLtwAK65+_V$b+%JzRf@*RfKWFS< zCl)q2*V(XuKy5@F0re9LGFn880j*u+r(@fW$WJJS--Q#7;1yU-7$h{ag5wH`h^dG& z;b-F|R#0$d;Z76=aPJ9@;HqHC#b`~-g30KNiXlL4ipKxFv=LaWsJ>G(7nQ3cdBHq} zSWSO6N5a#?7)pizIh1t*Zzrc_BW* z6Wzh`9p?Fil^sc*#3Pu>$W_lc#bd{DhpCJnFdVzbGdP@(&LkmIqM*a_!7s7l*pegM z@q)qGQU-#&Pu_U6Mt_`=9N}n9z`6Z3#lFKRpV@9qSJb|CDmdRx6)Yp0kFecsUOdvw z*dYyFjMR7_-bFm>V=sIa|AGHxvXqeX$MX(>+jI#VoMS%WL0`yUP>e7mmw}b-{ncX2 z$(+MV8xO~V{IU5bK+$ms@%rdw-iysrT1`mnA${tpcbaN!vX_l8rTnovw;SdUxA`2% zS-sfUQ9J|=9~Chwtf(<%b;(1p?pJQEC)wnKH)Ch_S#DI_-@t;0`XA3ae@`Cb=HmSa zcj(=k4UIDbd;!ti_)+OfM-hX4WU>Py@>GX>8)pmp?#Cuqh|groEn(QY4XJ0Q<|n00E)|dvC2U2rYZ$eQ z9{egsV?J-m&Moy6PnXZI{<=jr^=US&lGM41@zcsvG>@1`!F2ry5dkC@KJ4pQ3tmQ|0V(diJHJB z>gH$b_G$;OYn2DgE;~0yd0eG8y`Eb?th;0*Xi=c>L6_j z4=Yzo3QI3Xkh3$${Qa;T$jigQ+05nN!YMeI{Ug$U52UOvsm&_oW@~F=Y;XU5U!YgF zv~r<%|2mqPzei@K&;&WVSb-hvDFCb-T<;!*obSW{7EV?k77kV}cGh>2fj<3TBapXp z2T^dcb8_l4P}n#cyIQ({j49mzy7v829##M=*ZZ)PogK{FO#kZT-=qI$jQ2xoZYI_s zQ`dhRu)3A2?fVaZZ&i0Tc60=pQE;R<*^Hg>jip?KF0U=w$+b+fZ~p?D{IC_979tV~_sC-Pl2UPy?7O^ZU0 zf`bC^zP(FZt66!2C^+8P^=vW{6nX*x0GAoNsX54)gPo6?otN9xl%1OwWWommm;g+G z940~nKt2E$uQ7lNWX^7E1~LYi0ZqYN=6sxhzxGW`xY&gZ{$<@THGzPnl?oLnLXk&7 ziAtu3mxP_jLZn>v?ma=n7)?i^1WkQe{2~*oN-;Weg<4UBEm7wYunMAx@5iqwB4HLH zWd4E5Q%@)nQ4}DJ&5JSm0xRDIQx0GL^A6&h?2?g_O)wCA{-K61Qj~fc4x{t}qaqa2 zSg4(GB{`j4SEP(04PKgf=5^YS1bcMSLkS9&HrZBlXK> zPY+NwCjLSCtZ;!F8^T$H)b)QRGsWM3z<=NTzt;Ny`~P2?>c8#i|DpLua&a|wcJ+K; Rfsx)tL6B%@B$cF){ts=rV7dSR From 70196d3c88de10e234bfca0f1e6bab71fb5f01a2 Mon Sep 17 00:00:00 2001 From: Miguel Chavez Modena Date: Fri, 10 Jul 2026 12:08:42 +0200 Subject: [PATCH 5/9] update gitignore --- .gitignore | 2 ++ .../p_adaptation_policy_error.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p1.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p2.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p3.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p4.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p5.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p6.bin:Zone.Identifier | 0 .../p_adaptation_policy_error.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p1.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p2.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p3.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p4.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p5.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p6.bin:Zone.Identifier | 0 .../p_adaptation_policy_error.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p1.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p2.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p3.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p4.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p5.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p6.bin:Zone.Identifier | 0 .../p_adaptation_policy_error.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p1.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p2.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p3.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p4.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p5.bin:Zone.Identifier | 0 .../p_adaptation_policy_error_p6.bin:Zone.Identifier | 0 29 files changed, 2 insertions(+) delete mode 100644 Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier delete mode 100644 Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier diff --git a/.gitignore b/.gitignore index ce380f3ce..bcad5fda0 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,5 @@ doc/doc_output *.toc *.blg *.bbl +*.diff +*.patch diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_Overenriching_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier b/Solver/test/NavierStokes/Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier b/Solver/test/NavierStokes/EulerRK3_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p1.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p2.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p3.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p4.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p5.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 diff --git a/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier b/Solver/test/NavierStokes/IBM_Cylinder_pAdaptationRL/policy_padaptation/p_adaptation_policy_error_p6.bin:Zone.Identifier deleted file mode 100644 index e69de29bb..000000000 From c5c253d6398eaa2a4709fd925aaaa91ab3bc2840 Mon Sep 17 00:00:00 2001 From: Miguel Chavez Modena Date: Fri, 10 Jul 2026 15:10:24 +0200 Subject: [PATCH 6/9] remove if OpenMP --- Solver/src/libs/discretization/DGSEMClass.f90 | 16 +-- .../src/libs/discretization/EllipticBR1.f90 | 36 +---- .../src/libs/discretization/EllipticBR2.f90 | 44 +----- Solver/src/libs/discretization/EllipticIP.f90 | 46 ++----- Solver/src/libs/mesh/HexMesh.f90 | 107 ++++----------- Solver/src/libs/mesh/IBMClass.f90 | 34 ++--- Solver/src/libs/monitors/SurfaceIntegrals.f90 | 125 +++++------------- Solver/src/libs/monitors/VolumeIntegrals.f90 | 25 +--- 8 files changed, 98 insertions(+), 335 deletions(-) diff --git a/Solver/src/libs/discretization/DGSEMClass.f90 b/Solver/src/libs/discretization/DGSEMClass.f90 index 01e09bfc8..00851690e 100644 --- a/Solver/src/libs/discretization/DGSEMClass.f90 +++ b/Solver/src/libs/discretization/DGSEMClass.f90 @@ -643,12 +643,9 @@ FUNCTION ComputeMaxResiduals(mesh) RESULT(maxResidual) R6 = 0.0_RP c = 0.0_RP -#ifdef _OPENACC !$acc parallel loop gang present(mesh) reduction(max:R1,R2,R3,R4,R5,c) -#else !$omp parallel shared(maxResidual, R1, R2, R3, R4, R5, R6, c, mesh) default(private) !$omp do reduction(max:R1,R2,R3,R4,R5,R6,c) schedule(runtime) -#endif DO id = 1, SIZE( mesh % elements ) N = mesh % elements(id) % Nxyz @@ -703,12 +700,9 @@ FUNCTION ComputeMaxResiduals(mesh) RESULT(maxResidual) END DO -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end do !$omp end parallel -#endif +!$acc end parallel loop #if defined FLOW && (!(SPALARTALMARAS)) @@ -784,12 +778,9 @@ subroutine MaxTimeStep( self, cfl, dcfl, MaxDt , MaxDtVec) TimeStep_Conv = huge(1._RP) TimeStep_Visc = huge(1._RP) if (present(MaxDtVec)) MaxDtVec = huge(1._RP) -#ifdef _OPENACC !$acc parallel loop gang present(self) copyin(cfl, dcfl) reduction(min:TimeStep_Conv,TimeStep_Visc) -#else !$omp parallel shared(self,TimeStep_Conv,TimeStep_Visc,NodalStorage,cfl,dcfl,flowIsNavierStokes,MaxDtVec) default(private) !$omp do reduction(min:TimeStep_Conv,TimeStep_Visc) schedule(runtime) -#endif do eID = 1, SIZE(self % mesh % elements) N = self % mesh % elements(eID) % Nxyz @@ -885,12 +876,9 @@ subroutine MaxTimeStep( self, cfl, dcfl, MaxDt , MaxDtVec) end do ; end do ; end do end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end do !$omp end parallel -#endif +!$acc end parallel loop !!$acc update host(TimeStep_Conv, TimeStep_Visc) diff --git a/Solver/src/libs/discretization/EllipticBR1.f90 b/Solver/src/libs/discretization/EllipticBR1.f90 index 6c9a74bd1..20850b5c0 100644 --- a/Solver/src/libs/discretization/EllipticBR1.f90 +++ b/Solver/src/libs/discretization/EllipticBR1.f90 @@ -170,20 +170,14 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) ! Compute Riemann solvers of non-shared faces ! ******************************************* ! -#ifdef _OPENACC - !$acc parallel loop gang present(mesh, self) async(1) -#else !$omp do schedule(runtime) private(fID) -#endif + !$acc parallel loop gang present(mesh, self) async(1) do iFace = 1, size(mesh % faces_interior) fID = mesh % faces_interior(iFace) call BR1_ComputeElementInterfaceAverage(self, mesh % faces(fID), nEqn, nGradEqn) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do nowait -#endif nZones = size(mesh % zones) !$omp do schedule(runtime) private(zoneID) @@ -192,11 +186,8 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) enddo !$omp end do -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) async(1) -#else !$omp do schedule(runtime) private(eID) -#endif + !$acc parallel loop gang present(mesh) async(1) do iEl = 1, size(mesh % elements_sequential) eID = mesh % elements_sequential(iEl) ! @@ -204,11 +195,8 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) ! ------------------------- call BR1_GradientFaceLoop(nGradEqn, mesh % elements(eID), mesh) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif call HexMesh_ProlongGradientsToFaces(mesh, size(mesh % elements_sequential), mesh % elements_sequential, nGradEqn) @@ -219,27 +207,18 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) end if !$omp end single -#ifdef _OPENACC - !$acc parallel loop gang present(mesh, self) async(1) -#else !$omp do schedule(runtime) private(fID) -#endif + !$acc parallel loop gang present(mesh, self) async(1) do iFace = 1, size(mesh % faces_mpi) fID = mesh % faces_mpi(iFace) call BR1_ComputeMPIFaceAverage(self, mesh % faces(fID), nEqn, nGradEqn) end do -#ifdef _OPENACC !$acc end parallel loop -#else -!$omp end do -#endif +!$omp end do ! -#ifdef _OPENACC -!$acc parallel loop gang vector_length(128) present(mesh, self) async(1) -#else !$omp do schedule(runtime) private(eID) -#endif +!$acc parallel loop gang vector_length(128) present(mesh, self) async(1) do iEl = 1, size(mesh % elements_mpi) eID = mesh % elements_mpi(iEl) ! @@ -247,11 +226,8 @@ subroutine BR1_LiftGradients(self, nEqn, nGradEqn, mesh, time, GetGradients) ! ------------------------- call BR1_GradientFaceLoop(nGradEqn, mesh % elements(eID), mesh) end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end do -#endif +!$acc end parallel loop ! ! Prolong gradients ! ----------------- diff --git a/Solver/src/libs/discretization/EllipticBR2.f90 b/Solver/src/libs/discretization/EllipticBR2.f90 index de25c78dd..740767dff 100644 --- a/Solver/src/libs/discretization/EllipticBR2.f90 +++ b/Solver/src/libs/discretization/EllipticBR2.f90 @@ -199,20 +199,14 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient end do !$omp end do nowait else -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) private(fID) -#else !$omp do schedule(runtime) private(fID) -#endif +!$acc parallel loop gang present(mesh) private(fID) do iFace = 1, size(mesh % faces_interior) fID = mesh % faces_interior(iFace) call BR2_GradientInterfaceSolution(self, mesh % faces(fID), nEqn, nGradEqn) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do nowait -#endif end if !$acc wait print*, "I am in BR2 line 232" @@ -225,20 +219,14 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient end do !$omp end do else -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) -#else !$omp do schedule(runtime) private(fID) -#endif +!$acc parallel loop gang present(mesh) do iFace = 1, size(mesh % faces_boundary) fID = mesh % faces_boundary(iFace) call BR2_GradientInterfaceSolutionBoundary(mesh % faces(fID), nEqn, nGradEqn, time) end do -#ifdef _OPENACC !$acc end parallel loop -#else -!$omp end do -#endif +!$omp end do end if !$acc wait print*, "I am in BR2 line 254" @@ -256,20 +244,14 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient end do !$omp end do else -#ifdef _OPENACC +!$omp do schedule(runtime) private(eID) !$acc parallel loop gang present(mesh, self) copyin(self) -#else -!$omp do schedule(runtime) private(eID) -#endif do iEl = 1, size(mesh % elements_sequential) eID = mesh % elements_sequential(iEl) call BR2_ComputeGradientFaceIntegrals(self, nGradEqn, mesh % elements(eID), mesh) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif end if !$acc wait print*, "I am in BR2 line 278" @@ -290,20 +272,14 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient ! Compute MPI interface solutions ! ******************************* ! -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) -#else !$omp do schedule(runtime) private(fID) -#endif +!$acc parallel loop gang present(mesh) do iFace = 1, size(mesh % faces_mpi) fID = mesh % faces_mpi(iFace) call BR2_GradientInterfaceSolutionMPI(self, mesh % faces(fID), nEqn, nGradEqn) end do -#ifdef _OPENACC !$acc end parallel loop -#else -!$omp end do -#endif +!$omp end do ! ! ************************************************** ! Compute face integrals for elements with MPI faces @@ -317,20 +293,14 @@ subroutine BR2_ComputeGradient( self , nEqn, nGradEqn, mesh , time , GetGradient end do !$omp end do else -#ifdef _OPENACC -!$acc parallel loop gang present(mesh, self) copyin(self) -#else !$omp do schedule(runtime) private(eID) -#endif +!$acc parallel loop gang present(mesh, self) copyin(self) do iEl = 1, size(mesh % elements_mpi) eID = mesh % elements_mpi(iEl) call BR2_ComputeGradientFaceIntegrals(self, nGradEqn, mesh % elements(eID), mesh) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif end if #endif diff --git a/Solver/src/libs/discretization/EllipticIP.f90 b/Solver/src/libs/discretization/EllipticIP.f90 index 0ab056795..d71bf7931 100644 --- a/Solver/src/libs/discretization/EllipticIP.f90 +++ b/Solver/src/libs/discretization/EllipticIP.f90 @@ -265,20 +265,14 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO end do !$omp end do else -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) -#else !$omp do schedule(runtime) private(fID) -#endif +!$acc parallel loop gang present(mesh) do iFace = 1, size(mesh % faces_interior) fID = mesh % faces_interior(iFace) call IP_GradientInterfaceSolution(self, mesh % faces(fID), nEqn, nGradEqn) end do -#ifdef _OPENACC !$acc end parallel loop -#else -!$omp end do -#endif +!$omp end do end if if (HOElements) then @@ -290,21 +284,15 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO end do !$omp end do else -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) -#else !$omp do schedule(runtime) private(fID) -#endif +!$acc parallel loop gang present(mesh) do iFace = 1, size(mesh % faces_boundary) fID = mesh % faces_boundary(iFace) zoneBCName = mesh % zones (mesh % faces(fID) % zone) % zoneBCName call IP_GradientInterfaceSolutionBoundary(self, mesh % faces(fID), zoneBCName, nEqn, nGradEqn, time) end do -#ifdef _OPENACC !$acc end parallel loop -#else -!$omp end do -#endif +!$omp end do end if ! ! ********************** @@ -319,20 +307,14 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO end do !$omp end do else -#ifdef _OPENACC +!$omp do schedule(runtime) private(eID) !$acc parallel loop gang present(mesh, self) copyin(self) -#else -!$omp do schedule(runtime) private(eID) -#endif do iEl = 1, size(mesh % elements_sequential) eID = mesh % elements_sequential(iEl) call IP_ComputeGradientFaceIntegrals(self,nGradEqn, mesh % elements(eID), mesh) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif end if ! ! ****************** @@ -350,20 +332,14 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO ! Compute MPI interface solutions ! ******************************* ! -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) -#else !$omp do schedule(runtime) private(fID) -#endif +!$acc parallel loop gang present(mesh) do iFace = 1, size(mesh % faces_mpi) fID = mesh % faces_mpi(iFace) call IP_GradientInterfaceSolutionMPI(self, mesh % faces(fID), nEqn, nGradEqn) end do -#ifdef _OPENACC !$acc end parallel loop -#else -!$omp end do -#endif +!$omp end do ! ! ************************************************** ! Compute face integrals for elements with MPI faces @@ -377,20 +353,14 @@ subroutine IP_ComputeGradient(self, nEqn, nGradEqn, mesh, time, GetGradients, HO end do !$omp end do else -#ifdef _OPENACC -!$acc parallel loop gang present(mesh, self) copyin(self) -#else !$omp do schedule(runtime) private(eID) -#endif +!$acc parallel loop gang present(mesh, self) copyin(self) do iEl = 1, size(mesh % elements_mpi) eID = mesh % elements_mpi(iEl) call IP_ComputeGradientFaceIntegrals(self,nGradEqn, mesh % elements(eID), mesh) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif end if #endif diff --git a/Solver/src/libs/mesh/HexMesh.f90 b/Solver/src/libs/mesh/HexMesh.f90 index a96aa7b26..2e8a628cd 100644 --- a/Solver/src/libs/mesh/HexMesh.f90 +++ b/Solver/src/libs/mesh/HexMesh.f90 @@ -982,37 +982,25 @@ subroutine HexMesh_ProlongSolToFaces(self, nEqn) select case ( self %nodeType ) case(1) !Gauss -#ifdef _OPENACC -!$acc parallel loop gang collapse(2) present(self) num_gangs(size(self % elements)) vector_length(32) async(1) -#else !$omp do schedule(runtime) -#endif +!$acc parallel loop gang collapse(2) present(self) num_gangs(size(self % elements)) vector_length(32) async(1) do eID = 1, size(self % elements) do fID = 1, 6 - call HexElement_ProlongSolToFaces(self % elements(eID), nEqn, self % faces(self % elements(eID) % faceIDs(fID)), fID) + call HexElement_ProlongSolToFaces(self % elements(eID), nEqn, self % faces(self % elements(eID) % faceIDs(fID)), fID) end do ; end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif case(2) !Gauss-Lobatto -#ifdef _OPENACC -!$acc parallel loop gang collapse(2) present(self) num_gangs(size(self % elements)) vector_length(32) async(1) -#else !$omp do schedule(runtime) -#endif +!$acc parallel loop gang collapse(2) present(self) num_gangs(size(self % elements)) vector_length(32) async(1) do eID = 1, size(self % elements) do fID = 1, 6 - call HexElement_ProlongSolToFaces_GL(self % elements(eID), nEqn, self % faces(self % elements(eID) % faceIDs(fID)), fID) + call HexElement_ProlongSolToFaces_GL(self % elements(eID), nEqn, self % faces(self % elements(eID) % faceIDs(fID)), fID) end do ; end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif end select @@ -1036,11 +1024,8 @@ subroutine HexMesh_ProlongGradientsToFaces(self, size_element_list, element_list select case ( self %nodeType ) case(1) !Gauss -#ifdef _OPENACC !$acc parallel loop gang num_gangs(size_element_list) collapse(2) present(self, element_list) private(fIDs) async(1) -#else !$omp do schedule(runtime) private(eID) -#endif do iEl = 1, size_element_list do fid = 1,6 eID = element_list(iEl) @@ -1056,68 +1041,47 @@ subroutine HexMesh_ProlongGradientsToFaces(self, size_element_list, element_list call HexElement_ProlongGradientsToFaces(self % elements(eID), nGradEqn, & self % faces(fIDs(fid)), & self % elements(eID) % storage % U_z,fid, 3) - end do ; enddo -#ifdef _OPENACC + end do ; enddo !$acc end parallel loop -#else !$omp end do -#endif case(2) !Gauss-Lobatto -#ifdef _OPENACC -!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) -#else !$omp do schedule(runtime) -#endif +!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) do iEl = 1, size_element_list do fID = 1, 6 eID = element_list(iEl) call HexElement_ProlongGradientsToFaces_GL(self % elements(eID), nGradEqn, & self % faces(self % elements(eID) % faceIDs(fID)), & - self % elements(eID) % storage % U_x, fID,1) + self % elements(eID) % storage % U_x, fID,1) end do ; end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif -#ifdef _OPENACC -!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) -#else !$omp do schedule(runtime) -#endif +!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) do iEl = 1, size_element_list do fID = 1, 6 eID = element_list(iEl) call HexElement_ProlongGradientsToFaces_GL(self % elements(eID), nGradEqn, & self % faces(self % elements(eID) % faceIDs(fID)), & - self % elements(eID) % storage % U_y, fID,2) + self % elements(eID) % storage % U_y, fID,2) end do ; end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif - -#ifdef _OPENACC -!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) -#else + !$omp do schedule(runtime) -#endif +!$acc parallel loop gang collapse(2) num_gangs(size(self % elements)) vector_length(32) present(self,element_list) async(1) do iEl = 1, size_element_list do fID = 1, 6 eID = element_list(iEl) call HexElement_ProlongGradientsToFaces_GL(self % elements(eID), nGradEqn, & self % faces(self % elements(eID) % faceIDs(fID)), & - self % elements(eID) % storage % U_z, fID,3) + self % elements(eID) % storage % U_z, fID,3) end do ; end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif end select end subroutine HexMesh_ProlongGradientsToFaces @@ -1998,7 +1962,6 @@ subroutine HexMesh_CheckIfMeshIs2D(self) xNodesF1(:,nID) = self % nodes(face1Nodes(nID)) % x xNodesF2(:,nID) = self % nodes(face2Nodes(nID)) % x - end do ! ! Compute the delta x vectors @@ -2997,7 +2960,8 @@ subroutine CommunicateMPIFaceMinimumDistance(self) end do call mpi_waitall(MPI_Process % nProcs, sendReq, sendSt, ierr) - deallocate(hsend, hrecv) + + deallocate(hsend, hrecv) #endif end subroutine CommunicateMPIFaceMinimumDistance @@ -3996,7 +3960,7 @@ subroutine GetDiscretizationError(mesh,controlVariables) e => mesh % elements(eID) ) e_aux % globID = e % globID e_aux % Nxyz = e % Nxyz - NDOF = NDOF + product(e % Nxyz + 1) + NDOF = NDOF + product(e % Nxyz + 1) end associate end do @@ -5702,19 +5666,13 @@ subroutine HexMesh_ComputeLocalGradientNS(self, set_mu) integer :: eID !-------------------------------------------------- -#ifdef _OPENACC - !$acc parallel loop gang vector_length(128) present(self) async(1) -#else !$omp do schedule(runtime) -#endif + !$acc parallel loop gang vector_length(128) present(self) async(1) do eID = 1 , size(self % elements) call HexElement_ComputeLocalGradient(self % elements(eID), NCONS, NGRAD, self % elements(eID) % storage % Q) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do nowait -#endif end subroutine HexMesh_ComputeLocalGradientNS @@ -5728,25 +5686,19 @@ subroutine HexMesh_ComputeLocalGradientiNS(self) integer :: eID, i, j, k !-------------------------------------------------- -#ifdef _OPENACC - !$acc parallel loop gang vector_length(128) present(self) async(1) -#else !$omp do schedule(runtime) -#endif + !$acc parallel loop gang vector_length(128) present(self) async(1) do eID = 1 , size(self % elements) - !$acc loop vector collapse(3) + !$acc loop vector collapse(3) do k = 0, self % elements(eID) % Nxyz(3) ; do j = 0, self % elements(eID) % Nxyz(2) ; do i = 0, self % elements(eID) % Nxyz(1) call iNSGradientVariables(NCONS, NGRAD, self % elements(eID) % storage % Q(:,i,j,k), self % elements(eID) % storage % Q_grad_iNS(:,i,j,k)) end do ; end do ; end do call HexElement_ComputeLocalGradient(self % elements(eID), NCONS, NGRAD, self % elements(eID) % storage % Q_grad_iNS) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do nowait -#endif end subroutine HexMesh_ComputeLocalGradientiNS #endif @@ -5761,25 +5713,20 @@ subroutine HexMesh_ComputeLocalGradientCH(self, set_mu) integer :: eID, i, j, k !-------------------------------------------------- -#ifdef _OPENACC - !$acc parallel loop gang vector_length(128) present(self) copyin(set_mu) async(1) -#else !$omp do schedule(runtime) -#endif + !$acc parallel loop gang vector_length(128) present(self) copyin(set_mu) async(1) do eID = 1 , size(self % elements) - !$acc loop vector collapse(3) + !$acc loop vector collapse(3) do k = 0, self % elements(eID) % Nxyz(3) ; do j = 0, self % elements(eID) % Nxyz(2) ; do i = 0, self % elements(eID) % Nxyz(1) call chGradientVariables(NCOMP, NCOMP, self % elements(eID) % storage % Q(1:IMC,i,j,k), self % elements(eID) % storage % Q_grad_CH(1:IMC,i,j,k)) + !if ( set_mu ) self % elements(eID) % storage % Q_grad_CH(IGMU,i,j,k) = self % elements(eID) % storage % mu(1,i,j,k) end do ; end do ; end do call HexElement_ComputeLocalGradient(self % elements(eID), NCOMP, NCOMP, self % elements(eID) % storage % Q_grad_CH) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do nowait -#endif end subroutine HexMesh_ComputeLocalGradientCH @@ -5793,26 +5740,22 @@ subroutine HexMesh_ComputeLocalGradientMU(self, set_mu) integer :: eID, i, j, k !-------------------------------------------------- -#ifdef _OPENACC - !$acc parallel loop gang vector_length(128) present(self) copyin(set_mu) async(1) -#else !$omp do schedule(runtime) -#endif + !$acc parallel loop gang vector_length(128) present(self) copyin(set_mu) async(1) do eID = 1 , size(self % elements) - !$acc loop vector collapse(3) + !$acc loop vector collapse(3) do k = 0, self % elements(eID) % Nxyz(3) ; do j = 0, self % elements(eID) % Nxyz(2) ; do i = 0, self % elements(eID) % Nxyz(1) call mGradientVariables(NCONS, NGRAD, self % elements(eID) % storage % Q(:,i,j,k), self % elements(eID) % storage % Q_grad_mu(:,i,j,k), self % elements(eID) % storage % rho(i,j,k)) + !if ( set_mu == .true.) then ! This is not working - weird - above it works self % elements(eID) % storage % Q_grad_mu(IGMU,i,j,k) = self % elements(eID) % storage % mu(1,i,j,k) + !end if end do ; end do ; end do call HexElement_ComputeLocalGradient(self % elements(eID), NCONS, NGRAD, self % elements(eID) % storage % Q_grad_mu) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do nowait -#endif end subroutine HexMesh_ComputeLocalGradientMU #endif diff --git a/Solver/src/libs/mesh/IBMClass.f90 b/Solver/src/libs/mesh/IBMClass.f90 index e21dc8020..03338be5f 100644 --- a/Solver/src/libs/mesh/IBMClass.f90 +++ b/Solver/src/libs/mesh/IBMClass.f90 @@ -1072,19 +1072,16 @@ subroutine IBM_BandPoint_state( this, elements, STLNum, gradients ) end if !$acc wait -#ifdef _OPENACC - !$acc parallel loop present(this, elements) -#else !$omp parallel !$omp do schedule(runtime) private(i,j,k,eID) -#endif + !$acc parallel loop present(this, elements) do n = 1, this% BandRegion(STLNum)% NumOfObjs if( this% BandRegion(STLNum)% x(n)% partition .eq. MPI_Process% rank ) then i = this% BandRegion(STLNum)% x(n)% local_Position(1) j = this% BandRegion(STLNum)% x(n)% local_Position(2) k = this% BandRegion(STLNum)% x(n)% local_Position(3) eID = this% BandRegion(STLNum)% x(n)% element_index - this% BandRegion(STLNum)% Q(:,n) = elements(eID)% storage% Q(:,i,j,k) + this% BandRegion(STLNum)% Q(:,n) = elements(eID)% storage% Q(:,i,j,k) if( gradients ) then this% BandRegion(STLNum)% U_x(:,n) = elements(eID)% storage% U_x(:,i,j,k) this% BandRegion(STLNum)% U_y(:,n) = elements(eID)% storage% U_y(:,i,j,k) @@ -1092,12 +1089,9 @@ subroutine IBM_BandPoint_state( this, elements, STLNum, gradients ) end if end if end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do !$omp end parallel -#endif !$acc wait !Update CPU variables @@ -2098,24 +2092,19 @@ subroutine IBM_SemiImplicitCorrection( this, elements, dt ) !$acc wait if( this % Wallfunction ) call this % GetBandRegionStates( elements ) -#ifdef _OPENACC - !$acc parallel loop gang present(this, elements) -#else !$omp parallel -!$omp do schedule(runtime) private(i,j,k) -#endif + !$omp do schedule(runtime) private(i,j,k) + !$acc parallel loop gang present(this, elements) do eID = 1, SIZE( elements ) - !$acc loop vector collapse(3) + !$acc loop vector collapse(3) do i = 0, elements(eID) % Nxyz(1); do j = 0, elements(eID) % Nxyz(2); do k = 0, elements(eID) % Nxyz(3) - if( elements(eID) % isInsideBody(i,j,k) ) then - call IBM_GetSemiImplicitStep(this, eID, 0.5_RP*dt, elements(eID) % storage % Q(:,i,j,k) ) - end if + if( elements(eID) % isInsideBody(i,j,k) ) then + call IBM_GetSemiImplicitStep(this, eID, 0.5_RP*dt, elements(eID) % storage % Q(:,i,j,k) ) + end if end do; end do; end do end do -#ifdef _OPENACC !$acc end parallel loop -#else -!$omp end do +!$omp end do if( this% Wallfunction ) then !$omp do schedule(runtime) private(i,j,k) @@ -2130,8 +2119,7 @@ subroutine IBM_SemiImplicitCorrection( this, elements, dt ) end do !$omp end do end if -!$omp end parallel -#endif +!$omp end parallel end subroutine IBM_SemiImplicitCorrection subroutine IBM_GetBandRegionStates( this, elements ) @@ -3664,4 +3652,4 @@ subroutine IBM_CreateDeviceData(self) end subroutine IBM_CreateDeviceData -end module IBMClass \ No newline at end of file +end module IBMClass diff --git a/Solver/src/libs/monitors/SurfaceIntegrals.f90 b/Solver/src/libs/monitors/SurfaceIntegrals.f90 index 6807203c8..d2b04c234 100644 --- a/Solver/src/libs/monitors/SurfaceIntegrals.f90 +++ b/Solver/src/libs/monitors/SurfaceIntegrals.f90 @@ -70,11 +70,8 @@ function ScalarSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) ! ! Loop the zone to get faces and elements ! --------------------------------------- -#ifdef _OPENACC !$acc parallel loop gang reduction(+:val) present(mesh) -#else !$omp parallel do private(fID) reduction(+:val) schedule(runtime) -#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID @@ -86,11 +83,8 @@ function ScalarSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) val = val + ScalarSurfaceIntegral_Face(mesh % faces(fID), integralType) end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end parallel do -#endif +!$acc end parallel loop #ifdef _HAS_MPI_ localval = val @@ -280,17 +274,14 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) select case ( integralType ) case ( SURFACE ) -#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) -#else !$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) -#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID ! -------------- fID = mesh % zones(zoneID) % faces(zonefID) - + localx = 0.0_RP localy = 0.0_RP localz = 0.0_RP @@ -304,37 +295,31 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) val = NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) & * mesh % faces(fid) % geom % jacobian(i,j) * mesh % faces(fid) % geom % normal(:,i,j) - + localx = localx + val(1) localy = localy + val(2) localz = localz + val(3) - + end do ; end do - + valx = valx + localx valy = valy + localy valz = valz + localz - + end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end parallel do -#endif +!$acc end parallel loop case ( TOTAL_FORCE ) -#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) -#else !$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) -#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID ! -------------- fID = mesh % zones(zoneID) % faces(zonefID) - + localx = 0.0_RP localy = 0.0_RP localz = 0.0_RP @@ -352,40 +337,34 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) p = Pressure(mesh % faces(fid) % storage(1) % Q(:,i,j)) call getStressTensor(mesh % faces(fid) % storage(1) % Q(:,i,j), mesh % faces(fid) % storage(1) % U_x(:,i,j),& mesh % faces(fid) % storage(1) % U_y(:,i,j),mesh % faces(fid) % storage(1) % U_z(:,i,j), tau) - + localval = ( p * mesh % faces(fid) % geom % normal(:,i,j) - matmul(tau,mesh % faces(fid) % geom % normal(:,i,j)) ) & * mesh % faces(fid) % geom % jacobian(i,j) & * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - + localx = localx + localval(1) localy = localy + localval(2) localz = localz + localval(3) end do ; end do - + valx = valx + localx valy = valy + localy valz = valz + localz - + end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end parallel do -#endif +!$acc end parallel loop case ( PRESSURE_FORCE ) -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) -#else +!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) !$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) -#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID ! -------------- fID = mesh % zones(zoneID) % faces(zonefID) - + localx = 0.0_RP localy = 0.0_RP localz = 0.0_RP @@ -394,47 +373,41 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) ! Computes the pressure forces experienced by the zone ! F = \int p \vec{n}ds ! **************************************************** - ! + ! !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(val) do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) ! ! Compute the integral ! -------------------- p = Pressure(mesh % faces(fid) % storage(1) % Q(:,i,j)) - + val = ( p * mesh % faces(fid) % geom % normal(:,i,j) ) * mesh % faces(fid) % geom % jacobian(i,j) & * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) & * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - + localx = localx + val(1) localy = localy + val(2) localz = localz + val(3) end do ; end do - + valx = valx + localx valy = valy + localy valz = valz + localz - + end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end parallel do -#endif +!$acc end parallel loop case ( VISCOUS_FORCE ) -#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) -#else !$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) -#endif do zonefID = 1, mesh % zones(zoneID) % no_of_faces ! ! Face global ID ! -------------- fID = mesh % zones(zoneID) % faces(zonefID) - + localx = 0.0_RP localy = 0.0_RP localz = 0.0_RP @@ -451,26 +424,23 @@ function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) ! -------------------- call getStressTensor(mesh % faces(fid) % storage(1) % Q(:,i,j), mesh % faces(fid) % storage(1) % U_x(:,i,j),& mesh % faces(fid) % storage(1) % U_y(:,i,j),mesh % faces(fid) % storage(1) % U_z(:,i,j), tau) - + val = - matmul(tau,mesh % faces(fid) % geom % normal(:,i,j)) * mesh % faces(fid) % geom % jacobian(i,j) & * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) & * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - + localx = localx + val(1) localy = localy + val(2) localz = localz + val(3) end do ; end do - + valx = valx + localx valy = valy + localy valz = valz + localz - + end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end parallel do -#endif +!$acc end parallel loop end select @@ -1173,11 +1143,8 @@ subroutine ProlongToFaces(mesh, zoneID) select case ( mesh %nodeType ) case(1) !Gauss -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) -#else !$omp do schedule(runtime) private(eID) -#endif + !$acc parallel loop gang present(mesh) do zonefID = 1, mesh % zones(zoneID) % no_of_faces fID = mesh % zones(zoneID) % faces(zonefID) eID = mesh % faces(fID) % elementIDs(1) @@ -1193,67 +1160,46 @@ subroutine ProlongToFaces(mesh, zoneID) mesh % faces(fID), & mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1), 3) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif case(2) !Gauss-Lobatto -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) private(fID) -#else !$omp do schedule(runtime) -#endif + !$acc parallel loop gang present(mesh) private(fID) do zonefID = 1, mesh % zones(zoneID) % no_of_faces fID = mesh % zones(zoneID) % faces(zonefID) eID = mesh % faces(fID) % elementIDs(1) call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & mesh % faces(fID), & - mesh % elements(eID) % storage % U_x, mesh % faces(fID) % elementSide(1),1) + mesh % elements(eID) % storage % U_x, mesh % faces(fID) % elementSide(1),1) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) private(fID) -#else !$omp do schedule(runtime) -#endif + !$acc parallel loop gang present(mesh) private(fID) do zonefID = 1, mesh % zones(zoneID) % no_of_faces fID = mesh % zones(zoneID) % faces(zonefID) eID = mesh % faces(fID) % elementIDs(1) call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & mesh % faces(fID), & - mesh % elements(eID) % storage % U_y, mesh % faces(fID) % elementSide(1),2) + mesh % elements(eID) % storage % U_y, mesh % faces(fID) % elementSide(1),2) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif - -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) private(fID) -#else + !$omp do schedule(runtime) -#endif + !$acc parallel loop gang present(mesh) private(fID) do zonefID = 1, mesh % zones(zoneID) % no_of_faces fID = mesh % zones(zoneID) % faces(zonefID) eID = mesh % faces(fID) % elementIDs(1) call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & mesh % faces(fID), & - mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1),3) + mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1),3) end do -#ifdef _OPENACC !$acc end parallel loop -#else !$omp end do -#endif end select end if ! @@ -1266,4 +1212,3 @@ end subroutine ProlongToFaces end module SurfaceIntegrals #endif - diff --git a/Solver/src/libs/monitors/VolumeIntegrals.f90 b/Solver/src/libs/monitors/VolumeIntegrals.f90 index 55d60f112..353d9b78f 100644 --- a/Solver/src/libs/monitors/VolumeIntegrals.f90 +++ b/Solver/src/libs/monitors/VolumeIntegrals.f90 @@ -607,11 +607,8 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) case ( VELOCITY ) ! num_of_vars is equal to NDIM -#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(9700) reduction(+:val1,val2,val3) async(1) -#else !$omp parallel do reduction(+:val1,val2,val3) private(val) schedule(guided) -#endif do eID = 1, mesh % no_of_elements ! local1 = 0.0_RP @@ -648,19 +645,14 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) val3 = val3 + local3 end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end parallel do -#endif +!$acc end parallel loop + case ( MOMENTUM ) ! num_of_vars is equal to NDIM -#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(9700) reduction(+:val1,val2,val3) -#else !$omp parallel do reduction(+:val1,val2,val3) private(val) schedule(guided) -#endif do eID = 1, mesh % no_of_elements ! local1 = 0.0_RP @@ -692,11 +684,8 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) val3 = val3 + local3 end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end parallel do -#endif +!$acc end parallel loop ! case ( PSOURCE ) ! ! num_of_vars is equal to NCONS @@ -741,11 +730,8 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) case ( SOURCE ) ! num_of_vars is equal to NCONS -#ifdef _OPENACC !$acc parallel loop gang present(mesh) num_gangs(9700) reduction(+:val1,val2,val3,val4,val5) -#else !$omp parallel do reduction(+:val1,val2,val3,val4,val5) private(val) schedule(guided) -#endif do eID = 1, mesh % no_of_elements ! local1 = 0.0_RP @@ -791,11 +777,8 @@ function VectorVolumeIntegral(mesh, integralType, num_of_vars) result(val) val5 = val5 + local5 end do -#ifdef _OPENACC -!$acc end parallel loop -#else !$omp end parallel do -#endif +!$acc end parallel loop ! #endif ! From 0c06b85c5f1a9f48f540a233647598a348b729c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 05:42:01 +0000 Subject: [PATCH 7/9] Fix remaining nvfortran ICE in SCsensorClass: replace all Element pointers with associate Preventively convert all 'type(Element), pointer :: e' + 'e => array(eID)' patterns in SCsensorClass.f90 to associate blocks to avoid mk_mem_ptr_shape ICE in nvfortran. Affected subroutines: Sensor_integral_sqrt, Sensor_integral, Sensor_modal, Sensor_aliasing, Sensor_GMM. --- .../src/libs/discretization/SCsensorClass.f90 | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/Solver/src/libs/discretization/SCsensorClass.f90 b/Solver/src/libs/discretization/SCsensorClass.f90 index 551e2772f..e55044370 100755 --- a/Solver/src/libs/discretization/SCsensorClass.f90 +++ b/Solver/src/libs/discretization/SCsensorClass.f90 @@ -562,7 +562,6 @@ subroutine Sensor_integral_sqrt(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e integer :: eID integer :: i integer :: j @@ -572,7 +571,7 @@ subroutine Sensor_integral_sqrt(sensor, sem, t) !$omp parallel do default(private) shared(sem, sensor, NodalStorage) do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) val = 0.0_RP do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) contribution = GetSensedVariable( & @@ -590,12 +589,10 @@ subroutine Sensor_integral_sqrt(sensor, sem, t) end do ; end do ; end do e % storage % sensor = SinRamp(sensor, sqrt(val)) - + end associate end do !$omp end parallel do - nullify(e) - end subroutine Sensor_integral_sqrt ! !/////////////////////////////////////////////////////////////////////////////// @@ -613,7 +610,6 @@ subroutine Sensor_integral(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e integer :: eID integer :: i integer :: j @@ -623,7 +619,7 @@ subroutine Sensor_integral(sensor, sem, t) !$omp parallel do default(private) shared(sem, sensor, NodalStorage) do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) val = 0.0_RP do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) contribution = GetSensedVariable( & @@ -640,11 +636,10 @@ subroutine Sensor_integral(sensor, sem, t) * contribution end do ; end do ; end do e % storage % sensor = SinRamp(sensor, val) + end associate end do !$omp end parallel do - nullify(e) - end subroutine Sensor_integral ! !/////////////////////////////////////////////////////////////////////////////// @@ -667,7 +662,6 @@ subroutine Sensor_modal(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e integer :: eID integer :: i, j, k, r integer :: maxNx, maxNy, maxNz @@ -687,7 +681,7 @@ subroutine Sensor_modal(sensor, sem, t) !$omp parallel do default(private) shared(sem, sensor, NodalStorage) do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) Nx = e % Nxyz(1) Ny = e % Nxyz(2) Nz = e % Nxyz(3) @@ -776,11 +770,10 @@ subroutine Sensor_modal(sensor, sem, t) ! Sensor value as the ratio of num / den e % storage % sensor = SinRamp(sensor, log10( num / den )) - + end associate end do !$omp end parallel do - nullify(e) nullify(Lwx) nullify(Lwy) nullify(Lwz) @@ -895,7 +888,6 @@ subroutine Sensor_aliasing(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e type(NodalStorage_t), pointer :: spAxi, spAeta, spAzeta integer :: eID logical :: need_dealloc @@ -914,7 +906,7 @@ subroutine Sensor_aliasing(sensor, sem, t) select type (HyperbolicDiscretization) type is (SplitDG_t) - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) spAxi => NodalStorage(e % Nxyz(1)) spAeta => NodalStorage(e % Nxyz(2)) spAzeta => NodalStorage(e % Nxyz(3)) @@ -965,12 +957,12 @@ subroutine Sensor_aliasing(sensor, sem, t) else e % storage % sensor = SinRamp(sensor, log10(e % storage % sensor)) end if + end associate end select end do !$omp end parallel do - nullify(e) nullify(spAxi) nullify(spAeta) nullify(spAzeta) @@ -999,7 +991,6 @@ subroutine Sensor_GMM(sensor, sem, t) ! --------------- ! Local variables ! --------------- - type(Element), pointer :: e integer :: eID integer :: i, j, k integer :: cnt @@ -1016,9 +1007,7 @@ subroutine Sensor_GMM(sensor, sem, t) ! ----------------------------------------------------------------- cnt = 0 do eID = 1, sem % mesh % no_of_elements - - e => sem % mesh % elements(eID) - + associate(e => sem % mesh % elements(eID)) do k = 0, e % Nxyz(3) ; do j = 0, e % Nxyz(2) ; do i = 0, e % Nxyz(1) cnt = cnt + 1 @@ -1101,7 +1090,7 @@ subroutine Sensor_GMM(sensor, sem, t) sensor % x(2,cnt) = dp(1)**2 + dp(2)**2 + dp(3)**2 end do ; end do ; end do - + end associate end do ! ! Rescale the values @@ -1119,7 +1108,7 @@ subroutine Sensor_GMM(sensor, sem, t) nclusters = sensor % gmm % nclusters cnt = 0 do eID = 1, sem % mesh % no_of_elements - e => sem % mesh % elements(eID) + associate(e => sem % mesh % elements(eID)) if (nclusters <= 1) then e % storage % sensor = 0.0_RP else @@ -1128,10 +1117,9 @@ subroutine Sensor_GMM(sensor, sem, t) e % storage % sensor = real(cluster - 1, RP) / (nclusters - 1) end if cnt = cnt + n + end associate end do - nullify(e) - end subroutine Sensor_GMM ! !/////////////////////////////////////////////////////////////////////////////// From 8abce975a6441214971a38e34c04818225ea8d8a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 06:51:53 +0000 Subject: [PATCH 8/9] Fix GPU divergence: disable !$omp parallel in _OPENACC mode In ComputeTimeDerivative and ComputeTimeDerivativeIsolated, the outer !$omp parallel region caused all N OpenMP threads to independently launch identical !$acc parallel loop kernels for each GPU computation. Kernels doing QDot += accumulation (face contributions, gradient face loops) were therefore executed N times instead of once, producing N-fold or race-condition-corrupted results (O(10^62) residuals at t=0). Wrapping !$omp parallel / !$omp end parallel with #ifndef _OPENACC ensures that in GPU mode a single CPU thread dispatches all GPU kernels sequentially. The CPU/OpenMP path is unchanged. --- .../NavierStokesSolver/SpatialDiscretization.f90 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 b/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 index 1067a050f..498cd6f44 100644 --- a/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 +++ b/Solver/src/NavierStokesSolver/SpatialDiscretization.f90 @@ -275,16 +275,16 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) ! ------------------------------------------ ! Apply the Boundary conditions to the state ! ------------------------------------------ -! This was done in the compute boundary flux before +! This was done in the compute boundary flux before ! but it was called twice because we call it once in this file ! and one in the Elliptic discretisation. So now we compute it ! only once at the begining of time derivative and store it -! +! nZones = size(mesh % zones) do zoneID=1, nZones - CALL BCs(zoneID) % bc % FlowState(mesh, mesh % zones(zoneID)) + CALL BCs(zoneID) % bc % FlowState(mesh, mesh % zones(zoneID)) enddo -! +! ! ----------------- ! Compute gradients ! ----------------- @@ -295,7 +295,7 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) if ( computeGradients ) then call ViscousDiscretization % ComputeGradient( NCONS, NGRAD, mesh, time, GetGradients, HO_Elements) end if - + ! call ComputeArtificialViscosity(mesh) ! ! ----------------------- @@ -306,11 +306,11 @@ SUBROUTINE ComputeTimeDerivative( mesh, particles, time, mode, HO_Elements) call TimeDerivative_ComputeQDotHO(mesh = mesh , & particles = particles, & t = time) - else + else call TimeDerivative_ComputeQDot(mesh = mesh , & particles = particles, & t = time) - + end if #ifndef _OPENACC !$omp end parallel From dc5e53dad2f431d67269da2cea70ab65e3aca4f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 13:18:40 +0000 Subject: [PATCH 9/9] Build: compile mixed OMP/ACC files without -fopenmp for nvfortran GPU Add per-file Makefile rules using FFLAGS_NO_OMP (strips -fopenmp/-mp) for files that mix !$omp and !$acc directives in the same scope, which causes nvfortran S-0155 errors when both flags are active: - discretization: EllipticBR1, EllipticBR2, EllipticIP, DGSEMClass - mesh: HexMesh, IBMClass - monitors: SurfaceIntegrals, VolumeIntegrals Probe.f90 and Monitors.f90 continue to compile with -fopenmp. Also remove Monitors.f90, Probe.f90, SurfaceIntegrals.f90 and SurfaceMonitor.f90 accidentally committed under libs/mesh/ instead of their correct location libs/monitors/. --- Solver/src/libs/discretization/Makefile | 48 + Solver/src/libs/mesh/Makefile | 33 + Solver/src/libs/mesh/Monitors.f90 | 1503 --------------------- Solver/src/libs/mesh/Probe.f90 | 967 ------------- Solver/src/libs/mesh/SurfaceIntegrals.f90 | 1269 ----------------- Solver/src/libs/mesh/SurfaceMonitor.f90 | 560 -------- Solver/src/libs/monitors/Makefile | 34 + 7 files changed, 115 insertions(+), 4299 deletions(-) delete mode 100644 Solver/src/libs/mesh/Monitors.f90 delete mode 100644 Solver/src/libs/mesh/Probe.f90 delete mode 100644 Solver/src/libs/mesh/SurfaceIntegrals.f90 delete mode 100644 Solver/src/libs/mesh/SurfaceMonitor.f90 diff --git a/Solver/src/libs/discretization/Makefile b/Solver/src/libs/discretization/Makefile index ef42e362d..525127b8c 100644 --- a/Solver/src/libs/discretization/Makefile +++ b/Solver/src/libs/discretization/Makefile @@ -96,6 +96,54 @@ LIB = discretization all: lib$(LIB)_ns.a lib$(LIB)_nssa.a lib$(LIB)_ins.a lib$(LIB)_ch.a lib$(LIB)_mu.a lib$(LIB)_caa.a +# nvfortran GPU: strip -fopenmp from files with mixed OMP/ACC directives +FFLAGS_NO_OMP := $(filter-out -fopenmp -mp,$(FFLAGS)) + +./build_ns/EllipticBR1.o \ +./build_ns/EllipticBR2.o \ +./build_ns/EllipticIP.o \ +./build_ns/DGSEMClass.o: ./build_ns/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -I./include_ns $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_ns -c $< -o $@ $(MOD_FLAG) ./include_ns + +./build_nssa/EllipticBR1.o \ +./build_nssa/EllipticBR2.o \ +./build_nssa/EllipticIP.o \ +./build_nssa/DGSEMClass.o: ./build_nssa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -DSPALARTALMARAS -I./include_nssa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_nssa -c $< -o $@ $(MOD_FLAG) ./include_nssa + +./build_ins/EllipticBR1.o \ +./build_ins/EllipticBR2.o \ +./build_ins/EllipticIP.o \ +./build_ins/DGSEMClass.o: ./build_ins/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DINCNS -I./include_ins $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_ins -c $< -o $@ $(MOD_FLAG) ./include_ins + +./build_ch/EllipticBR1.o \ +./build_ch/EllipticBR2.o \ +./build_ch/EllipticIP.o \ +./build_ch/DGSEMClass.o: ./build_ch/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DCAHNHILLIARD -I./include_ch $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_ch -c $< -o $@ $(MOD_FLAG) ./include_ch + +./build_mu/EllipticBR1.o \ +./build_mu/EllipticBR2.o \ +./build_mu/EllipticIP.o \ +./build_mu/DGSEMClass.o: ./build_mu/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DMULTIPHASE -DCAHNHILLIARD -I./include_mu $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_mu -c $< -o $@ $(MOD_FLAG) ./include_mu + +./build_caa/DGSEMClass.o: ./build_caa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DACOUSTIC -I./include_caa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_caa -c $< -o $@ $(MOD_FLAG) ./include_caa + ./build_ns/%.o: %.f90 @echo @echo $< diff --git a/Solver/src/libs/mesh/Makefile b/Solver/src/libs/mesh/Makefile index 04b73175f..68ab963b3 100644 --- a/Solver/src/libs/mesh/Makefile +++ b/Solver/src/libs/mesh/Makefile @@ -198,6 +198,39 @@ all: lib$(LIB)_ns.a lib$(LIB)_nssa.a lib$(LIB)_ins.a lib$(LIB)_ch.a lib$(LIB)_mu @echo $< $(FC) -cpp -O2 -ffree-line-length-0 -DACOUSTIC -I./include_caa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDE)_caa -c $< -o $@ $(MOD_FLAG) ./include_caa +# nvfortran GPU: strip -fopenmp from files with mixed OMP/ACC directives +FFLAGS_NO_OMP := $(filter-out -fopenmp -mp,$(FFLAGS)) + +./build_ns/HexMesh.o ./build_ns/IBMClass.o: ./build_ns/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -I./include_ns $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ns -c $< -o $@ $(MOD_FLAG) ./include_ns + +./build_nssa/HexMesh.o ./build_nssa/IBMClass.o: ./build_nssa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -DSPALARTALMARAS -I./include_nssa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_nssa -c $< -o $@ $(MOD_FLAG) ./include_nssa + +./build_ins/HexMesh.o ./build_ins/IBMClass.o: ./build_ins/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DINCNS -I./include_ins $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ins -c $< -o $@ $(MOD_FLAG) ./include_ins + +./build_ch/HexMesh.o ./build_ch/IBMClass.o: ./build_ch/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DCAHNHILLIARD -I./include_ch $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ch -c $< -o $@ $(MOD_FLAG) ./include_ch + +./build_mu/HexMesh.o ./build_mu/IBMClass.o: ./build_mu/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DMULTIPHASE -DCAHNHILLIARD -I./include_mu $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_mu -c $< -o $@ $(MOD_FLAG) ./include_mu + +./build_caa/HexMesh.o ./build_caa/IBMClass.o: ./build_caa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DACOUSTIC -I./include_caa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_caa -c $< -o $@ $(MOD_FLAG) ./include_caa + ./build_ns/%.o: %.f90 @echo @echo $< diff --git a/Solver/src/libs/mesh/Monitors.f90 b/Solver/src/libs/mesh/Monitors.f90 deleted file mode 100644 index eaa4f1cff..000000000 --- a/Solver/src/libs/mesh/Monitors.f90 +++ /dev/null @@ -1,1503 +0,0 @@ -#include "Includes.h" -module MonitorsClass - use SMConstants - use NodalStorageClass - use HexMeshClass - use MonitorDefinitions - use ResidualsMonitorClass - use VolumeMonitorClass - use LoadBalancingMonitorClass - use FileReadingUtilities , only: getFileName -#ifdef FLOW - use ProbeClass -#endif -#if defined(NAVIERSTOKES) || defined(INCNS) - use StatisticsMonitor - use SurfaceMonitorClass -#endif - implicit none -! - - private - public Monitor_t -! -! ***************************** -! Main monitor class definition -! ***************************** -! - type Monitor_t - character(len=LINE_LENGTH) :: solution_file - character(len=LINE_LENGTH) :: probes_solution_file = "" - integer :: no_of_probes - integer :: no_of_surfaceMonitors - integer :: no_of_volumeMonitors - integer :: no_of_loadBalancingMonitors - integer :: no_of_fileProbes = 0 - character(len=LINE_LENGTH) :: probesFileName = "" - character(len=STR_LEN_MONITORS), allocatable :: probesVariables(:) - real(kind=RP) :: probeFileSaveTimestep = 0.0_RP - character(len=8) :: probeFileOutputFormat = "ASCII" - real(kind=RP) :: fp_lastSavedTime = -huge(0.0_RP) - integer :: bufferLine - integer , allocatable :: iter(:) - integer :: dt_restriction - logical :: write_dt_restriction - real(kind=RP) , allocatable :: t(:) - real(kind=RP) , allocatable :: SolverSimuTime(:) - real(kind=RP) , allocatable :: TotalSimuTime(:) - type(Residuals_t) :: residuals - class(VolumeMonitor_t) , allocatable :: volumeMonitors(:) - class(LoadBalancingMonitor_t), allocatable :: loadBalancingMonitors(:) -#ifdef FLOW - class(Probe_t) , allocatable :: probes(:) -#endif -#if defined(NAVIERSTOKES) || defined(INCNS) - class(SurfaceMonitor_t) , allocatable :: surfaceMonitors(:) - type(StatisticsMonitor_t) :: stats -#endif - contains - procedure :: Construct => Monitors_Construct - procedure :: WriteLabel => Monitor_WriteLabel - procedure :: WriteUnderlines => Monitor_WriteUnderlines - procedure :: WriteValues => Monitor_WriteValues - procedure :: UpdateValues => Monitor_UpdateValues - procedure :: WriteToFile => Monitor_WriteToFile - procedure :: WriteProbesFileSummary => Monitor_WriteProbesFileSummary - procedure :: destruct => Monitor_Destruct - procedure :: copy => Monitor_Assign - generic :: assignment(=) => copy - end type Monitor_t -! -! ======== - contains -! ======== -! -!/////////////////////////////////////////////////////////////////////////////////////// -! - subroutine Monitors_Construct( Monitors, mesh, controlVariables ) - use FTValueDictionaryClass - use mainKeywordsModule - use MPI_Process_Info - implicit none - class(Monitor_t) :: Monitors - class(HexMesh), intent(in) :: mesh - class(FTValueDictionary), intent(in) :: controlVariables - -! -! --------------- -! Local variables -! --------------- -! - integer :: fID , io - integer :: i - character(len=STR_LEN_MONITORS) :: line - character(len=STR_LEN_MONITORS) :: solution_file - logical, save :: FirstCall = .TRUE. - logical :: saveGradients - character(len=LINE_LENGTH) :: probesFileName - character(len=LINE_LENGTH) :: probesVariablesLine - character(len=STR_LEN_MONITORS), allocatable :: probesVariables(:) - integer :: no_of_fileProbes - integer :: no_of_probesVariables - real(kind=RP) :: probeFileSaveTimestep - character(len=8) :: probeFileOutputFormat - character(len=LINE_LENGTH) :: probes_solution_file - character(len=LINE_LENGTH) :: probes_dir - integer :: last_slash -! -! Setup the buffer -! ---------------- - if (controlVariables % containsKey("monitors flush interval") ) then - BUFFER_SIZE = controlVariables % integerValueForKey("monitors flush interval") - end if - - allocate ( Monitors % TotalSimuTime(BUFFER_SIZE), & - Monitors % SolverSimuTime(BUFFER_SIZE), & - Monitors % t(BUFFER_SIZE), & - Monitors % iter(BUFFER_SIZE) ) -! -! Get the solution file name -! -------------------------- - solution_file = controlVariables % stringValueForKey( solutionFileNameKey, requestedLength = STR_LEN_MONITORS ) -! -! Remove the *.hsol termination -! ----------------------------- - solution_file = trim(getFileName(solution_file)) - Monitors % solution_file = trim(solution_file) -! -! Build the probes output subdirectory: /probes/ -! ---------------------------------------------------------- - last_slash = index(trim(solution_file), '/', back=.true.) - if (last_slash .gt. 0) then - probes_dir = trim(solution_file(1:last_slash)) // "probes" - probes_solution_file = trim(solution_file(1:last_slash)) // "probes/" // trim(solution_file(last_slash+1:)) - else - probes_dir = "probes" - probes_solution_file = "probes/" // trim(solution_file) - end if - if (MPI_Process % isRoot) then - call execute_command_line('mkdir -p "' // trim(probes_dir) // '"', wait=.true.) - end if - Monitors % probes_solution_file = trim(probes_solution_file) -! -! Search in case file for probes, surface monitors, and volume monitors -! --------------------------------------------------------------------- - no_of_fileProbes = 0 - if (mesh % child) then ! Return doing nothing if this is a child mesh - Monitors % no_of_probes = 0 - Monitors % no_of_surfaceMonitors = 0 - Monitors % no_of_volumeMonitors = 0 - Monitors % no_of_loadBalancingMonitors = 0 - else - call getNoOfMonitors( Monitors % no_of_probes, Monitors % no_of_surfaceMonitors, Monitors % no_of_volumeMonitors, Monitors % no_of_loadBalancingMonitors ) -! -! Check for an additional probes definition file (one probe per -! line: "x y z variable1 [variable2 ...]"), allowing several -! variables to be sampled and saved for the same probe location -! --------------------------------------------------------------- -#ifdef FLOW - call readProbesFileBlock( probesFileName, probesVariablesLine, no_of_probesVariables, probeFileSaveTimestep, probeFileOutputFormat ) - - if ( len_trim(probesFileName) .gt. 0 ) then - call countProbesInFile( trim(probesFileName), no_of_fileProbes ) - - if ( len_trim(probesVariablesLine) .gt. 0 ) then - call splitIntoTokens( trim(probesVariablesLine), probesVariables, no_of_probesVariables ) - else - write(STD_OUT,*) "Error: 'variables' must be specified inside the '#define probe file' block." - stop - end if - end if -#endif - end if -! -! Initialize the Monitors class in the GPU -! ---------------------------------------- - !!$acc enter data copyin(Monitors) - !$acc update device(Monitors) - -! Pro tip: This is necessary to avoid the compiler doing behind the scenes copies of the class. -! Pro tip(cont): Its not really necessary for the class itself, but for the arrays inside the class. -! -! Initialize -! ---------- - call Monitors % residuals % Initialization( solution_file , FirstCall ) - - allocate ( Monitors % volumeMonitors ( Monitors % no_of_volumeMonitors ) ) - !$acc update device(Monitors) - do i = 1 , Monitors % no_of_volumeMonitors - call Monitors % volumeMonitors(i) % Initialization ( mesh , i, solution_file , FirstCall ) - end do - - allocate ( Monitors % loadBalancingMonitors ( Monitors % no_of_loadBalancingMonitors ) ) - !$acc update device(Monitors) - do i = 1 , Monitors % no_of_loadBalancingMonitors - call Monitors % loadBalancingMonitors(i) % Initialization ( mesh , i, solution_file , FirstCall ) - end do - -#ifdef FLOW - allocate ( Monitors % probes ( Monitors % no_of_probes + no_of_fileProbes ) ) - !$acc update device(Monitors) - do i = 1 , Monitors % no_of_probes - call Monitors % probes(i) % Initialization ( mesh , i, probes_solution_file , FirstCall ) - end do - - if ( no_of_fileProbes .gt. 0 ) then - call InitializeProbesFromFile( trim(probesFileName), Monitors % probes, Monitors % no_of_probes, & - mesh, probes_solution_file, FirstCall, probesVariables, probeFileSaveTimestep, & - probeFileOutputFormat ) - Monitors % probesFileName = trim(probesFileName) - Monitors % probeFileSaveTimestep = probeFileSaveTimestep - Monitors % probeFileOutputFormat = probeFileOutputFormat - Monitors % fp_lastSavedTime = -huge(0.0_RP) - allocate( Monitors % probesVariables(size(probesVariables)) ) - Monitors % probesVariables = probesVariables -#ifdef HAS_HDF5 - if ( trim(probeFileOutputFormat) .eq. "HDF5" ) then - call Monitor_InitFileProbesHDF5( Monitors, no_of_fileProbes ) - end if -#endif - end if - - Monitors % no_of_fileProbes = no_of_fileProbes - Monitors % no_of_probes = Monitors % no_of_probes + no_of_fileProbes -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) - saveGradients = controlVariables % logicalValueForKey(saveGradientsToSolutionKey) - call Monitors % stats % Construct(mesh, saveGradients) - - - allocate ( Monitors % surfaceMonitors ( Monitors % no_of_surfaceMonitors ) ) - !$acc update device(Monitors) - do i = 1 , Monitors % no_of_surfaceMonitors - call Monitors % surfaceMonitors(i) % Initialization ( mesh , i, solution_file , FirstCall ) - end do -#endif - - Monitors % write_dt_restriction = controlVariables % logicalValueForKey( "write dt restriction" ) - - Monitors % bufferLine = 0 - - FirstCall = .FALSE. -! -! Include the latest changes in the GPU -! ---------------------------------------- - !$acc update device(Monitors) - - end subroutine Monitors_Construct - - subroutine Monitor_WriteLabel ( self ) -! -! *************************************************** -! This subroutine prints the labels for the time -! integrator Display procedure. -! *************************************************** -! - use MPI_Process_Info - implicit none - class(Monitor_t) :: self - integer :: i - - if ( .not. MPI_Process % isRoot ) return -! -! Write "Iteration" and "Time" -! ---------------------------- - write ( STD_OUT , ' ( A10 ) ' , advance = "no" ) "Iteration" - write ( STD_OUT , ' ( 3X,A10 ) ' , advance = "no" ) "Time" -! -! Write residuals labels -! ---------------------- - call self % residuals % WriteLabel -! -! Write volume monitors labels -! ----------------------------- - do i = 1 , self % no_of_volumeMonitors - call self % volumeMonitors(i) % WriteLabel - end do -! -! Write load balancing monitor labels -! ------------------------------------ - do i = 1 , self % no_of_loadBalancingMonitors - call self % loadBalancingMonitors(i) % WriteLabel - end do - -#ifdef FLOW -! -! Write probes labels (file-based probes excluded for readability) -! --------------------------------------------------------------- - do i = 1 , self % no_of_probes - self % no_of_fileProbes - call self % probes(i) % WriteLabel - end do -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) -! -! Write surface monitors labels -! ----------------------------- - do i = 1 , self % no_of_surfaceMonitors - call self % surfaceMonitors(i) % WriteLabel - end do - - call self % stats % WriteLabel -#endif -! -! Write label for dt restriction -! ------------------------------ - if (self % write_dt_restriction) write ( STD_OUT , ' ( 3X,A10 ) ' , advance = "no" ) "dt restr." - - write(STD_OUT , *) - - end subroutine Monitor_WriteLabel - - subroutine Monitor_WriteUnderlines( self ) -! -! ******************************************************** -! This subroutine displays the underlines for the -! time integrator Display procedure. -! ******************************************************** -! - use PhysicsStorage - use MPI_Process_Info - implicit none - class(Monitor_t) :: self -! -! --------------- -! Local variables -! --------------- -! - integer :: i, j - character(len=MONITOR_LENGTH), parameter :: dashes = "----------" - - if ( .not. MPI_Process % isRoot ) return -! -! Print dashes for "Iteration" and "Time" -! --------------------------------------- - write ( STD_OUT , ' ( A10 ) ' , advance = "no" ) trim ( dashes ) - write ( STD_OUT , ' ( 3X,A10 ) ' , advance = "no" ) trim ( dashes ) -! -! Print dashes for residuals -! -------------------------- - do i = 1 , NCONS - write(STD_OUT , '(3X,A10)' , advance = "no" ) trim(dashes) - end do -! -! Print dashes for volume monitors -! -------------------------------- - do i = 1 , self % no_of_volumeMonitors ; do j=1, size ( self % volumeMonitors(i) % values, 1 ) - write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % volumeMonitors(i) % monitorName ) + 2 ) ) - end do ; end do -! -! Print dashes for load balancing monitor -! -------------------------------------- - do i = 1 , self % no_of_loadBalancingMonitors ; do j=1, size ( self % loadBalancingMonitors(i) % values, 1 ) - write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % loadBalancingMonitors(i) % monitorName ) + 2 ) ) - end do ; end do - -#ifdef FLOW -! -! Print dashes for probes (file-based probes excluded for readability) -! ---------------------------------------------------------------- - do i = 1 , self % no_of_probes - self % no_of_fileProbes - if ( self % probes(i) % active ) then - write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % probes(i) % monitorName ) + 2 ) ) - end if - end do -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) -! -! Print dashes for surface monitors -! --------------------------------- - do i = 1 , self % no_of_surfaceMonitors - write(STD_OUT , '(3X,A10)' , advance = "no" ) dashes(1 : min(10 , len_trim( self % surfaceMonitors(i) % monitorName ) + 2 ) ) - end do - - if ( self % stats % state .ne. 0 ) write(STD_OUT,'(3X,A10)',advance="no") trim(dashes) -#endif - - -! -! Print dashes for dt restriction -! ------------------------------- - if (self % write_dt_restriction) write ( STD_OUT , ' ( 3X,A10 ) ' , advance = "no" ) trim ( dashes ) - - write(STD_OUT , *) - - end subroutine Monitor_WriteUnderlines - - subroutine Monitor_WriteValues ( self ) -! -! ******************************************************* -! This subroutine prints the values for the time -! integrator Display procedure. -! ******************************************************* -! - use MPI_Process_Info - implicit none - class(Monitor_t) :: self - integer :: i - - if ( .not. MPI_Process % isRoot ) return -! -! Print iteration and time -! ------------------------ - write ( STD_OUT , ' ( I10 ) ' , advance = "no" ) self % iter ( self % bufferLine ) - write ( STD_OUT , ' ( 1X,A,1X,ES10.3 ) ' , advance = "no" ) "|" , self % t ( self % bufferLine ) -! -! Print residuals -! --------------- - call self % residuals % WriteValues( self % bufferLine ) -! -! Print volume monitors -! --------------------- - do i = 1 , self % no_of_volumeMonitors - call self % volumeMonitors(i) % WriteValues ( self % bufferLine ) - end do -! -! Print load balancing monitors -! ----------------------------- - do i = 1 , self % no_of_loadBalancingMonitors - call self % loadBalancingMonitors(i) % WriteValues ( self % bufferLine ) - end do - -#ifdef FLOW -! -! Print probes (file-based probes excluded for readability) -! ---------------------------------------------------------- - do i = 1 , self % no_of_probes - self % no_of_fileProbes - call self % probes(i) % WriteValues ( self % bufferLine ) - end do -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) -! -! Print surface monitors -! ---------------------- - do i = 1 , self % no_of_surfaceMonitors - call self % surfaceMonitors(i) % WriteValues ( self % bufferLine ) - end do - - call self % stats % WriteValue -#endif -! -! Print dt restriction -! -------------------- - if (self % write_dt_restriction) then - select case (self % dt_restriction) - case (DT_FIXED) ; write ( STD_OUT , ' ( 1X,A,1X,A10) ' , advance = "no" ) "|" , 'Fixed' - case (DT_DIFF) ; write ( STD_OUT , ' ( 1X,A,1X,A10) ' , advance = "no" ) "|" , 'Diffusive' - case (DT_CONV) ; write ( STD_OUT , ' ( 1X,A,1X,A10) ' , advance = "no" ) "|" , 'Convective' - end select - end if - - write(STD_OUT , *) - - end subroutine Monitor_WriteValues - - subroutine Monitor_UpdateValues ( self, mesh, t , iter, maxResiduals, Autosave, dt ) -! -! *************************************************************** -! This subroutine updates the values for the residuals, -! for the probes, surface and volume monitors. -! *************************************************************** -! - use PhysicsStorage - use StopwatchClass - implicit none - class(Monitor_t) :: self - class(HexMesh) :: mesh - real(kind=RP) :: t - integer :: iter - real(kind=RP) :: maxResiduals(NCONS), dt - logical :: Autosave -! -! --------------- -! Local variables -! --------------- -! - integer :: i -! -! Move to next buffer line -! ------------------------ - self % bufferLine = self % bufferLine + 1 -! -! Save time, iteration and CPU-time -! ----------------------- - self % t ( self % bufferLine ) = t - self % iter ( self % bufferLine ) = iter - self % SolverSimuTime ( self % bufferLine ) = Stopwatch % ElapsedTime("Solver") - self % TotalSimuTime ( self % bufferLine ) = Stopwatch % ElapsedTime("TotalTime") -! -! Compute current residuals -! ------------------------- - call self % residuals % Update( mesh, maxResiduals, self % bufferLine ) -! -! Update volume monitors -! ---------------------- - do i = 1 , self % no_of_volumeMonitors - call self % volumeMonitors(i) % Update( mesh , self % bufferLine ) - end do -! -! Update load balancing monitors -! ------------------------------ - do i = 1 , self % no_of_loadBalancingMonitors - call self % loadBalancingMonitors(i) % Update( mesh , self % bufferLine ) - end do - -#ifdef FLOW -! -! Update standard probes (GPU path with per-probe MPI) -! ----------------------------------------------------- - do i = 1 , self % no_of_probes - self % no_of_fileProbes - call self % probes(i) % Update( mesh , self % bufferLine ) - end do -! -! Update file probes: compute into slot 1, then flush immediately. -! File probes use a size-1 values buffer to avoid large memory -! allocations when the number of probes is O(1e6). -! ---------------------------------------------------------- - if ( self % no_of_fileProbes .gt. 0 ) then - call Monitor_UpdateFileProbes( self, mesh, 1 ) - call Monitor_FlushFileProbesNow( self, t, iter ) - end if -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) -! -! Update surface monitors -! ----------------------- - do i = 1 , self % no_of_surfaceMonitors - call self % surfaceMonitors(i) % Update( mesh , self % bufferLine, iter, autosave, dt ) - end do -! -! Update statistics -! ----------------- - call self % stats % Update(mesh, iter, t, trim(self % solution_file) ) -#endif - -! -! Update dt restriction -! --------------------- - if (self % write_dt_restriction) self % dt_restriction = mesh % dt_restriction - - end subroutine Monitor_UpdateValues - - subroutine Monitor_WriteToFile ( self , mesh, force) -! -! ****************************************************************** -! This routine has a double behaviour: -! force = .true. -> Writes to file and resets buffers -! force = .false. -> Just writes to file if the buffer is full -! ****************************************************************** -! - use MPI_Process_Info - implicit none - class(Monitor_t) :: self - class(HexMesh) :: mesh - logical, optional :: force -! ------------------------------------------------ - integer :: i - logical :: forceVal - - if ( present ( force ) ) then - forceVal = force - - else - forceVal = .false. - - end if - - if ( forceVal ) then -! -! In this case the monitors are exported to their files and the buffer is reset -! ----------------------------------------------------------------------------- - call self % residuals % WriteToFile ( self % iter , self % t, self % TotalSimuTime, self % SolverSimuTime , self % bufferLine ) - - do i = 1 , self % no_of_volumeMonitors - call self % volumeMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) - end do - - do i = 1 , self % no_of_loadBalancingMonitors - call self % loadBalancingMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) - end do - -#ifdef FLOW - do i = 1 , self % no_of_probes - self % no_of_fileProbes - call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) - end do -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) - do i = 1 , self % no_of_surfaceMonitors - call self % surfaceMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) - end do -! -! Write statistics -! ---------------- - if ( self % bufferLine .eq. 0 ) then - i = 1 - else - i = self % bufferLine - end if - call self % stats % WriteFile(mesh, self % iter(i), self % t(i), self % solution_file) -#endif -! -! Reset buffer -! ------------ - self % bufferLine = 0 - - else -! -! The monitors are exported just if the buffer is full -! ---------------------------------------------------- - if ( self % bufferLine .eq. BUFFER_SIZE ) then - - call self % residuals % WriteToFile ( self % iter , self % t, self % TotalSimuTime, self % SolverSimuTime, BUFFER_SIZE ) - - do i = 1 , self % no_of_volumeMonitors - call self % volumeMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) - end do - - do i = 1 , self % no_of_loadBalancingMonitors - call self % loadBalancingMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) - end do - -#ifdef FLOW - do i = 1 , self % no_of_probes - self % no_of_fileProbes - call self % probes(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) - end do -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) - do i = 1 , self % no_of_surfaceMonitors - call self % surfaceMonitors(i) % WriteToFile ( self % iter , self % t , self % bufferLine ) - end do -#endif -! -! Reset buffer -! ------------ - self % bufferLine = 0 - - end if - end if - - end subroutine Monitor_WriteToFile - - subroutine Monitor_WriteProbesFileSummary ( self ) -! -! ******************************************************************** -! Prints a startup-log summary of the bulk probes-file monitor, -! shown instead of including the file-based probes in the -! per-iteration screen log. -! ******************************************************************** -! - use Headers - use MPI_Process_Info - implicit none - class(Monitor_t) :: self -! -! --------------- -! Local variables -! --------------- -! - integer :: j - - if ( .not. MPI_Process % isRoot ) return - if ( self % no_of_fileProbes .le. 0 ) return - - write(STD_OUT,'(/)') - call Section_Header("Probes file monitor") - write(STD_OUT,'(/)') - - write(STD_OUT,'(30X,A,A28,A)') "->" , "File: " , trim(self % probesFileName) - write(STD_OUT,'(30X,A,A28,I10)') "->" , "Number of probes: " , self % no_of_fileProbes - write(STD_OUT,'(30X,A,A28)',advance="no") "->" , "Variables: " - do j = 1 , size(self % probesVariables) - write(STD_OUT,'(A)',advance="no") trim(self % probesVariables(j)) // " " - end do - write(STD_OUT,*) - if ( self % probeFileSaveTimestep .gt. 0.0_RP ) then - write(STD_OUT,'(30X,A,A28,ES14.6)') "->" , "Save timestep: " , self % probeFileSaveTimestep - end if - write(STD_OUT,'(30X,A,A28,A)') "->" , "Output format: " , trim(self % probeFileOutputFormat) - - end subroutine Monitor_WriteProbesFileSummary - - subroutine Monitor_Destruct (self) - implicit none - class(Monitor_t) :: self - - deallocate (self % iter) - deallocate (self % t) - deallocate (self % TotalSimuTime) - deallocate (self % SolverSimuTime) - - call self % residuals % destruct - - call self % volumeMonitors % destruct - safedeallocate(self % volumeMonitors) - - call self % loadBalancingMonitors % destruct - safedeallocate(self % loadBalancingMonitors) - -#ifdef FLOW - call self % probes % destruct - safedeallocate (self % probes) -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) - call self % surfaceMonitors % destruct - safedeallocate (self % surfaceMonitors) - - !call self % stats % destruct -#endif - end subroutine - - impure elemental subroutine Monitor_Assign ( to, from ) - implicit none - !-arguments-------------------------------------- - class(Monitor_t), intent(inout) :: to - type(Monitor_t) , intent(in) :: from - !-local-variables-------------------------------- - !------------------------------------------------ - - to % solution_file = from % solution_file - to % probes_solution_file = from % probes_solution_file - to % no_of_probes = from % no_of_probes - to % no_of_surfaceMonitors = from % no_of_surfaceMonitors - to % no_of_volumeMonitors = from % no_of_volumeMonitors - to % no_of_loadBalancingMonitors = from % no_of_loadBalancingMonitors - to % no_of_fileProbes = from % no_of_fileProbes - to % probesFileName = from % probesFileName - to % probeFileSaveTimestep = from % probeFileSaveTimestep - to % probeFileOutputFormat = from % probeFileOutputFormat - to % fp_lastSavedTime = from % fp_lastSavedTime - if ( allocated(from % probesVariables) ) then - safedeallocate ( to % probesVariables ) - allocate ( to % probesVariables ( size(from % probesVariables) ) ) - to % probesVariables = from % probesVariables - end if - to % bufferLine = from % bufferLine - - safedeallocate ( to % iter ) - allocate ( to % iter ( size(from % iter) ) ) - to % iter = from % iter - - to % dt_restriction = from % dt_restriction - to % write_dt_restriction = from % write_dt_restriction - - safedeallocate (to % t) - allocate (to % t (size (from % t) ) ) - to % t = from % t - - safedeallocate ( to % TotalSimuTime ) - allocate ( to % TotalSimuTime ( size(from % TotalSimuTime) ) ) - to % TotalSimuTime = from % TotalSimuTime - - safedeallocate ( to % SolverSimuTime ) - allocate ( to % SolverSimuTime ( size(from % SolverSimuTime) ) ) - to % SolverSimuTime = from % SolverSimuTime - - to % residuals = from % residuals - - safedeallocate ( to % volumeMonitors ) - allocate ( to % volumeMonitors ( size(from % volumeMonitors) ) ) - to % volumeMonitors = from % volumeMonitors - - safedeallocate ( to % loadBalancingMonitors ) - allocate ( to % loadBalancingMonitors ( size(from % loadBalancingMonitors) ) ) - to % loadBalancingMonitors = from % loadBalancingMonitors - -#ifdef FLOW - safedeallocate ( to % probes ) - allocate ( to % probes ( size(from % probes) ) ) - to % probes = from % probes -#endif - -#if defined(NAVIERSTOKES) || defined(INCNS) - safedeallocate ( to % surfaceMonitors ) - allocate ( to % surfaceMonitors ( size(from % surfaceMonitors) ) ) - to % surfaceMonitors = from % surfaceMonitors - - to % stats = from % stats -#endif - - end subroutine Monitor_Assign - -! -!////////////////////////////////////////////////////////////////////////////// -! -! Auxiliars -! -!////////////////////////////////////////////////////////////////////////////// -! - subroutine getNoOfMonitors(no_of_probes, no_of_surfaceMonitors, no_of_volumeMonitors, no_of_loadBalancingMonitors) - use ParamfileRegions - implicit none - integer, intent(out) :: no_of_probes - integer, intent(out) :: no_of_surfaceMonitors - integer, intent(out) :: no_of_volumeMonitors - integer, intent(out) :: no_of_loadBalancingMonitors -! -! --------------- -! Local variables -! --------------- -! - character(len=LINE_LENGTH) :: case_name, line - integer :: fID - integer :: io -! -! Initialize -! ---------- - no_of_probes = 0 - no_of_surfaceMonitors = 0 - no_of_volumeMonitors = 0 - no_of_loadBalancingMonitors = 0 -! -! Get case file name -! ------------------ - call get_command_argument(1, case_name) - -! -! Open case file -! -------------- - open ( newunit = fID , file = case_name , status = "old" , action = "read" ) - -! -! Read the whole file to find monitors -! ------------------------------------ -readloop:do - read ( fID , '(A)' , iostat = io ) line - - if ( io .lt. 0 ) then -! -! End of file -! ----------- - line = "" - exit readloop - - elseif ( io .gt. 0 ) then -! -! Error -! ----- - errorMessage(STD_OUT) - error stop "Stopped." - - else -! -! Succeeded -! --------- - line = getSquashedLine( line ) - - if ( index ( line , '#defineprobefile' ) .gt. 0 ) then -! -! The probe-file block is not an individual probe definition -! ----------------------------------------------------------- - - elseif ( index ( line , '#defineprobe' ) .gt. 0 ) then - no_of_probes = no_of_probes + 1 - - elseif ( index ( line , '#definesurfacemonitor' ) .gt. 0 ) then - no_of_surfaceMonitors = no_of_surfaceMonitors + 1 - - elseif ( index ( line , '#definevolumemonitor' ) .gt. 0 ) then - no_of_volumeMonitors = no_of_volumeMonitors + 1 - - elseif ( index ( line , '#defineloadbalancingmonitor' ) .gt. 0 ) then - no_of_loadBalancingMonitors = no_of_loadBalancingMonitors + 1 - - end if - - end if - - end do readloop -! -! Close case file -! --------------- - close(fID) - -end subroutine getNoOfMonitors - -! -!/////////////////////////////////////////////////////////////////////////////////// -! -! Probes-from-file auxiliary routines -! -! The probes file is a plain text file. Blank lines and lines starting -! with "#" are ignored. Every remaining line holds the coordinates of -! one probe: -! -! x y z -! -! The list of variables to sample (shared by every probe in the file) -! is given through the "probes file variables" control-file keyword. -! -! A separate output file ".probe_.probe" is created for -! each probe, with one column per variable, and one row written per -! saved time step. -! -!/////////////////////////////////////////////////////////////////////////////////// -! - subroutine readProbesFileBlock(fileName, variablesLine, no_of_probesVariables, saveTimestep, outputFormat) -! -! ****************************************************************** -! Reads the "#define probe file ... #end" block from the case -! file, if present: -! -! #define probe file -! file = Probe.dat -! variables = u -! probe save timestep = 1.0E-3 -! output format = HDF5 -! #end -! -! Note: this is a hand-rolled parser (rather than readValueInRegion) -! because readValueInRegion lower-cases every line it reads, which -! would corrupt a case-sensitive file path. -! ****************************************************************** -! - use ParamfileRegions, only: getSquashedLine - implicit none - character(len=LINE_LENGTH), intent(out) :: fileName - character(len=LINE_LENGTH), intent(out) :: variablesLine - integer, intent(out) :: no_of_probesVariables - real(kind=RP), intent(out) :: saveTimestep - character(len=8), intent(out) :: outputFormat -! -! --------------- -! Local variables -! --------------- -! - character(len=LINE_LENGTH) :: paramFile - character(len=LINE_LENGTH) :: line, squashed, valStr - integer :: fID, io, position - - logical :: inside - - fileName = "" - variablesLine = "" - no_of_probesVariables = 0 - saveTimestep = 0.0_RP - outputFormat = "ASCII" - inside = .false. - - call get_command_argument(1, paramFile) - - open ( newunit = fID , file = trim(paramFile) , status = "old" , action = "read" ) - - do - read ( fID , '(A)' , iostat = io ) line - if ( io .ne. 0 ) exit - - squashed = getSquashedLine(line) - - if ( squashed .eq. getSquashedLine("#define probe file") ) then - inside = .true. - cycle - elseif ( squashed .eq. getSquashedLine("#end") ) then - inside = .false. - cycle - end if - - if ( .not. inside ) cycle -! -! Strip a trailing comment, keeping the value's original case -! ------------------------------------------------------------- - position = index(line , '!') - if ( position .gt. 0 ) line = line(1:position-1) - - position = max( index(line,'='), index(line,':') ) - if ( position .eq. 0 ) cycle - - if ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("file") ) then - fileName = adjustl( removeQuotes( line(position+1:) ) ) - elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("variables") ) then - variablesLine = adjustl( removeQuotes( line(position+1:) ) ) - elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("probe save timestep") ) then - valStr = adjustl( removeQuotes( line(position+1:) ) ) - read( valStr , * ) saveTimestep - elseif ( getSquashedLine(line(1:position-1)) .eq. getSquashedLine("output format") ) then - valStr = adjustl( removeQuotes( line(position+1:) ) ) - valStr = adjustl( getSquashedLine(valStr) ) - if ( index(valStr, "HDF5") .gt. 0 ) then - outputFormat = "HDF5" - else - outputFormat = "ASCII" - end if - end if - end do - - close(fID) - - end subroutine readProbesFileBlock - - function removeQuotes(str) result(res) - implicit none - character(len=*), intent(in) :: str - character(len=LINE_LENGTH) :: res - character(len=LINE_LENGTH) :: auxstr - integer :: i, j - - auxstr = trim(adjustl(str)) - res = "" - j = 0 - - do i = 1 , len_trim(auxstr) - if ( auxstr(i:i) .eq. '"' .or. auxstr(i:i) .eq. "'" ) cycle - j = j + 1 - res(j:j) = auxstr(i:i) - end do - - end function removeQuotes - - subroutine countProbesInFile(fileName, n) - implicit none - character(len=*), intent(in) :: fileName - integer, intent(out) :: n -! -! --------------- -! Local variables -! --------------- -! - integer :: fID, io - character(len=LINE_LENGTH) :: line - - n = 0 - open ( newunit = fID , file = fileName , status = "old" , action = "read" ) - - do - read ( fID , '(A)' , iostat = io ) line - if ( io .ne. 0 ) exit - - line = adjustl(line) - if ( len_trim(line) .eq. 0 ) cycle - if ( line(1:1) .eq. '#' ) cycle - - n = n + 1 - end do - - close(fID) - - end subroutine countProbesInFile - - subroutine splitIntoTokens(line, tokens, n) - implicit none - character(len=*), intent(in) :: line - character(len=STR_LEN_MONITORS), allocatable, intent(out) :: tokens(:) - integer, intent(out) :: n -! -! --------------- -! Local variables -! --------------- -! - character(len=LINE_LENGTH) :: auxline - character(len=STR_LEN_MONITORS) :: buffer(64) - integer :: pos - - auxline = adjustl(line) - n = 0 - - do while ( len_trim(auxline) .gt. 0 ) - pos = index(trim(auxline), " ") - n = n + 1 - - if ( pos .eq. 0 ) then - buffer(n) = trim(auxline) - auxline = "" - else - buffer(n) = auxline(1:pos-1) - auxline = adjustl(auxline(pos+1:)) - end if - end do - - allocate( tokens(n) ) - tokens(1:n) = buffer(1:n) - - end subroutine splitIntoTokens - -#ifdef FLOW - subroutine Monitor_FlushFileProbesNow(self, t_now, iter_now) -! -! Writes the current (single-slot) file-probe values to disk, -! applying the probeFileSaveTimestep filter. Called every timestep -! from Monitor_UpdateValues after Monitor_UpdateFileProbes. -! - use MPI_Process_Info - implicit none - class(Monitor_t), intent(inout) :: self - real(kind=RP), intent(in) :: t_now - integer, intent(in) :: iter_now -! -! --------------- -! Local variables -! --------------- -! - integer :: i - integer :: iter_arr(1) - real(kind=RP) :: t_arr(1) - - iter_arr(1) = iter_now - t_arr(1) = t_now - -#ifdef HAS_HDF5 - if ( trim(self % probeFileOutputFormat) .eq. "HDF5" ) then - call Monitor_WriteFileProbesHDF5( self, iter_arr, t_arr, 1 ) - else -#endif - !$omp parallel do schedule(dynamic,16) default(shared) - do i = self % no_of_probes - self % no_of_fileProbes + 1, self % no_of_probes - call self % probes(i) % WriteToFile( iter_arr, t_arr, 1 ) - end do - !$omp end parallel do -#ifdef HAS_HDF5 - end if -#endif - - end subroutine Monitor_FlushFileProbesNow - - subroutine Monitor_UpdateFileProbes(self, mesh, bufferPos) -! -! Evaluates all file-probes using OpenMP threads (CPU path, no OpenACC), -! then performs a single MPI_Allreduce(SUM) to share results across ranks. -! This replaces N_fileProbes individual MPI_Bcast calls with one collective. -! - use MPI_Process_Info -#ifdef _HAS_MPI_ - use mpi -#endif - implicit none - class(Monitor_t), intent(inout) :: self - class(HexMesh), intent(in) :: mesh - integer, intent(in) :: bufferPos -! -! --------------- -! Local variables -! --------------- -! - integer :: i, v, j, nfp, nv, fp_offset, ierr - real(kind=RP), allocatable :: buf(:) - - nfp = self % no_of_fileProbes - nv = size(self % probesVariables) - fp_offset = self % no_of_probes - nfp - - ! Parallel CPU computation — each probe writes to its own values(:,bufferPos). - ! Non-owning ranks store 0 so MPI_Allreduce(SUM) gives the correct result. - !$omp parallel do schedule(dynamic,16) default(shared) - do i = fp_offset + 1, self % no_of_probes - call self % probes(i) % ComputeLocal(mesh, bufferPos) - end do - !$omp end parallel do - -#ifdef _HAS_MPI_ - if ( MPI_Process % doMPIAction ) then - ! Pack all file-probe values into a flat buffer - allocate( buf(nfp * nv) ) - j = 0 - do i = fp_offset + 1, self % no_of_probes - do v = 1, nv - j = j + 1 - buf(j) = self % probes(i) % values(v, bufferPos) - end do - end do - - ! Single collective instead of nfp individual Bcasts - call MPI_Allreduce(MPI_IN_PLACE, buf, nfp * nv, MPI_DOUBLE_PRECISION, MPI_SUM, MPI_COMM_WORLD, ierr) - - ! Unpack - j = 0 - do i = fp_offset + 1, self % no_of_probes - do v = 1, nv - j = j + 1 - self % probes(i) % values(v, bufferPos) = buf(j) - end do - end do - deallocate(buf) - end if -#endif - - end subroutine Monitor_UpdateFileProbes - - subroutine InitializeProbesFromFile(fileName, probes, offset, mesh, solution_file, FirstCall, variables, saveTimestep, outputFormat) - implicit none - character(len=*), intent(in) :: fileName - class(Probe_t), intent(inout) :: probes(:) - integer, intent(in) :: offset - class(HexMesh), intent(in) :: mesh - character(len=*), intent(in) :: solution_file - logical, intent(in) :: FirstCall - character(len=*), intent(in) :: variables(:) - real(kind=RP), intent(in) :: saveTimestep - character(len=*), intent(in) :: outputFormat -! -! --------------- -! Local variables -! --------------- -! - integer :: fID, io, idx, nTok - character(len=LINE_LENGTH) :: line - real(kind=RP) :: x(NDIM) - character(len=STR_LEN_MONITORS), allocatable :: tokens(:) - character(len=STR_LEN_MONITORS) :: pname - - idx = offset - open ( newunit = fID , file = fileName , status = "old" , action = "read" ) - - do - read ( fID , '(A)' , iostat = io ) line - if ( io .ne. 0 ) exit - - line = adjustl(line) - if ( len_trim(line) .eq. 0 ) cycle - if ( line(1:1) .eq. '#' ) cycle - - call splitIntoTokens(line, tokens, nTok) - - if ( nTok .lt. 3 ) then - deallocate(tokens) - cycle - end if - - read(tokens(1),*) x(1) - read(tokens(2),*) x(2) - read(tokens(3),*) x(3) - - idx = idx + 1 - write(pname,'(A,I0)') "probe_", idx - - call probes(idx) % Initialization( mesh, idx, solution_file, FirstCall, & - x_in = x, variables_in = variables, name_in = trim(pname), & - isFileProbe_in = .true., outputFormat_in = trim(outputFormat) ) - probes(idx) % saveTimestep = saveTimestep - - deallocate(tokens) - end do - - close(fID) - - end subroutine InitializeProbesFromFile - -! ============================================================ -! HDF5 collective I/O for file-based probes -! Compiled only when HAS_HDF5 is defined. -! ============================================================ - -#ifdef HAS_HDF5 - subroutine Monitor_InitFileProbesHDF5(self, no_of_fileProbes) -! -! Creates the HDF5 output file for bulk file-probes with -! the following structure: -! -! /coordinates (3, nProbes) — fixed, written once -! /time (extendible) — one value per saved step -! /iteration (extendible) — one value per saved step -! / (nProbes, ext) — one row per saved step -! - use HDF5 - use MPI_Process_Info - implicit none - class(Monitor_t), intent(inout) :: self - integer, intent(in) :: no_of_fileProbes -! -! --------------- -! Local variables -! --------------- -! - integer(HID_T) :: file_id, dset_id, dspace_id, dcpl_id - integer(HSIZE_T) :: dims2(2), maxdims2(2), chunk2(2) - integer(HSIZE_T) :: dims1(1), maxdims1(1), chunk1(1) - integer :: iError, j, v, fp_offset, nv - character(len=LINE_LENGTH) :: fname - real(kind=RP), allocatable :: coords(:,:) - - if ( .not. MPI_Process % isRoot ) return - - nv = size(self % probesVariables) - fp_offset = self % no_of_probes ! probes are allocated 1..no_of_probes+no_of_fileProbes - ! but at this point no_of_probes has not yet been bumped - - write(fname,'(A,A)') trim(self % probes_solution_file), ".probes.h5" - - call h5open_f(iError) - call h5fcreate_f(trim(fname), H5F_ACC_TRUNC_F, file_id, iError) - - ! /coordinates (3, nProbes) — fixed at creation - allocate( coords(3, no_of_fileProbes) ) - do j = 1, no_of_fileProbes - coords(:, j) = self % probes(fp_offset + j) % x - end do - dims2 = [ int(3, HSIZE_T), int(no_of_fileProbes, HSIZE_T) ] - call h5screate_simple_f(2, dims2, dspace_id, iError) - call h5dcreate_f(file_id, "coordinates", H5T_NATIVE_DOUBLE, dspace_id, dset_id, iError) - call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, coords, dims2, iError) - call h5dclose_f(dset_id, iError) - call h5sclose_f(dspace_id, iError) - deallocate(coords) - - ! /time (extendible 1-D) - dims1(1) = 0 - maxdims1(1) = H5S_UNLIMITED_F - chunk1(1) = int(max(BUFFER_SIZE, 1), HSIZE_T) - call h5screate_simple_f(1, dims1, dspace_id, iError, maxdims1) - call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) - call h5pset_chunk_f(dcpl_id, 1, chunk1, iError) - call h5dcreate_f(file_id, "time", H5T_NATIVE_DOUBLE, dspace_id, dset_id, iError, dcpl_id) - call h5dclose_f(dset_id, iError) - call h5sclose_f(dspace_id, iError) - call h5pclose_f(dcpl_id, iError) - - ! /iteration (extendible 1-D) - call h5screate_simple_f(1, dims1, dspace_id, iError, maxdims1) - call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) - call h5pset_chunk_f(dcpl_id, 1, chunk1, iError) - call h5dcreate_f(file_id, "iteration", H5T_NATIVE_INTEGER, dspace_id, dset_id, iError, dcpl_id) - call h5dclose_f(dset_id, iError) - call h5sclose_f(dspace_id, iError) - call h5pclose_f(dcpl_id, iError) - - ! / (extendible × nProbes) — h5ls shows {nProbes, Inf} - ! chunk2(1)=1: one time step per chunk avoids pre-allocation waste when - ! the save-timestep filter makes n_write << BUFFER_SIZE per flush. - dims2(1) = 0 - dims2(2) = int(no_of_fileProbes, HSIZE_T) - maxdims2(1) = H5S_UNLIMITED_F - maxdims2(2) = int(no_of_fileProbes, HSIZE_T) - chunk2(1) = int(1, HSIZE_T) - chunk2(2) = int(no_of_fileProbes, HSIZE_T) - - do v = 1, nv - call h5screate_simple_f(2, dims2, dspace_id, iError, maxdims2) - call h5pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, iError) - call h5pset_chunk_f(dcpl_id, 2, chunk2, iError) - call h5dcreate_f(file_id, trim(self % probesVariables(v)), H5T_NATIVE_DOUBLE, & - dspace_id, dset_id, iError, dcpl_id) - call h5dclose_f(dset_id, iError) - call h5sclose_f(dspace_id, iError) - call h5pclose_f(dcpl_id, iError) - end do - - call h5fclose_f(file_id, iError) - call h5close_f(iError) - - end subroutine Monitor_InitFileProbesHDF5 - - subroutine Monitor_WriteFileProbesHDF5(self, iter, t, no_of_lines) -! -! Appends one buffer of file-probe data to the HDF5 output file. -! Only lines that satisfy the saveTimestep filter are written. -! Called from Monitor_WriteToFile in place of the per-probe ASCII loop. -! - use HDF5 - use MPI_Process_Info - implicit none - class(Monitor_t), intent(inout) :: self - integer, intent(in) :: iter(:) - real(kind=RP), intent(in) :: t(:) - integer, intent(in) :: no_of_lines -! -! --------------- -! Local variables -! --------------- -! - integer(HID_T) :: file_id, dset_id, dspace_id, mspace_id - integer(HSIZE_T) :: cur1(1), max1(1), off1(1), cnt1(1) - integer(HSIZE_T) :: new1(1) - integer(HSIZE_T) :: cur2(2), max2(2), off2(2), cnt2(2) - integer(HSIZE_T) :: new2(2) - integer :: iError, i, j, k, v, nfp, nv, fp_offset, n_write - character(len=LINE_LENGTH) :: fname - logical, allocatable :: wmask(:) - integer, allocatable :: ibuf(:) - real(kind=RP), allocatable :: rbuf(:), vbuf(:) - - if ( .not. MPI_Process % isRoot ) return - - nfp = self % no_of_fileProbes - nv = size(self % probesVariables) - fp_offset = self % no_of_probes - nfp - - ! Build write mask (apply saveTimestep filter) - allocate( wmask(no_of_lines) ) - wmask = .false. - do i = 1, no_of_lines - if ( self % probeFileSaveTimestep .gt. 0.0_RP ) then - if ( t(i) .lt. self % fp_lastSavedTime + self % probeFileSaveTimestep ) cycle - end if - wmask(i) = .true. - self % fp_lastSavedTime = t(i) - end do - n_write = count(wmask) - if ( n_write .eq. 0 ) then - deallocate(wmask) - return - end if - - write(fname,'(A,A)') trim(self % probes_solution_file), ".probes.h5" - - call h5open_f(iError) - call h5fopen_f(trim(fname), H5F_ACC_RDWR_F, file_id, iError) - - ! Query current extent of /time to get append offset - call h5dopen_f(file_id, "time", dset_id, iError) - call h5dget_space_f(dset_id, dspace_id, iError) - call h5sget_simple_extent_dims_f(dspace_id, cur1, max1, iError) - call h5sclose_f(dspace_id, iError) - call h5dclose_f(dset_id, iError) - ! cur1(1) = number of time steps already written - - cnt1(1) = int(n_write, HSIZE_T) - off1(1) = cur1(1) - new1(1) = cur1(1) + cnt1(1) - - ! Pack filtered iteration values - allocate( ibuf(n_write) ) - j = 0 - do i = 1, no_of_lines - if ( .not. wmask(i) ) cycle - j = j + 1 - ibuf(j) = iter(i) - end do - - call h5dopen_f(file_id, "iteration", dset_id, iError) - call h5dextend_f(dset_id, new1, iError) - call h5dget_space_f(dset_id, dspace_id, iError) - call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off1, cnt1, iError) - call h5screate_simple_f(1, cnt1, mspace_id, iError) - call h5dwrite_f(dset_id, H5T_NATIVE_INTEGER, ibuf, cnt1, iError, mspace_id, dspace_id) - call h5sclose_f(mspace_id, iError) - call h5sclose_f(dspace_id, iError) - call h5dclose_f(dset_id, iError) - deallocate(ibuf) - - ! Pack filtered time values - allocate( rbuf(n_write) ) - j = 0 - do i = 1, no_of_lines - if ( .not. wmask(i) ) cycle - j = j + 1 - rbuf(j) = t(i) - end do - - call h5dopen_f(file_id, "time", dset_id, iError) - call h5dextend_f(dset_id, new1, iError) - call h5dget_space_f(dset_id, dspace_id, iError) - call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off1, cnt1, iError) - call h5screate_simple_f(1, cnt1, mspace_id, iError) - call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, rbuf, cnt1, iError, mspace_id, dspace_id) - call h5sclose_f(mspace_id, iError) - call h5sclose_f(dspace_id, iError) - call h5dclose_f(dset_id, iError) - deallocate(rbuf) - - ! Write each variable: one extend + one write per flush (avoids chunk waste). - ! vbuf2(nfp, n_write) packs all filtered timesteps into a contiguous 2D block. - off2(1) = cur1(1) ! append after existing time steps - off2(2) = int(0, HSIZE_T) - cnt2(1) = int(n_write, HSIZE_T) - cnt2(2) = int(nfp, HSIZE_T) - new2(1) = cur1(1) + int(n_write, HSIZE_T) - new2(2) = int(nfp, HSIZE_T) - - allocate( vbuf(nfp * n_write) ) - - do v = 1, nv - call h5dopen_f(file_id, trim(self % probesVariables(v)), dset_id, iError) - - ! Pack into column-major 2D layout vbuf(n_write, nfp): - ! time index j varies fastest (Fortran column-major with cnt2=[n_write,nfp]) - j = 0 - do i = 1, no_of_lines - if ( .not. wmask(i) ) cycle - j = j + 1 - do k = 1, nfp - vbuf( j + (k-1)*n_write ) = self % probes(fp_offset + k) % values(v, i) - end do - end do - - call h5dextend_f(dset_id, new2, iError) - call h5dget_space_f(dset_id, dspace_id, iError) - call h5sselect_hyperslab_f(dspace_id, H5S_SELECT_SET_F, off2, cnt2, iError) - call h5screate_simple_f(2, cnt2, mspace_id, iError) - call h5dwrite_f(dset_id, H5T_NATIVE_DOUBLE, vbuf, cnt2, iError, mspace_id, dspace_id) - call h5sclose_f(mspace_id, iError) - call h5sclose_f(dspace_id, iError) - - call h5dclose_f(dset_id, iError) - end do - - deallocate(vbuf) - deallocate(wmask) - - call h5fclose_f(file_id, iError) - call h5close_f(iError) - - end subroutine Monitor_WriteFileProbesHDF5 -#endif ! HAS_HDF5 - -#endif ! FLOW - -end module MonitorsClass -! -!/////////////////////////////////////////////////////////////////////////////////// -! diff --git a/Solver/src/libs/mesh/Probe.f90 b/Solver/src/libs/mesh/Probe.f90 deleted file mode 100644 index e91279dbc..000000000 --- a/Solver/src/libs/mesh/Probe.f90 +++ /dev/null @@ -1,967 +0,0 @@ -#include "Includes.h" -#ifdef FLOW -module ProbeClass - use SMConstants - use HexMeshClass - use MonitorDefinitions - use PhysicsStorage - use VariableConversion - use MPI_Process_Info - use FluidData - use FileReadingUtilities, only: getRealArrayFromString - use NodalStorageClass , only: NodalStorage -#ifdef _HAS_MPI_ - use mpi -#endif - implicit none - - private - public Probe_t -! -! ********************** -! Probe class definition -! ********************** -! - type Probe_t - logical :: active - logical :: isFileProbe - integer :: rank - integer :: ID - integer :: eID - integer :: nVars - real(kind=RP) :: x(NDIM) - real(kind=RP) :: xi(NDIM) - real(kind=RP), allocatable :: values(:,:) - real(kind=RP), allocatable :: lxi(:) , leta(:), lzeta(:) - real(kind=RP), allocatable :: var(:,:,:) - real(kind=RP) :: saveTimestep - real(kind=RP) :: lastSavedTime - character(len=STR_LEN_MONITORS) :: fileName - character(len=STR_LEN_MONITORS) :: monitorName - character(len=STR_LEN_MONITORS), allocatable :: variableNames(:) - contains - procedure :: Initialization => Probe_Initialization - procedure :: Update => Probe_Update - procedure :: ComputeLocal => Probe_ComputeLocal - procedure :: WriteLabel => Probe_WriteLabel - procedure :: WriteValues => Probe_WriteValue - procedure :: WriteToFile => Probe_WriteToFile - procedure :: LookInOtherPartitions => Probe_LookInOtherPartitions - procedure :: destruct => Probe_Destruct - procedure :: copy => Probe_Assign - generic :: assignment(=) => copy - end type Probe_t - - contains - - subroutine Probe_Initialization(self, mesh, ID, solution_file, FirstCall, x_in, variables_in, name_in, isFileProbe_in, outputFormat_in) - use ParamfileRegions - use MPI_Process_Info - use Utilities, only: toLower - implicit none - class(Probe_t) :: self - class(HexMesh) :: mesh - integer :: ID - character(len=*) :: solution_file - logical, intent(in) :: FirstCall - real(kind=RP), intent(in), optional :: x_in(NDIM) - character(len=*), intent(in), optional :: variables_in(:) - character(len=*), intent(in), optional :: name_in - logical, intent(in), optional :: isFileProbe_in - character(len=*), intent(in), optional :: outputFormat_in -! -! --------------- -! Local variables -! --------------- -! - integer :: i, j, k, v, fid - character(len=STR_LEN_MONITORS) :: in_label - character(len=STR_LEN_MONITORS) :: fileName - character(len=STR_LEN_MONITORS) :: paramFile - character(len=STR_LEN_MONITORS) :: coordinates - character(len=STR_LEN_MONITORS) :: variable - character(len=STR_LEN_MONITORS) :: outputFormat - - self % isFileProbe = .false. - if ( present(isFileProbe_in) ) self % isFileProbe = isFileProbe_in - - outputFormat = "ASCII" - if ( present(outputFormat_in) ) outputFormat = trim(outputFormat_in) - - if (FirstCall) then -! -! Get monitor ID -! -------------- - self % ID = ID -! -! Get the probe definition, either from a probes file (x_in/variables_in -! provided) or from a "#define probe" block in the case file -! ------------------------------------------------------------------------ - if ( present(x_in) ) then - self % monitorName = name_in - self % x = x_in - self % nVars = size(variables_in) - allocate( self % variableNames(self % nVars) ) - self % variableNames = variables_in - - else -! -! Search for the parameters in the case file -! ------------------------------------------ - write(in_label , '(A,I0)') "#define probe " , self % ID - - call get_command_argument(1, paramFile) - call readValueInRegion(trim(paramFile), "name" , self % monitorName, in_label, "# end" ) - call readValueInRegion(trim(paramFile), "variable", variable , in_label, "# end" ) - call readValueInRegion(trim(paramFile), "position", coordinates , in_label, "# end" ) -! -! Get the coordinates -! ------------------- - self % x = getRealArrayFromString(coordinates) - - self % nVars = 1 - allocate( self % variableNames(1) ) - self % variableNames(1) = variable - end if -! -! Allocate memory -! --------------- - if (self % isFileProbe) then - allocate ( self % values(self % nVars, 1) ) - else - allocate ( self % values(self % nVars, BUFFER_SIZE) ) - end if - self % saveTimestep = 0.0_RP - self % lastSavedTime = -huge(self % lastSavedTime) -! -! Check the variables -! -------------------- - do v = 1, self % nVars - call tolower(self % variableNames(v)) - - select case ( trim(self % variableNames(v)) ) -#ifdef NAVIERSTOKES - case ("pressure") - case ("velocity") - case ("u") - case ("v") - case ("w") - case ("mach") - case ("k") - case ("rho") - case default - print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' - print*, "Options available are:" - print*, " * pressure" - print*, " * velocity" - print*, " * u" - print*, " * v" - print*, " * w" - print*, " * Mach" - print*, " * K" - print*, " * rho" - end select -#endif -#ifdef INCNS - case ("pressure") - case ("velocity") - case ("u") - case ("v") - case ("w") - case ("rho") - case default - print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' - print*, "Options available are:" - print*, " * pressure" - print*, " * velocity" - print*, " * u" - print*, " * v" - print*, " * w" - print*, " * rho" - end select -#endif -#ifdef MULTIPHASE - case ("static-pressure") - - case default - print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' - print*, "Options available are:" - print*, " * static-pressure" - - end select -#endif -#ifdef ACOUSTIC - case ("pressure") - case ("density") - case ("u") - case ("v") - case ("w") - case default - print*, 'Probe variable "',trim(self % variableNames(v)),'" not implemented.' - print*, "Options available are:" - print*, " * pressure" - print*, " * velocity" - print*, " * u" - print*, " * v" - print*, " * w" - end select -#endif - end do - -! -! Find the requested point in the mesh -! ------------------------------------ - self % active = mesh % FindPointWithCoords(self % x, self % eID, self % xi) -! -! Check whether the probe is located in other partition -! ----------------------------------------------------- - call self % LookInOtherPartitions -! -! Disable the probe if the point is not found -! ------------------------------------------- - if ( .not. self % active ) then - if ( MPI_Process % isRoot ) then - write(STD_OUT,'(A,I0,A)') "Probe ", ID, " was not successfully initialized." - print*, "Probe is set to inactive." - end if - - return - end if -! -! Set the fileName -! ---------------- - write( self % fileName , '(A,A,A,A)') trim(solution_file) , "." , & - trim(self % monitorName) , ".probe" - end if -! -! -! Return if the process does not contain the partition -! ---------------------------------------------------- - if ( self % rank .ne. MPI_Process % rank ) then - self % eID = 1 - return - end if - -! -! If this is not the first call, just reload the reference frame coordinates -! -------------------------------------------------------------------------- - if (.not. firstCall) self % active = mesh % elements(self % eID) % FindPointWithCoords(self % x,mesh % dir2D_ctrl, self % xi) -! -! Get the Lagrange interpolants -! ----------------------------- - associate(e => mesh % elements(self % eID)) - associate( spAxi => NodalStorage(e % Nxyz(1)), & - spAeta => NodalStorage(e % Nxyz(2)), & - spAzeta => NodalStorage(e % Nxyz(3)) ) - safedeallocate(self % lxi ) ; allocate( self % lxi(0 : e % Nxyz(1)) ) - safedeallocate(self % leta ) ; allocate( self % leta(0 : e % Nxyz(2)) ) - safedeallocate(self % lzeta) ; allocate( self % lzeta(0 : e % Nxyz(3)) ) - self % lxi = spAxi % lj(self % xi(1)) - self % leta = spAeta % lj(self % xi(2)) - self % lzeta = spAzeta % lj(self % xi(3)) -! -! Allocate storage for the probe -! ----------------------------- - safedeallocate(self % var ) ; allocate( self % var(0 : e % Nxyz(1),0 : e % Nxyz(2),0 : e % Nxyz(3)) ) - self % var = 0.0_RP - - if ( .not. self % isFileProbe ) then - !$acc enter data copyin(self) - !$acc enter data copyin(self % eiD) - !$acc enter data copyin(self % id) - !$acc enter data copyin(self % var) - !$acc enter data copyin(self % lxi) - !$acc enter data copyin(self % leta) - !$acc enter data copyin(self % lzeta) - end if -! -! **************** -! Prepare the file -! **************** -! -! Create file (skip for file-probes using HDF5 output) -! ----------------------------------------------------- - if (FirstCall .and. .not. (self % isFileProbe .and. trim(outputFormat) .eq. "HDF5")) then - open ( newunit = fID , file = trim(self % fileName) , status = "unknown" , action = "write" ) -! -! Write the file headers -! ---------------------- - write( fID , '(A20,A )') "Monitor name: ", trim(self % monitorName) - write( fID , '(A25,ES24.10,2(4X,ES24.10))') "x, y, z coordinates: ", self % x(1), self % x(2), self % x(3) - - write( fID , * ) - write( fID , '(A10,2X,A24)' , advance = "no") "Iteration" , "Time" - do v = 1 , self % nVars - write( fID , '(2X,A24)' , advance = "no") trim(self % variableNames(v)) - end do - write( fID , * ) - - close ( fID ) - end if - end associate - end associate - end subroutine Probe_Initialization - - subroutine Probe_Update(self, mesh, bufferPosition) - use Physics - use MPI_Process_Info - implicit none - class(Probe_t) :: self - type(HexMesh) :: mesh - integer :: bufferPosition -! -! --------------- -! Local variables -! --------------- -! - integer :: i, j, k, v, ierr - real(kind=RP) :: value - - if ( .not. self % active ) return - - if ( MPI_Process % rank .eq. self % rank ) then - -! -! Update the probe -! ---------------- - do v = 1, self % nVars - - if ( self % isFileProbe ) then -! -! CPU path for file-probes (no OpenACC to avoid runtime table overflow) -! ---------------------------------------------------------------------- - select case (trim(self % variableNames(v))) -#ifdef NAVIERSTOKES - case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) - end do ; end do ; end do - case("velocity") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("mach") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) - self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& - (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) - end do ; end do ; end do - case("k") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("rho") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do -#endif -#ifdef INCNS - case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) - end do ; end do ; end do - case("velocity") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - case("rho") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do -#endif -#ifdef MULTIPHASE - case("static-pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & - - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & - - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) - end do ; end do ; end do -#endif -#ifdef ACOUSTIC - case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAP,i,j,k) - end do ; end do ; end do - case("density") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAARHO,i,j,k) - end do ; end do ; end do - case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAU,i,j,k) - end do ; end do ; end do - case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAV,i,j,k) - end do ; end do ; end do - case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAW,i,j,k) - end do ; end do ; end do -#endif - end select - - value = 0.0_RP - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) - end do ; end do ; end do - - else -! -! GPU path for standard probes (OpenACC) -! --------------------------------------- - select case (trim(self % variableNames(v))) -#ifdef NAVIERSTOKES - case("pressure") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) - end do ; end do ; end do - !$acc end parallel loop - - case("velocity") - !$acc parallel loop collapse(3) present(mesh, self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("u") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("v") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("w") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("mach") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) ! Vabs**2 - self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& - (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) - end do ; end do ; end do - !$acc end parallel loop - - case("k") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("rho") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop -#endif -#ifdef INCNS - - case("pressure") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("velocity") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & - POW2( mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("u") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("v") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("w") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop - - case("rho") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - !$acc end parallel loop -#endif -#ifdef MULTIPHASE - case("static-pressure") - !$acc parallel loop collapse(3) present(mesh,self) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & - - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & - - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) - end do ; end do ; end do - !$acc end parallel loop -#endif -#ifdef ACOUSTIC - case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAP,i,j,k) - end do ; end do ; end do - - case("density") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAARHO,i,j,k) - end do ; end do ; end do - - case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAU,i,j,k) - end do ; end do ; end do - - case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAV,i,j,k) - end do ; end do ; end do - - case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAW,i,j,k) - end do ; end do ; end do -#endif - end select - - value = 0.0_RP - !$acc parallel loop collapse(3) present(mesh, self) reduction(+:value) async(self % ID) - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) - end do ; end do ; end do - !$acc end parallel loop - - !$acc wait - - end if ! isFileProbe - - self % values(v, bufferPosition) = value - - end do -! -#ifdef _HAS_MPI_ - if ( MPI_Process % doMPIAction ) then -! -! Share the result with the rest of the processes -! ----------------------------------------------- - call mpi_bcast(self % values(:,bufferPosition), self % nVars, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) - - end if -#endif - else -! -! Receive the result from the rank that contains the probe -! -------------------------------------------------------- -#ifdef _HAS_MPI_ - if ( MPI_Process % doMPIAction ) then - call mpi_bcast(self % values(:,bufferPosition), self % nVars, MPI_DOUBLE, self % rank, MPI_COMM_WORLD, ierr) - end if -#endif - end if - end subroutine Probe_Update - - subroutine Probe_ComputeLocal(self, mesh, bufferPosition) -! -! ************************************************************* -! CPU-only computation for file-probes, no MPI. -! Non-owning ranks set values to 0 so a caller can -! accumulate results with a single MPI_Allreduce(SUM). -! ************************************************************* -! - use Physics - use MPI_Process_Info - implicit none - class(Probe_t) :: self - type(HexMesh) :: mesh - integer :: bufferPosition -! -! --------------- -! Local variables -! --------------- -! - integer :: i, j, k, v - real(kind=RP) :: value - - if ( .not. self % active ) then - self % values(:, bufferPosition) = 0.0_RP - return - end if - - if ( MPI_Process % rank .ne. self % rank ) then - self % values(:, bufferPosition) = 0.0_RP - return - end if - - do v = 1, self % nVars - - select case (trim(self % variableNames(v))) -#ifdef NAVIERSTOKES - case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Pressure(mesh % elements(self % eID) % storage % Q(:,i,j,k)) - end do ; end do ; end do - case("velocity") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("mach") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/POW2(mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)) - self % var(i,j,k) = sqrt( self % var(i,j,k) / ( thermodynamics % gamma*(thermodynamics % gamma-1.0_RP)*& - (mesh % elements(self % eID) % storage % Q(IRHOE,i,j,k)/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k)-0.5_RP * self % var(i,j,k)) ) ) - end do ; end do ; end do - case("k") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = 0.5_RP * (POW2(mesh % elements(self % eID) % storage % Q(IRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(IRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do - case("rho") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IRHO,i,j,k) - end do ; end do ; end do -#endif -#ifdef INCNS - case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSP,i,j,k) - end do ; end do ; end do - case("velocity") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = sqrt(POW2(mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k)) + & - POW2(mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k)))/mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOU,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOV,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHOW,i,j,k) / mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do - case("rho") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(INSRHO,i,j,k) - end do ; end do ; end do -#endif -#ifdef MULTIPHASE - case("static-pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = mesh % elements(self % eID) % storage % Q(IMP,i,j,k) + mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*mesh % elements(self % eID) % storage % mu(1,i,j,k) & - - 12.0_RP*multiphase%sigma*multiphase%invEps*(POW2(mesh % elements(self % eID) % storage % Q(IMC,i,j,k)*(1.0_RP-mesh % elements(self % eID) % storage % Q(IMC,i,j,k)))) & - - 0.25_RP*3.0_RP*multiphase % sigma * multiphase % eps * (POW2(mesh % elements(self % eID) % storage % c_x(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_y(1,i,j,k))+POW2(mesh % elements(self % eID) % storage % c_z(1,i,j,k))) - end do ; end do ; end do -#endif -#ifdef ACOUSTIC - case("pressure") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAP,i,j,k) - end do ; end do ; end do - case("density") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAARHO,i,j,k) - end do ; end do ; end do - case("u") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAU,i,j,k) - end do ; end do ; end do - case("v") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAV,i,j,k) - end do ; end do ; end do - case("w") - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - self % var(i,j,k) = Q(ICAAW,i,j,k) - end do ; end do ; end do -#endif - end select - - value = 0.0_RP - do k = 0, mesh % elements(self % eID) % Nxyz(3) ; do j = 0, mesh % elements(self % eID) % Nxyz(2) ; do i = 0, mesh % elements(self % eID) % Nxyz(1) - value = value + self % var(i,j,k) * self % lxi(i) * self % leta(j) * self % lzeta(k) - end do ; end do ; end do - - self % values(v, bufferPosition) = value - - end do - - end subroutine Probe_ComputeLocal - - subroutine Probe_WriteLabel ( self ) -! -! ************************************************************* -! This subroutine writes the label for the probe, -! when invoked from the time integrator Display -! procedure. -! ************************************************************* -! - implicit none - class(Probe_t) :: self - - write(STD_OUT , '(3X,A10)' , advance = "no") trim(self % monitorName(1 : MONITOR_LENGTH)) - - end subroutine Probe_WriteLabel - - subroutine Probe_WriteValue ( self , bufferLine ) -! -! ************************************************************* -! This subroutine writes the monitor value for the time -! integrator Display procedure. -! ************************************************************* -! - implicit none - class(Probe_t) :: self - integer :: bufferLine - - write(STD_OUT , '(1X,A,1X,ES10.3)' , advance = "no") "|" , self % values ( 1 , bufferLine ) - - end subroutine Probe_WriteValue - - subroutine Probe_WriteToFile ( self , iter , t , no_of_lines) -! -! ************************************************************* -! This subroutine writes the buffer to the file. -! ************************************************************* -! - implicit none - class(Probe_t) :: self - integer :: iter(:) - real(kind=RP) :: t(:) - integer :: no_of_lines -! -! --------------- -! Local variables -! --------------- -! - integer :: i, v - integer :: fID - - if ( .not. self % active ) then - if ( no_of_lines .ne. 0 ) self % values(:,1) = self % values(:,no_of_lines) - return - end if - - if ( MPI_Process % isRoot ) then - open( newunit = fID , file = trim ( self % fileName ) , action = "write" , access = "append" , status = "old" ) - - do i = 1 , no_of_lines - if ( self % saveTimestep > 0.0_RP ) then - if ( t(i) < self % lastSavedTime + self % saveTimestep ) cycle - end if - write( fID , '(I10,2X,ES24.16)' , advance = "no" ) iter(i) , t(i) - do v = 1 , self % nVars - write( fID , '(2X,ES24.16)' , advance = "no" ) self % values(v,i) - end do - write( fID , * ) - self % lastSavedTime = t(i) - end do - - close ( fID ) - end if - - - if ( no_of_lines .ne. 0 ) self % values(:,1) = self % values(:,no_of_lines) - - end subroutine Probe_WriteToFile - - subroutine Probe_LookInOtherPartitions(self) - use MPI_Process_Info - implicit none - class(Probe_t) :: self - integer :: allActives(MPI_Process % nProcs) - integer :: active, ierr - - if ( MPI_Process % doMPIAction ) then -#ifdef _HAS_MPI_ -! -! Cast the logicals onto integers -! ------------------------------- - if ( self % active ) then - active = 1 - else - active = 0 - end if -! -! Gather all data from all processes -! ---------------------------------- - call mpi_allgather(active, 1, MPI_INT, allActives, 1, MPI_INT, MPI_COMM_WORLD, ierr) -! -! Check if any of them found the probe -! ------------------------------------ - if ( any(allActives .eq. 1) ) then -! -! Assign the domain of the partition that contains the probe -! ---------------------------------------------------------- - self % active = .true. - self % rank = maxloc(allActives, dim = 1) - 1 - - else -! -! Disable the probe -! ----------------- - self % active = .false. - self % rank = -1 - - end if -#endif - else -! -! Without MPI select the rank 0 as default -! ---------------------------------------- - self % rank = 0 - - end if - - end subroutine Probe_LookInOtherPartitions - - elemental subroutine Probe_Destruct (self) - implicit none - class(Probe_t), intent(inout) :: self - - safedeallocate (self % values) - safedeallocate (self % lxi) - safedeallocate (self % leta) - safedeallocate (self % lzeta) - safedeallocate (self % variableNames) - end subroutine Probe_Destruct - - elemental subroutine Probe_Assign (to, from) - implicit none - class(Probe_t), intent(inout) :: to - type(Probe_t) , intent(in) :: from - - to % active = from % active - to % isFileProbe = from % isFileProbe - to % rank = from % rank - to % ID = from % ID - to % eID = from % eID - to % nVars = from % nVars - to % x = from % x - to % xi = from % xi - - if ( allocated(from % values) ) then - safedeallocate ( to % values ) - allocate ( to % values ( size(from % values, 1), size(from % values, 2) ) ) - to % values = from % values - end if - - if ( allocated(from % lxi) ) then - safedeallocate ( to % lxi ) - allocate ( to % lxi ( size(from % lxi) ) ) - to % lxi = from % lxi - end if - - if ( allocated(from % leta) ) then - safedeallocate ( to % leta ) - allocate ( to % leta ( size(from % leta) ) ) - to % leta = from % leta - end if - - if ( allocated(from % lzeta) ) then - safedeallocate ( to % lzeta ) - allocate ( to % lzeta ( size(from % lzeta) ) ) - to % lzeta = from % lzeta - end if - - to % saveTimestep = from % saveTimestep - to % lastSavedTime = from % lastSavedTime - to % fileName = from % fileName - to % monitorName = from % monitorName - - safedeallocate ( to % variableNames ) - allocate ( to % variableNames ( size(from % variableNames) ) ) - to % variableNames = from % variableNames - - end subroutine Probe_Assign - -end module ProbeClass -#endif diff --git a/Solver/src/libs/mesh/SurfaceIntegrals.f90 b/Solver/src/libs/mesh/SurfaceIntegrals.f90 deleted file mode 100644 index 6807203c8..000000000 --- a/Solver/src/libs/mesh/SurfaceIntegrals.f90 +++ /dev/null @@ -1,1269 +0,0 @@ -#include "Includes.h" -#if defined(NAVIERSTOKES) || (defined(INCNS)) -module SurfaceIntegrals - use SMConstants - use PhysicsStorage - use Physics - use FaceClass - use ElementClass - use HexMeshClass - use VariableConversion, only: Pressure - use NodalStorageClass -#ifdef _HAS_MPI_ - use mpi -#endif - implicit none - - private - public SURFACE, TOTAL_FORCE, PRESSURE_FORCE, VISCOUS_FORCE, MASS_FLOW, FLOW_RATE, PRESSURE_DISTRIBUTION - public ScalarSurfaceIntegral, VectorSurfaceIntegral, ScalarDataReconstruction, VectorDataReconstruction - - integer, parameter :: SURFACE = 1 - integer, parameter :: TOTAL_FORCE = 2 - integer, parameter :: PRESSURE_FORCE = 3 - integer, parameter :: VISCOUS_FORCE = 4 - integer, parameter :: MASS_FLOW = 5 - integer, parameter :: FLOW_RATE = 6 - integer, parameter :: PRESSURE_DISTRIBUTION = 7 - integer, parameter :: USER_DEFINED = 99 - !$acc declare copyin(SURFACE, TOTAL_FORCE, PRESSURE_FORCE, VISCOUS_FORCE, MASS_FLOW, FLOW_RATE, PRESSURE_DISTRIBUTION, USER_DEFINED) -! -! ======== - contains -! ======== -! -!//////////////////////////////////////////////////////////////////////////////////////// -! -! SCALAR INTEGRALS PROCEDURES -! -!//////////////////////////////////////////////////////////////////////////////////////// -! - function ScalarSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) -! -! ----------------------------------------------------------- -! This function computes scalar integrals, that is, those -! in the form: -! val = \int \vec{v}·\vec{n}dS -! Implemented integrals are: -! * Surface: computes the zone surface. -! * Mass flow: computes the mass flow across the zone. -! * Flow: computes the volumetric flow across the zone. -! ----------------------------------------------------------- -! - implicit none - class(HexMesh), intent(inout), target :: mesh - integer, intent(in) :: zoneID - integer, intent(in) :: integralType, iter - real(kind=RP) :: val, localval -! -! --------------- -! Local variables -! --------------- -! - integer :: zonefID, fID, eID, fIDs(6), ierr -! -! Initialization -! -------------- - val = 0.0_RP - - call ProlongToFaces(mesh, zoneID) -! -! Loop the zone to get faces and elements -! --------------------------------------- -#ifdef _OPENACC -!$acc parallel loop gang reduction(+:val) present(mesh) -#else -!$omp parallel do private(fID) reduction(+:val) schedule(runtime) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces -! -! Face global ID -! -------------- - fID = mesh % zones(zoneID) % faces(zonefID) -! -! Compute the integral -! -------------------- - val = val + ScalarSurfaceIntegral_Face(mesh % faces(fID), integralType) - - end do -#ifdef _OPENACC -!$acc end parallel loop -#else -!$omp end parallel do -#endif - -#ifdef _HAS_MPI_ - localval = val - call mpi_allreduce(localval, val, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) -#endif - - end function ScalarSurfaceIntegral - - function ScalarSurfaceIntegral_Face(f, integralType) result(val) - !$acc routine vector - implicit none - class(Face), intent(in) :: f - integer, intent(in) :: integralType - real(kind=RP) :: val -! -! --------------- -! Local variables -! --------------- -! - integer :: i, j ! Face indices - real(kind=RP) :: p -! -! Initialization -! -------------- - val = 0.0_RP -! -! Perform the numerical integration -! --------------------------------- - select case ( integralType ) - case ( SURFACE ) -! -! ********************************** -! Computes the surface integral -! val = \int dS -! ********************************** -! - !$acc loop vector collapse(2) reduction(+:val) - do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) - val = val + NodalStorage(f % Nf(1)) % w(i) & - * NodalStorage(f % Nf(2)) % w(j) & - * f % geom % jacobian(i,j) - end do ; end do - - case ( MASS_FLOW ) -! -! *********************************** -! Computes the mass-flow integral -! I = \int rho \vec{v}·\vec{n}dS -! *********************************** -! -#if defined(NAVIERSTOKES) - !$acc loop vector collapse(2) reduction(+:val) - do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) -! -! Compute the integral -! -------------------- - val = val +(f % storage(1) % Q(IRHOU,i,j) * f % geom % normal(1,i,j) & - + f % storage(1) % Q(IRHOV,i,j) * f % geom % normal(2,i,j) & - + f % storage(1) % Q(IRHOW,i,j) * f % geom % normal(3,i,j) ) & - * NodalStorage(f % Nf(1)) % w(i) & - * NodalStorage(f % Nf(2)) % w(j) & - * f % geom % jacobian(i,j) - - end do ; end do -#endif -#if defined(INCNS) - !$acc loop vector collapse(2) reduction(+:val) - do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) -! -! Compute the integral -! -------------------- - val = val +(f % storage(1) % Q(INSRHOU,i,j) * f % geom % normal(1,i,j) & - + f % storage(1) % Q(INSRHOV,i,j) * f % geom % normal(2,i,j) & - + f % storage(1) % Q(INSRHOW,i,j) * f % geom % normal(3,i,j) ) & - * NodalStorage(f % Nf(1)) % w(i) & - * NodalStorage(f % Nf(2)) % w(j) & - * f % geom % jacobian(i,j) - - end do ; end do -#endif - case ( FLOW_RATE ) -! -! *********************************** -! Computes the flow integral -! val = \int \vec{v}·\vec{n}dS -! *********************************** -! -#if defined(NAVIERSTOKES) - !$acc loop vector collapse(2) reduction(+:val) - do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) -! -! Compute the integral -! -------------------- - val = val + (1.0_RP / f % storage(1) % Q(IRHO,i,j))*(f % storage(1) % Q(IRHOU,i,j) * f % geom % normal(1,i,j) & - + f % storage(1) % Q(IRHOV,i,j) * f % geom % normal(2,i,j) & - + f % storage(1) % Q(IRHOW,i,j) * f % geom % normal(3,i,j) ) & - * NodalStorage(f % Nf(1)) % w(i) & - * NodalStorage(f % Nf(2)) % w(j) & - * f % geom % jacobian(i,j) - end do ; end do -#endif -#if defined(INCNS) - !$acc loop vector collapse(2) reduction(+:val) - do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) -! -! Compute the integral -! -------------------- - val = val + (1.0_RP / f % storage(1) % Q(INSRHO,i,j))*(f % storage(1) % Q(INSRHOU,i,j) * f % geom % normal(1,i,j) & - + f % storage(1) % Q(INSRHOV,i,j) * f % geom % normal(2,i,j) & - + f % storage(1) % Q(INSRHOW,i,j) * f % geom % normal(3,i,j) ) & - * NodalStorage(f % Nf(1)) % w(i) & - * NodalStorage(f % Nf(2)) % w(j) & - * f % geom % jacobian(i,j) - end do ; end do -#endif - case ( PRESSURE_FORCE ) -! -! *********************************** -! Computes the pressure integral -! val = \int pdS -! *********************************** -! - !$acc loop vector collapse(2) reduction(+:val) - do j = 0, f % Nf(2) ; do i = 0, f % Nf(1) -! -! Compute the integral -! -------------------- - p = Pressure(f % storage(1) % Q(:,i,j)) - val = val + p * NodalStorage(f % Nf(1)) % w(i) & - * NodalStorage(f % Nf(2)) % w(j) & - * f % geom % jacobian(i,j) - end do ; end do - - - case ( USER_DEFINED ) ! TODO - end select - - end function ScalarSurfaceIntegral_Face -! -!//////////////////////////////////////////////////////////////////////////////////////// -! -! VECTOR INTEGRALS PROCEDURES -! -!//////////////////////////////////////////////////////////////////////////////////////// -! - function VectorSurfaceIntegral(mesh, zoneID, integralType, iter) result(val) -! -! ----------------------------------------------------------- -! This function computes scalar integrals, that is, those -! in the form: -! val = \int \vec{v}·\vec{n}dS -! Implemented integrals are: -! * Surface: computes the zone surface. -! * Mass flow: computes the mass flow across the zone. -! * Flow: computes the volumetric flow across the zone. -! ----------------------------------------------------------- -! -#ifdef _HAS_MPI_ - use mpi -#endif - implicit none - type(HexMesh), intent(inout) :: mesh - integer, intent(in) :: zoneID - integer, intent(in) :: integralType, iter - real(kind=RP) :: val(NDIM) -! -! --------------- -! Local variables -! --------------- -! - integer :: zonefID, fID, eID, i, j, ierr - real(kind=RP) :: localval(NDIM) - real(kind=RP) :: valx, valy, valz - real(kind=RP) :: localx, localy, localz - real(kind=RP) :: p, tau(1:NDIM, 1:NDIM) -! -! Initialization -! -------------- - val = 0.0_RP - localval = 0.0_RP - valx = 0.0_RP - valy = 0.0_RP - valz = 0.0_RP - - call ProlongToFaces(mesh, zoneID) - - select case ( integralType ) - case ( SURFACE ) - -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) -#else -!$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - ! - ! Face global ID - ! -------------- - fID = mesh % zones(zoneID) % faces(zonefID) - - localx = 0.0_RP - localy = 0.0_RP - localz = 0.0_RP - ! - ! ********************************** - ! Computes the surface integral - ! val = \int \vec{n} dS - ! ********************************** - ! - !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(val) - do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) - val = NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) & - * mesh % faces(fid) % geom % jacobian(i,j) * mesh % faces(fid) % geom % normal(:,i,j) - - localx = localx + val(1) - localy = localy + val(2) - localz = localz + val(3) - - end do ; end do - - valx = valx + localx - valy = valy + localy - valz = valz + localz - - end do -#ifdef _OPENACC -!$acc end parallel loop -#else -!$omp end parallel do -#endif - - case ( TOTAL_FORCE ) - -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) -#else -!$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - ! - ! Face global ID - ! -------------- - fID = mesh % zones(zoneID) % faces(zonefID) - - localx = 0.0_RP - localy = 0.0_RP - localz = 0.0_RP - ! - ! ************************************************ - ! Computes the total force experienced by the zone - ! F = \int p \vec{n}ds - \int tau'·\vec{n}ds - ! ************************************************ - ! - !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(tau, localval) - do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) - ! - ! Compute the integral - ! -------------------- - p = Pressure(mesh % faces(fid) % storage(1) % Q(:,i,j)) - call getStressTensor(mesh % faces(fid) % storage(1) % Q(:,i,j), mesh % faces(fid) % storage(1) % U_x(:,i,j),& - mesh % faces(fid) % storage(1) % U_y(:,i,j),mesh % faces(fid) % storage(1) % U_z(:,i,j), tau) - - localval = ( p * mesh % faces(fid) % geom % normal(:,i,j) - matmul(tau,mesh % faces(fid) % geom % normal(:,i,j)) ) & - * mesh % faces(fid) % geom % jacobian(i,j) & - * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - - localx = localx + localval(1) - localy = localy + localval(2) - localz = localz + localval(3) - end do ; end do - - valx = valx + localx - valy = valy + localy - valz = valz + localz - - end do -#ifdef _OPENACC -!$acc end parallel loop -#else -!$omp end parallel do -#endif - - case ( PRESSURE_FORCE ) - -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) -#else -!$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - ! - ! Face global ID - ! -------------- - fID = mesh % zones(zoneID) % faces(zonefID) - - localx = 0.0_RP - localy = 0.0_RP - localz = 0.0_RP - ! - ! **************************************************** - ! Computes the pressure forces experienced by the zone - ! F = \int p \vec{n}ds - ! **************************************************** - ! - !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(val) - do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) - ! - ! Compute the integral - ! -------------------- - p = Pressure(mesh % faces(fid) % storage(1) % Q(:,i,j)) - - val = ( p * mesh % faces(fid) % geom % normal(:,i,j) ) * mesh % faces(fid) % geom % jacobian(i,j) & - * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) & - * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - - localx = localx + val(1) - localy = localy + val(2) - localz = localz + val(3) - end do ; end do - - valx = valx + localx - valy = valy + localy - valz = valz + localz - - end do -#ifdef _OPENACC -!$acc end parallel loop -#else -!$omp end parallel do -#endif - - case ( VISCOUS_FORCE ) - -#ifdef _OPENACC -!$acc parallel loop gang present(mesh) num_gangs(mesh % zones(zoneID) % no_of_faces) reduction(+:valx, valy, valz) -#else -!$omp parallel do private(fID,localVal,localx,localy,localz) reduction(+:valx, valy, valz) schedule(runtime) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - ! - ! Face global ID - ! -------------- - fID = mesh % zones(zoneID) % faces(zonefID) - - localx = 0.0_RP - localy = 0.0_RP - localz = 0.0_RP - ! - ! ************************************************ - ! Computes the total force experienced by the zone - ! F = - \int tau'·\vec{n}ds - ! ************************************************ - ! - !$acc loop vector collapse(2) reduction(+:localx, localy, localz) private(tau, val) - do j = 0, mesh % faces(fid) % Nf(2) ; do i = 0, mesh % faces(fid) % Nf(1) - ! - ! Compute the integral - ! -------------------- - call getStressTensor(mesh % faces(fid) % storage(1) % Q(:,i,j), mesh % faces(fid) % storage(1) % U_x(:,i,j),& - mesh % faces(fid) % storage(1) % U_y(:,i,j),mesh % faces(fid) % storage(1) % U_z(:,i,j), tau) - - val = - matmul(tau,mesh % faces(fid) % geom % normal(:,i,j)) * mesh % faces(fid) % geom % jacobian(i,j) & - * NodalStorage(mesh % faces(fid) % Nf(1)) % w(i) & - * NodalStorage(mesh % faces(fid) % Nf(2)) % w(j) - - localx = localx + val(1) - localy = localy + val(2) - localz = localz + val(3) - end do ; end do - - valx = valx + localx - valy = valy + localy - valz = valz + localz - - end do -#ifdef _OPENACC -!$acc end parallel loop -#else -!$omp end parallel do -#endif - - end select - - val(1:3) = [valx, valy, valz] - -#ifdef _HAS_MPI_ - localVal = val - call mpi_allreduce(localVal, val, NDIM, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) -#endif - - end function VectorSurfaceIntegral -! -!//////////////////////////////////////////////////////////////////////////////////////// -! -! INTEGRALS PROCEDURES FOR IBM DATA RECONSTRUCTION -! -! SURFACE INTEGRALS -! -!//////////////////////////////////////////////////////////////////////////////////////// - subroutine ScalarDataReconstruction( IBM, elements, STLNum, integralType, iter, autosave, dt ) - use TessellationTypes - use MappedGeometryClass - use IBMClass - use OrientedBoundingBox - use KDClass - use MPI_Process_Info - use MPI_IBMUtilities -#ifdef _HAS_MPI_ - use mpi -#endif -! -! ----------------------------------------------------------------------------------------- -! This function computes Scalar integrals, that is, those -! in the form: -! val = \int \vec{v}·\vec{n}dS -! The data at the boundary point (BP) is computed through a Inverse Distance Weight -! procedure. -! ----------------------------------------------------------------------------------------- - implicit none - !-arguments-------------------------------------------------------- - type(IBM_type), intent(inout) :: IBM - type(element), intent(inout) :: elements(:) - integer, intent(in) :: integralType, STLNum, iter - real(kind=RP), intent(in) :: dt - !-local-variables------------------------------------------------- - real(kind=rp), allocatable :: Qsurf(:,:), U_xsurf(:,:), U_ysurf(:,:), U_zsurf(:,:) - integer :: i, j - logical :: found, autosave - - if( .not. IBM% Integral(STLNum)% compute ) return - - allocate( Qsurf(NCONS,IBM% NumOfInterPoints) ) - call IBM% BandPoint_state( elements, STLNum, .true. ) - - if( IBM% stlSurfaceIntegrals(STLNum)% move ) then - if( IBM% stlSurfaceIntegrals(STLNum)% motionType .eq. ROTATION ) then - call IBM% stlSurfaceIntegrals(STLNum)% getRotationaMatrix( dt ) - call OBB(STLNum)% STL_rotate( IBM% stlSurfaceIntegrals(STLNum), .true. ) - elseif( IBM% stlSurfaceIntegrals(STLNum)% motionType .eq. LINEAR ) then - call IBM% stlSurfaceIntegrals(STLNum)% getDisplacement( dt ) - call OBB(STLNum)% STL_translate( IBM% stlSurfaceIntegrals(STLNum), .true. ) - end if - end if - - if( .not. MPI_Process% isRoot ) return -!$omp parallel -!$omp do schedule(runtime) private(j,found) - do i = 1, IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs - - do j = 1, NumOfVertices + 4 - call GetSurfaceState( IBM, IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i), IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j), STLNum ) - Qsurf = IBM% BandRegion(STLNum)% Q(:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) - end do - - do j = 1, NumOfVertices + 4 - IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% ScalarValue = IntegratedScalarValue( Q = Qsurf, & - vertex = IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j), & - normal = IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% normal, & - integralType = integralType, & - InterpolationType = IBM% InterpolationType ) - end do - end do -!$omp end do -!$omp end parallel - if( IBM% stl(STLNum)% move ) then - IBM% Integral(STLNum)% ListComputed = .false. - else - IBM% Integral(STLNum)% ListComputed = .true. - end if - - if( autosave ) call GenerateScalarmonitorTECfile( IBM, STLNum, integralType, iter ) - - end subroutine ScalarDataReconstruction -! -!//////////////////////////////////////////////////////////////////////////////////////// -! -! VECTOR INTEGRALS -! -!//////////////////////////////////////////////////////////////////////////////////////// - subroutine VectorDataReconstruction( IBM, elements, STLNum, integralType, iter, autosave, dt ) - use TessellationTypes - use MappedGeometryClass - use IBMClass - use OrientedBoundingBox - use KDClass - use MPI_Process_Info - use MPI_IBMUtilities - use omp_lib -#ifdef _HAS_MPI_ - use mpi -#endif -! -! ----------------------------------------------------------------------------------------- -! This function computes Vector integrals, that is, those -! in the form: -! val = \int \vec{v}·\vec{n}dS -! The data at the boundary point (BP) is computed through a Inverse Distance Weight -! procedure. -! ----------------------------------------------------------------------------------------- - implicit none - !-arguments--------------------------------------------------------------------------------- - type(IBM_type), intent(inout) :: IBM - type(element), intent(inout) :: elements(:) - integer, intent(in) :: integralType, STLNum, iter - real(kind=RP), intent(in) :: dt - !-local-variables--------------------------------------------------------------------------- - real(kind=rp), allocatable :: Qsurf(:,:,:), U_xsurf(:,:,:), U_ysurf(:,:,:), U_zsurf(:,:,:) - integer :: i, j - logical :: found, autosave - - if( .not. IBM% Integral(STLNum)% compute ) return - - allocate( Qsurf(NCONS,IBM% NumOfInterPoints,NumOfVertices + 4), & - U_xsurf(NCONS,IBM% NumOfInterPoints,NumOfVertices + 4), & - U_ysurf(NCONS,IBM% NumOfInterPoints,NumOfVertices + 4), & - U_zsurf(NCONS,IBM% NumOfInterPoints,NumOfVertices + 4) ) - call IBM% BandPoint_state( elements, STLNum, .true. ) - - if( .not. MPI_Process% isRoot ) return - - if( IBM% stlSurfaceIntegrals(STLNum)% move ) then - if( IBM% stlSurfaceIntegrals(STLNum)% motionType .eq. ROTATION ) then - call IBM% stlSurfaceIntegrals(STLNum)% getRotationaMatrix( dt ) - call OBB(STLNum)% STL_rotate( IBM% stlSurfaceIntegrals(STLNum), .true. ) - elseif( IBM% stlSurfaceIntegrals(STLNum)% motionType .eq. LINEAR ) then - call IBM% stlSurfaceIntegrals(STLNum)% getDisplacement( dt ) - call OBB(STLNum)% STL_translate( IBM% stlSurfaceIntegrals(STLNum), .true. ) - end if - end if -!$omp parallel -!$omp do schedule(runtime) private(j,found,Qsurf,U_xsurf,U_ysurf,U_zsurf) - do i = 1, IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs - - do j = 1, NumOfVertices + 4 - call GetSurfaceState( IBM, IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i), IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j), STLNum ) - - Qsurf(:,:,j) = IBM% BandRegion(STLNum)% Q (:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) - U_xsurf(:,:,j) = IBM% BandRegion(STLNum)% U_x(:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) - U_ysurf(:,:,j) = IBM% BandRegion(STLNum)% U_y(:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) - U_zsurf(:,:,j) = IBM% BandRegion(STLNum)% U_z(:,IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% nearestPoints) - end do - - do j = 1, NumOfVertices + 4 - IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j)% VectorValue = IntegratedVectorValue( Q = Qsurf(:,:,j), & - U_x = U_xsurf(:,:,j), & - U_y = U_ysurf(:,:,j), & - U_z = U_zsurf(:,:,j), & - vertex = IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% vertices(j), & - normal = IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i)% normal, & - y = IBM% IP_Distance, & - Wallfunction = IBM% Wallfunction, & - integralType = integralType, & - InterpolationType = IBM% InterpolationType ) - end do - end do -!$omp end do -!$omp end parallel - deallocate( Qsurf, U_xsurf, U_ysurf, U_zsurf ) - - if( IBM% stl(STLNum)% move ) then - IBM% Integral(STLNum)% ListComputed = .false. - else - IBM% Integral(STLNum)% ListComputed = .true. - end if - - if( autosave ) call GenerateVectormonitorTECfile( IBM, STLNum, integralType, iter ) - - end subroutine VectorDataReconstruction - - subroutine GetSurfaceState( IBM, obj, vertex, STLNum ) - use TessellationTypes - use IBMClass - use MPI_Process_Info - use omp_lib -#ifdef _HAS_MPI_ - use mpi -#endif - implicit none - - class(IBM_type), intent(inout) :: IBM - type(object_type), intent(inout) :: obj - type(point_type), intent(inout) :: vertex - integer, intent(in) :: STLNum - - real(kind=RP) :: Dist - integer :: i, j, k - - if( IBM% Integral(STLNum)% ListComputed ) return - - vertex% nearestPoints = 0 - do k = 1, IBM% NumOfInterPoints - if( IBM% Wallfunction ) then - call MinimumDistancePoints( vertex% coords + IBM% IP_Distance * obj% Normal, & - IBM% rootPoints(STLNum), IBM% BandRegion(STLNum), & - Dist, k, vertex% nearestPoints ) - else - call MinimumDistancePoints( vertex% coords, IBM% rootPoints(STLNum), & - IBM% BandRegion(STLNum), Dist, k, & - vertex% nearestPoints ) - end if - end do - call GetMatrixInterpolationSystem( vertex% coords, & - IBM% BandRegion(STLNum)% x(vertex% nearestPoints), & - vertex% invPhi, & - vertex% b, IBM% InterpolationType ) - - end subroutine GetSurfaceState - - subroutine GetSurfaceState_HO( IBM, obj, vertex, STLNum, elements, Qs, U_xs, U_ys, U_zs, gradients, found ) - use TessellationTypes - use IBMClass - use MPI_Process_Info - use omp_lib -#ifdef _HAS_MPI_ - use mpi -#endif - implicit none - - class(IBM_type), intent(in) :: IBM - type(object_type), intent(inout) :: obj - type(point_type), intent(inout) :: vertex - integer, intent(in) :: STLNum - type(element), intent(inout) :: elements(:) - real(kind=RP), intent(inout) :: Qs(NCONS,1) - real(kind=RP), intent(inout) :: U_xs(NCONS,1), U_ys(NCONS,1), U_zs(NCONS,1) - logical, intent(in) :: gradients - logical, intent(out) :: found - - real(kind=RP) :: xi(NDIM) - integer :: eID, i, j, k - - Qs = 0.0_RP - if( gradients ) then - U_xs = 0.0_RP; U_ys = 0.0_RP; U_zs = 0.0_RP - end if - - if( IBM% Integral(STLNum)% ListComputed ) then - if( vertex% partition .eq. MPI_Process% rank ) then - eID = vertex% element_index - xi = vertex% xi - - associate( e => elements(eID) ) - - Qs(:,1) = elements(eID)% EvaluateSolutionAtPoint(NCONS, xi) - - if( gradients ) then - U_xs(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IX) - U_ys(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IY) - U_zs(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IZ) - end if - - end associate - found = .true. - else - found = .false. - end if - return - end if - - do eID = 1, size(elements) - associate(e => elements(eID) ) - found = e% FindPointWithCoords( vertex% coords, 0, xi ) - if( found ) then - vertex% element_index = eID - vertex% partition = MPI_Process% rank - vertex% xi = xi - - Qs(:,1) = elements(eID)% EvaluateSolutionAtPoint(NCONS, xi) - - if( gradients ) then - U_xs(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IX) - U_ys(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IY) - U_zs(:,1) = elements(eID)% EvaluateGradientAtPoint(NCONS, xi, IZ) - end if - exit - end if - end associate - end do - - end subroutine GetSurfaceState_HO -! -!//////////////////////////////////////////////////////////////////////////////////////// -! -! INVERSE DISTANCE WEIGHTED INTERPOLATION PROCEDURES FOR IBM DATA RECONSTRUCTION -! -! SCALAR INTERPOLATION -! -!//////////////////////////////////////////////////////////////////////////////////////// - function IntegratedScalarValue( Q, vertex, normal, integralType, InterpolationType ) result( outvalue ) - use IBMClass - implicit none -! -! ----------------------------------------------------------- -! This function computes the IDW interpolat for a scalar -! quantity in the point "Point". -! Available scalars are: -! Mass flow -! Flow rate -! Pressure -! ----------------------------------------------------------- - !-arguments-------------------------------------------------------------- - real(kind=rp), intent(in) :: Q(:,:), normal(:) - type(point_type), intent(inout) :: vertex - integer, intent(in) :: integralType, InterpolationType - real(kind=rp) :: outvalue - !-local-variables-------------------------------------------------------- - real(kind=rp) :: Qi(NCONS), P - integer :: i - - outvalue = 0.0_RP - - select case( integralType ) - - case( MASS_FLOW ) - - do i = 1, NCONS - Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) - end do - -#if defined(NAVIERSTOKES) - outvalue = - (1.0_RP / Qi(IRHO))*(Qi(IRHOU)*normal(1) + Qi(IRHOV)*normal(2) + Qi(IRHOW)*normal(3)) -#endif -#if defined(INCNS) - outvalue = - (1.0_RP / Qi(INSRHO))*(Qi(INSRHOU)*normal(1) + Qi(INSRHOV)*normal(2) + Qi(INSRHOW)*normal(3)) -#endif - case ( FLOW_RATE ) - - do i = 1, NCONS - Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) - end do - -#if defined(NAVIERSTOKES) - outvalue = - (Qi(IRHOU)*normal(1) + Qi(IRHOV)*normal(2) + Qi(IRHOW)*normal(3)) -#endif -#if defined(INCNS) - outvalue = - (Qi(INSRHOU)*normal(1) + Qi(INSRHOV)*normal(2) + Qi(INSRHOW)*normal(3)) -#endif - case( PRESSURE_DISTRIBUTION ) - - do i = 1, NCONS - Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) - end do - - outvalue = pressure(Qi) - case ( USER_DEFINED ) ! TODO - - end select - - end function IntegratedScalarValue -! -!//////////////////////////////////////////////////////////////////////////////////////// -! -! VECTOR INTERPOLATION -! -!//////////////////////////////////////////////////////////////////////////////////////// - function IntegratedVectorValue( Q, U_x, U_y, U_z, vertex, normal, & - y, Wallfunction, integralType, & - InterpolationType ) result( outvalue ) - use IBMClass - use VariableConversion - use FluidData -#if defined(NAVIERSTOKES) - use WallFunctionBC -#endif - implicit none -! -! ----------------------------------------------------------- -! This function computes the IDW interpolat for a vector -! quantity in the point "Point". -! Available scalars are: -! Total force -! Pressure force -! Viscous force -! ----------------------------------------------------------- - !-arguments----------------------------------------------------------------- - real(kind=rp), intent(in) :: Q(:,:), U_x(:,:), U_y(:,:), & - U_z(:,:), normal(NDIM) - type(point_type), intent(inout) :: vertex - real(kind=rp), intent(in) :: y - logical, intent(in) :: Wallfunction - integer, intent(in) :: integralType, InterpolationType - real(kind=rp) :: outvalue(NDIM) - !-local-variables----------------------------------------------------------- - integer :: i - real(kind=rp) :: viscStress(NDIM), U(NDIM), U_t(NDIM), tangent(NDIM), & - Qi(NCONS), U_xi(NCONS), U_yi(NCONS), U_zi(NCONS), & - tau(NDIM,NDIM), P, T, T_w, rho_w, mu, nu, u_II, u_tau, & - tau_w, kappa_ - - outvalue = 0.0_RP - - select case( integralType ) - - case ( TOTAL_FORCE ) - - do i = 1, NCONS - Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) - end do - - P = pressure(Qi) - - if( Wallfunction ) then -#if defined(NAVIERSTOKES) - T = Temperature(Qi) - call get_laminar_mu_kappa(Qi,mu,kappa_) - nu = mu/Qi(IRHO) - - U = Qi(IRHOU:IRHOW)/Qi(IRHO) - U_t = U - ( dot_product(U,normal) * normal ) - - tangent = U_t/norm2(U_t) - - u_II = dot_product(U,tangent) - - u_tau = u_tau_f( u_II, y, nu, u_tau0=0.1_RP ) - - T_w = T + (dimensionless% Pr)**(1._RP/3._RP)/(2.0_RP*thermodynamics% cp) * POW2(u_II) - T_w = T_w * refvalues% T - rho_w = P*refvalues% p/(thermodynamics% R * T_w) - rho_w = rho_w/refvalues% rho -#endif - tau_w = rho_w*POW2(u_tau) - - viscStress = tau_w*tangent - else - - do i = 1, NCONS - U_xi(i) = GetInterpolatedValue( U_x(i,:), vertex% invPhi, vertex% b, InterpolationType ) - U_yi(i) = GetInterpolatedValue( U_y(i,:), vertex% invPhi, vertex% b, InterpolationType ) - U_zi(i) = GetInterpolatedValue( U_z(i,:), vertex% invPhi, vertex% b, InterpolationType ) - end do - - call getStressTensor(Qi, U_xi, U_yi, U_zi, tau) - - viscStress = matmul(tau,normal) - end if - - outvalue = -P * normal + viscStress - - case( PRESSURE_FORCE ) - - do i = 1, NCONS - Qi(i) = GetInterpolatedValue( Q(i,:), vertex% invPhi, vertex% b, InterpolationType ) - end do - - P = pressure(Qi) - - outvalue = -P * normal - - case( VISCOUS_FORCE ) - - if( Wallfunction ) then -#if defined(NAVIERSTOKES) - T = Temperature(Qi) - call get_laminar_mu_kappa(Qi,mu,kappa_) - nu = mu/Qi(IRHO) - - U = Qi(IRHOU:IRHOW)/Qi(IRHO) - U_t = U - ( dot_product(U,normal) * normal ) - - tangent = U_t/norm2(U_t) - - u_II = dot_product(U,tangent) - - u_tau = u_tau_f( u_II, y, nu, u_tau0=0.1_RP ) - - T_w = T + (dimensionless% Pr)**(1._RP/3._RP)/(2.0_RP*thermodynamics% cp) * POW2(u_II) - T_w = T_w * refvalues% T - rho_w = P*refvalues% p/(thermodynamics% R * T_w) - rho_w = rho_w/refvalues% rho -#endif - tau_w = rho_w*POW2(u_tau) - - viscStress = tau_w*tangent - else - - do i = 1, NCONS - U_xi(i) = GetInterpolatedValue( U_x(i,:), vertex% invPhi, vertex% b, InterpolationType ) - U_yi(i) = GetInterpolatedValue( U_y(i,:), vertex% invPhi, vertex% b, InterpolationType ) - U_zi(i) = GetInterpolatedValue( U_z(i,:), vertex% invPhi, vertex% b, InterpolationType ) - end do - - call getStressTensor(Qi, U_xi, U_yi, U_zi, tau) - - viscStress = matmul(tau,normal) - end if - - outvalue = viscStress - - case ( USER_DEFINED ) ! TODO - - end select - - end function IntegratedVectorValue - - subroutine GenerateScalarmonitorTECfile( IBM, STLNum, integralType, iter ) - use MPI_Process_Info - use TessellationTypes - use MPI_IBMUtilities - use IBMClass - implicit none - !-arguments------------------------------------------------------- - type(IBM_type), intent(in) :: IBM - integer, intent(in) :: STLNum, integralType, iter - !-local-variables------------------------------------------------- - real(kind=RP), allocatable :: x(:), y(:), z(:), scalar(:), & - local_sum(:), global_sum(:) - integer :: i, j, index, NumOfObjs - character(len=LINE_LENGTH) :: FileName, FinalName -#ifdef _HAS_MPI_ - integer :: ierr -#endif - NumOfObjs = NumOfVertices * IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs - - allocate( x(NumOfObjs), & - y(NumOfObjs), & - z(NumOfObjs), & - scalar(NumOfObjs) ) - - index = 0 - - do i = 1, IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs - associate( obj => IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i) ) - do j = 1, NumOfVertices - index = index + 1 - x(index) = obj% vertices(j)% coords(IX) - y(index) = obj% vertices(j)% coords(IY) - z(index) = obj% vertices(j)% coords(IZ) - scalar(index) = obj% vertices(j)% ScalarValue - end do - end associate - end do -#ifdef _HAS_MPI_ - if( MPI_Process% doMPIAction ) then - allocate(local_sum(NumOfObjs),global_sum(NumOfObjs)) - local_sum = scalar - call mpi_allreduce(local_sum, global_sum, NumOfObjs, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) - scalar = global_sum - deallocate(local_sum,global_sum) - end if -#endif - if( .not. MPI_Process% isRoot ) then - deallocate(x, y, z, scalar) - return - end if - - if( .not. MPI_Process% isRoot ) return - - select case(integralType) - case( MASS_FLOW ) - FileName = 'MASS_FLOW_' - write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' - call STLScalarTEC( x, y, z, scalar, STLNum, FinalName, 'MASS FLOW', '"x","y","z","MassFlow"' ) - case( FLOW_RATE ) - FileName = 'FLOW_RATE_FORCE_' - write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' - call STLScalarTEC( x, y, z, scalar, STLNum, FinalName, 'FLOW RATE', '"x","y","z","FlowRate"' ) - case( PRESSURE_DISTRIBUTION ) - FileName = 'PRESSURE_' - write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' - call STLScalarTEC( x, y, z, scalar, STLNum, FinalName, 'PRESSURE DISTRIBUTION', '"x","y","z","Pressure"' ) - end select - - deallocate(x, y, z, scalar) - - end subroutine GenerateScalarmonitorTECfile - - subroutine GenerateVectormonitorTECfile( IBM, STLNum, integralType, iter ) - use MPI_Process_Info - use TessellationTypes - use MPI_IBMUtilities - use IBMClass - implicit none - !-arguments--------------------------------------------------------- - type(IBM_type), intent(in) :: IBM - integer, intent(in) :: STLNum, integralType, iter - !-local-variables--------------------------------------------------- - real(kind=RP), allocatable :: x(:), y(:), z(:), vector_x(:), & - vector_y(:), vector_z(:), & - local_sum(:), global_sum(:) - character(len=LINE_LENGTH) :: FileName, FinalName - integer :: index, NumOfObjs, i, j -#ifdef _HAS_MPI_ - integer :: ierr -#endif - NumOfObjs = NumOfVertices * IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs - - allocate( x(NumOfObjs), & - y(NumOfObjs), & - z(NumOfObjs), & - vector_x(NumOfObjs), & - vector_y(NumOfObjs), & - vector_z(NumOfObjs) ) - - index = 0 - - do i = 1, IBM% stlSurfaceIntegrals(STLNum)% NumOfObjs - associate( obj => IBM% stlSurfaceIntegrals(STLNum)% ObjectsList(i) ) - do j = 1, NumOfVertices - index = index + 1 - x(index) = obj% vertices(j)% coords(IX) - y(index) = obj% vertices(j)% coords(IY) - z(index) = obj% vertices(j)% coords(IZ) - vector_x(index) = obj% vertices(j)% VectorValue(IX) - vector_y(index) = obj% vertices(j)% VectorValue(IY) - vector_z(index) = obj% vertices(j)% VectorValue(IZ) - end do - end associate - end do -#ifdef _HAS_MPI_ - if( MPI_Process% doMPIAction ) then - allocate(local_sum(NumOfObjs),global_sum(NumOfObjs)) - local_sum = vector_x - call mpi_allreduce(local_sum, global_sum, NumOfObjs, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) - vector_x = global_sum - local_sum = vector_y - call mpi_allreduce(local_sum, global_sum, NumOfObjs, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) - vector_y = global_sum - local_sum = vector_z - call mpi_allreduce(local_sum, global_sum, NumOfObjs, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD, ierr) - vector_z = global_sum - deallocate(local_sum,global_sum) - end if -#endif - if( .not. MPI_Process% isRoot ) then - deallocate(x, y, z, vector_x, vector_y, vector_z) - return - end if - - select case(integralType) - case( TOTAL_FORCE ) - FileName = 'TOTAL_FORCE_' - write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' - call STLvectorTEC( x, y, z, vector_x, vector_y, vector_z, STLNum, FinalName, 'TOTAL FORCE', '"x","y","z","Ftot_x","Ftot_y","Ftot_z"' ) - case( PRESSURE_FORCE ) - FileName = 'PRESSURE_FORCE_' - write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' - call STLvectorTEC( x, y, z, vector_x, vector_y, vector_z, STLNum, FinalName, 'PRESSURE FORCE', '"x","y","z","Fpres_x","Fpres_y","Fpres_z"' ) - case( VISCOUS_FORCE ) - FileName = 'VISCOUS_FORCE_' - write(FinalName,'(A,A,I10.10,A)') trim(FileName),trim(OBB(STLNum)% FileName)//'_',iter,'.tec' - call STLvectorTEC( x, y, z, vector_x, vector_y, vector_z, STLNum, FinalName, 'VISCOUS FORCE', '"x","y","z","Fvisc_x","Fvisc_y","Fvisc_z"' ) - end select - - deallocate(x, y, z, vector_x, vector_y, vector_z) - - end subroutine GenerateVectormonitorTECfile - - subroutine ProlongToFaces(mesh, zoneID) - implicit none - type(HexMesh), intent(inout) :: mesh - integer, intent(in) :: zoneID - - !-------- local variables ---------- - integer :: zonefID, fID, eID - - !$acc wait - - select case ( mesh %nodeType ) - case(1) !Gauss-Legendre - !$acc parallel loop gang present(mesh) - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - fID = mesh % zones(zoneID) % faces(zonefID) - eID = mesh % faces(fID) % elementIDs(1) - call HexElement_ProlongSolToFaces(mesh % elements(eID), NCONS, mesh % faces(fID), mesh % faces(fID) % elementSide(1)) - end do - !$acc end parallel loop - - case(2) - !$acc parallel loop gang present(mesh) - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - fID = mesh % zones(zoneID) % faces(zonefID) - eID = mesh % faces(fID) % elementIDs(1) - call HexElement_ProlongSolToFaces_GL(mesh % elements(eID), NCONS, mesh % faces(fID), mesh % faces(fID) % elementSide(1)) - end do - !$acc end parallel loop - end select - - if ( computeGradients ) then - select case ( mesh %nodeType ) - case(1) !Gauss - -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) -#else - !$omp do schedule(runtime) private(eID) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - fID = mesh % zones(zoneID) % faces(zonefID) - eID = mesh % faces(fID) % elementIDs(1) - call HexElement_ProlongGradientsToFaces(mesh % elements(eID), NGRAD, & - mesh % faces(fID), & - mesh % elements(eID) % storage % U_x, mesh % faces(fID) % elementSide(1), 1) - - call HexElement_ProlongGradientsToFaces(mesh % elements(eID), NGRAD, & - mesh % faces(fID), & - mesh % elements(eID) % storage % U_y, mesh % faces(fID) % elementSide(1), 2) - - call HexElement_ProlongGradientsToFaces(mesh % elements(eID), NGRAD, & - mesh % faces(fID), & - mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1), 3) - end do -#ifdef _OPENACC - !$acc end parallel loop -#else - !$omp end do -#endif - - case(2) !Gauss-Lobatto - -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) private(fID) -#else - !$omp do schedule(runtime) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - fID = mesh % zones(zoneID) % faces(zonefID) - eID = mesh % faces(fID) % elementIDs(1) - call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & - mesh % faces(fID), & - mesh % elements(eID) % storage % U_x, mesh % faces(fID) % elementSide(1),1) - end do -#ifdef _OPENACC - !$acc end parallel loop -#else - !$omp end do -#endif - -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) private(fID) -#else - !$omp do schedule(runtime) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - fID = mesh % zones(zoneID) % faces(zonefID) - eID = mesh % faces(fID) % elementIDs(1) - call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & - mesh % faces(fID), & - mesh % elements(eID) % storage % U_y, mesh % faces(fID) % elementSide(1),2) - end do -#ifdef _OPENACC - !$acc end parallel loop -#else - !$omp end do -#endif - -#ifdef _OPENACC - !$acc parallel loop gang present(mesh) private(fID) -#else - !$omp do schedule(runtime) -#endif - do zonefID = 1, mesh % zones(zoneID) % no_of_faces - fID = mesh % zones(zoneID) % faces(zonefID) - eID = mesh % faces(fID) % elementIDs(1) - call HexElement_ProlongGradientsToFaces_GL(mesh % elements(eID), NGRAD, & - mesh % faces(fID), & - mesh % elements(eID) % storage % U_z, mesh % faces(fID) % elementSide(1),3) - end do -#ifdef _OPENACC - !$acc end parallel loop -#else - !$omp end do -#endif - end select - end if -! -! Loop the zone to get faces and elements -! --------------------------------------- - !$acc wait - - end subroutine ProlongToFaces - -end module SurfaceIntegrals -#endif - - diff --git a/Solver/src/libs/mesh/SurfaceMonitor.f90 b/Solver/src/libs/mesh/SurfaceMonitor.f90 deleted file mode 100644 index 161dae0f9..000000000 --- a/Solver/src/libs/mesh/SurfaceMonitor.f90 +++ /dev/null @@ -1,560 +0,0 @@ -#include "Includes.h" -module SurfaceMonitorClass - use SMConstants - use HexMeshClass - use MonitorDefinitions - use PhysicsStorage - use MPI_Process_Info - use FluidData - use FileReadingUtilities, only: getRealArrayFromString - implicit none - - -#if defined(NAVIERSTOKES) || defined(INCNS) - private - public SurfaceMonitor_t - - -! -! ******************************** -! Surface monitor class definition -! ******************************** -! - type SurfaceMonitor_t - logical :: active - logical :: isDimensionless, IBM = .false. - integer :: ID - real(kind=RP) :: direction(NDIM) - integer :: marker - real(kind=RP), allocatable :: referenceSurface - real(kind=RP), allocatable :: values(:) - real(kind=RP) :: dynamicPressure - character(len=STR_LEN_MONITORS) :: monitorName - character(len=STR_LEN_MONITORS) :: fileName - character(len=STR_LEN_MONITORS) :: variable - contains - procedure :: Initialization => SurfaceMonitor_Initialization - procedure :: Update => SurfaceMonitor_Update - procedure :: WriteLabel => SurfaceMonitor_WriteLabel - procedure :: WriteValues => SurfaceMonitor_WriteValue - procedure :: WriteToFile => SurfaceMonitor_WriteToFile - procedure :: destruct => SurfaceMonitor_Destruct - procedure :: copy => SurfaceMonitor_Assign - generic :: assignment(=) => copy - end type SurfaceMonitor_t - - contains -! -!///////////////////////////////////////////////////////////////////////// -! -! SURFACE MONITOR PROCEDURES -! -------------------------- -!///////////////////////////////////////////////////////////////////////// -! - subroutine SurfaceMonitor_Initialization( self , mesh , ID, solution_file , FirstCall) -! -! ***************************************************************************** -! This subroutine initializes the surface monitor. The following -! data is obtained from the case file: -! -> Name: The monitor name (10 characters maximum) -! -> Marker: The surface marker in which the monitor will be computed. -! -> Variable: The variable to be monitorized. -! -> Reference surface (optional): Reference surface for lift/drag coefficients -! -> Direction (optional): Direction in which the forces are computed -! ***************************************************************************** -! - use ParamfileRegions - implicit none - class(SurfaceMonitor_t) :: self - class(HexMesh) :: mesh - integer :: ID - character(len=*) :: solution_file - logical, intent(in) :: FirstCall -! -! --------------- -! Local variables -! --------------- -! - character(len=STR_LEN_MONITORS) :: in_label - character(len=STR_LEN_MONITORS) :: fileName - character(len=STR_LEN_MONITORS) :: paramFile - character(len=STR_LEN_MONITORS) :: directionName - integer, allocatable :: marker - character(len=STR_LEN_MONITORS) :: markerName - integer :: pos, i, STLNum - integer :: fID - integer :: zoneID - real(kind=RP) :: directionValue(NDIM) -! -! Get monitor ID -! -------------- - self % ID = ID -! -! Search for the parameters in the case file -! ------------------------------------------ - write(in_label , '(A,I0)') "#define surface monitor " , self % ID - - call get_command_argument(1, paramFile) - call readValueInRegion ( trim ( paramFile ) , "name" , self % monitorName , in_label , "# end" ) - call readValueInRegion ( trim ( paramFile ) , "marker" , markerName , in_label , "# end" ) - call readValueInRegion ( trim ( paramFile ) , "variable" , self % variable , in_label , "# end" ) - call readValueInRegion ( trim ( paramFile ) , "reference surface" , self % referenceSurface , in_label , "# end" ) - call readValueInRegion ( trim ( paramFile ) , "direction" , directionName , in_label , "# end" ) -! -! Enable the monitor -! ------------------ - self % active = .true. - allocate ( self % values(BUFFER_SIZE) ) -! -! Get the surface marker -! ---------------------- - self % marker = -1 - if( mesh% IBM% active ) then - do STLNum = 1, mesh% IBM% NumOfSTL - if( mesh% IBM% Integral(STLNum)% compute ) cycle - if( trim(mesh% IBM% STLfilename(STLNum)) .eq. trim(markerName) ) then - if( .not. mesh% IBM% ComputeBandRegion ) then - write(*,'(A)') "Warning: for surface monitors with IBM, 'band region' must be set '.true.'" - error stop - end if - self% marker = STLNum - self% IBM = .true. - mesh% IBM% Integral(STLNum)% compute = .true. - mesh% IBM% Integral(STLNum)% ListComputed = .false. - if( MPI_Process% isRoot ) then - call mesh% IBM% stlSurfaceIntegrals(STLNum)% ReadTessellation( mesh% IBM% STLfilename(STLNum) ) - if( mesh% IBM% ClipAxis .ne. 0 ) call mesh% IBM% stlSurfaceIntegrals(STLNum)% Clip( mesh% IBM% minCOORDS, mesh% IBM% maxCOORDS, mesh% IBM% ClipAxis, .false. ) - call mesh% IBM% stlSurfaceIntegrals(STLNum)% SetIntegrationPoints() - call mesh% IBM% stlSurfaceIntegrals(STLNum)% SetIntegration( mesh% IBM% NumOfInterPoints ) - end if - call mesh% IBM% SetIntegration( STLNum ) - exit - end if - end do - else - do zoneID = 1, size(mesh % zones) - if ( trim(mesh % zones(zoneID) % name) .eq. trim(markerName) ) then - self % marker = zoneID - exit - end if - end do - end if - - if ( self % marker .eq. -1 ) then - self % active = .false. - write(*,'(A,I0)') "Warning: Marker not specified for surface monitor ", self % ID - write(*,'(A,I0,A)') " Surface monitor ", self % ID, " disabled." - end if -! -! Select the variable from the available list, and compute auxiliary variables if needed -! -------------------------------------------------------------------------------------- -! -! **************************************** - select case ( trim ( self % variable ) ) -! **************************************** -! - case ("mass-flow") - self % isDimensionless = .false. - - case ("flow") - self % isDimensionless = .false. - - case ("pressure-force") - self % isDimensionless = .false. - if ( len_trim(directionName) .eq. 0 ) then - print*, "Direction not specified for pressure-force in surface monitor " , self % ID , "." - error stop "error stopped" - - else - directionValue = getRealArrayFromString(directionName) - if ( size(directionValue) .ne. 3 ) then - print*, "Incorrect direction for monitor ", self % ID, "." - - else - self % direction = directionValue - - end if - end if - - case ("viscous-force") - self % isDimensionless = .false. - if ( len_trim(directionName) .eq. 0 ) then - print*, "Direction not specified for pressure-force in surface monitor " , self % ID , "." - error stop "error stopped" - - else - directionValue = getRealArrayFromString(directionName) - if ( size(directionValue) .ne. 3 ) then - print*, "Incorrect direction for monitor ", self % ID, "." - - else - self % direction = directionValue - - end if - end if - - case ("force") - self % isDimensionless = .false. - - if ( len_trim(directionName) .eq. 0 ) then - print*, "Direction not specified for pressure-force in surface monitor " , self % ID , "." - error stop "error stopped" - - else - directionValue = getRealArrayFromString(directionName) - if ( size(directionValue) .ne. 3 ) then - print*, "Incorrect direction for monitor ", self % ID, "." - - else - self % direction = directionValue - - end if - end if - - - case ("lift") - self % isDimensionless = .true. - - if ( .not. allocated ( self % referenceSurface ) ) then - print*, "Reference surface not specified for lift surface monitor " , self % ID , "." - error stop "error stopped" - end if - - if ( len_trim(directionName) .eq. 0 ) then - print*, "Direction not specified for lift in surface monitor " , self % ID , "." - print*, " ... Using [0,1,0] as default." - self % direction = [0._RP,1._RP,0._RP] - - else - directionValue = getRealArrayFromString(directionName) - if ( size(directionValue) .ne. 3 ) then - print*, "Incorrect direction for monitor ", self % ID, "." - - else - self % direction = directionValue - - end if - end if - - self % dynamicPressure = 0.5_RP * refValues % rho * POW2(refValues % V)* self % referenceSurface - - case ("drag") - self % isDimensionless = .true. - - if ( .not. allocated ( self % referenceSurface ) ) then - print*, "Reference surface not specified for drag surface monitor " , self % ID , "." - error stop "error stopped" - end if - - if ( len_trim(directionName) .eq. 0 ) then - print*, "Direction not specified for drag in surface monitor " , self % ID , "." - print*, " ... Using [1,0,0] as default." - self % direction = [1._RP,0._RP,0._RP] - - else - directionValue = getRealArrayFromString(directionName) - if ( size(directionValue) .ne. 3 ) then - print*, "Incorrect direction for monitor ", self % ID, "." - - else - self % direction = directionValue - - end if - end if - - self % dynamicPressure = 0.5_RP * refValues % rho * refValues % V * refValues % V * self % referenceSurface - - case ("pressure-average") - self % isDimensionless = .false. - - case default - - if ( len_trim (self % variable) .eq. 0 ) then - print*, "Variable was not specified for surface monitor " , self % ID , "." - else - print*, 'Variable "',trim(self % variable),'" surface monitor ', self % ID, ' not implemented yet.' - print*, "Options available are:" - print*, " * mass-flow" - print*, " * flow" - print*, " * pressure-force" - print*, " * viscous-force" - print*, " * force" - print*, " * lift" - print*, " * drag" - print*, " * pressure-average" - error stop "error stopped." - - end if -! -! ********** - end select -! ********** -! -! Prepare the file in which the monitor is exported -! ------------------------------------------------- - write( self % fileName , '(A,A,A,A)') trim(solution_file) , "." , trim(self % monitorName) , ".surface" -! -! Create file -! ----------- - if (FirstCall) then - open ( newunit = fID , file = trim(self % fileName) , status = "unknown" , action = "write" ) -! -! Write the file headers -! ---------------------- - write( fID , '(A20,A )') "Monitor name: ", trim(self % monitorName) - write( fID , '(A20,I0 )') "Surface marker: ", self % marker - write( fID , '(A20,A )') "Selected variable: " , trim(self % variable) - - if ( self % isDimensionless ) then - write(fID , '(A20,ES24.10)') "Dynamic pressure: " , self % dynamicPressure - end if - - write( fID , * ) - write( fID , '(A10,2X,A24,2X,A24)' ) "Iteration" , "Time" , trim(self % variable) - - close ( fID ) - end if - end subroutine SurfaceMonitor_Initialization - - subroutine SurfaceMonitor_Update ( self, mesh, bufferPosition, iter, autosave, dt ) -! -! ******************************************************************* -! This subroutine updates the monitor value computing it from -! the mesh. It is stored in the "bufferPosition" position of the -! buffer. -! ******************************************************************* -! - use SurfaceIntegrals - use IBMClass - implicit none - class ( SurfaceMonitor_t ) :: self - class ( HexMesh ) :: mesh - integer :: bufferPosition, iter, STLNum - real(kind=RP) :: F(NDIM) - real(kind=RP) :: dt - logical :: autosave - - select case ( trim ( self % variable ) ) - - case ("mass-flow") - if( self% IBM ) then - STLNum = self% marker - call ScalarDataReconstruction( mesh% IBM, mesh% elements, STLNum, MASS_FLOW, iter, autosave, dt ) - self % values(bufferPosition) = mesh% IBM% stlSurfaceIntegrals(STLNum)% ComputeScalarIntegral() - else - self % values(bufferPosition) = ScalarSurfaceIntegral(mesh, self % marker, MASS_FLOW, iter) - end if - case ("flow") - if( self% IBM ) then - STLNum = self% marker - call ScalarDataReconstruction( mesh% IBM, mesh% elements, STLNum, FLOW_RATE, iter, autosave, dt ) - self % values(bufferPosition) = mesh% IBM% stlSurfaceIntegrals(STLNum)% ComputeScalarIntegral() - else - self % values(bufferPosition) = ScalarSurfaceIntegral(mesh, self % marker, FLOW_RATE, iter) - end if - - case ("pressure-force") - if( self% IBM ) then - STLNum = self% marker - call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, PRESSURE_FORCE, iter, autosave, dt ) - F = mesh% IBM% stlSurfaceIntegrals(STLNum)% ComputeVectorIntegral() - else - F = VectorSurfaceIntegral(mesh, self % marker, PRESSURE_FORCE, iter) - end if - F = refValues % rho * POW2(refValues % V) * POW2(Lref) * F - self % values(bufferPosition) = dot_product(F, self % direction) - - case ("viscous-force") - if( self% IBM ) then - STLNum = self% marker - call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, VISCOUS_FORCE, iter, autosave, dt ) - F = mesh% IBM% stlSurfaceIntegrals(STLNum)% ComputeVectorIntegral() - else - F = VectorSurfaceIntegral(mesh, self % marker, VISCOUS_FORCE, iter) - end if - F = refValues % rho * POW2(refValues % V) * POW2(Lref) * F - self % values(bufferPosition) = dot_product(F, self % direction) - - case ("force") - if( self% IBM ) then - STLNum = self% marker - call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, TOTAL_FORCE, iter, autosave, dt ) - F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() - else - F = VectorSurfaceIntegral(mesh, self % marker, TOTAL_FORCE, iter) - end if - F = refValues % rho * POW2(refValues % V) * POW2(Lref) * F - self % values(bufferPosition) = dot_product(F, self % direction) - - case ("lift") - if( self% IBM ) then - STLNum = self% marker - call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, TOTAL_FORCE, iter, autosave, dt ) - F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() - else - F = VectorSurfaceIntegral(mesh, self % marker, TOTAL_FORCE, iter) - end if - F = 2.0_RP * POW2(Lref) * F / self % referenceSurface - self % values(bufferPosition) = dot_product(F, self % direction) - -#if defined (NAVIERSTOKES) - case ("drag") - if (flowIsNavierStokes) then - if( self% IBM ) then - STLNum = self% marker - call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, TOTAL_FORCE, iter, autosave, dt ) - F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() - else - F = VectorSurfaceIntegral(mesh, self % marker, TOTAL_FORCE, iter) - end if - else - if( self% IBM ) then - STLNum = self% marker - call VectorDataReconstruction( mesh% IBM, mesh% elements,STLNum, TOTAL_FORCE, iter, autosave, dt ) - F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() - else - F = VectorSurfaceIntegral(mesh, self % marker, PRESSURE_FORCE, iter) - end if - end if - F = 2.0_RP * POW2(Lref) * F / self % referenceSurface - self % values(bufferPosition) = dot_product(F, self % direction) -#endif -!TODO if true -#if defined (INCNS) - case ("drag") - if (.true.) then - if( self% IBM ) then - STLNum = self% marker - call VectorDataReconstruction( mesh% IBM, mesh% elements, STLNum, TOTAL_FORCE, iter, autosave, dt ) - F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() - else - F = VectorSurfaceIntegral(mesh, self % marker, TOTAL_FORCE, iter) - end if - else - if( self% IBM ) then - STLNum = self% marker - call VectorDataReconstruction( mesh% IBM, mesh% elements,STLNum, TOTAL_FORCE, iter, autosave, dt ) - F = mesh% IBM% stlSurfaceIntegrals(STLNum) % ComputeVectorIntegral() - else - F = VectorSurfaceIntegral(mesh, self % marker, PRESSURE_FORCE, iter) - end if - end if - F = 2.0_RP * POW2(Lref) * F / self % referenceSurface - self % values(bufferPosition) = dot_product(F, self % direction) -#endif - case ("pressure-average") - self % values(bufferPosition) = ScalarSurfaceIntegral(mesh, self % marker, PRESSURE_FORCE, iter) / ScalarSurfaceIntegral(mesh, self % marker, SURFACE, iter) - - end select - - - end subroutine SurfaceMonitor_Update - - subroutine SurfaceMonitor_WriteLabel ( self ) -! -! ************************************************************* -! This subroutine writes the label for the surface -! monitor, when invoked from the time integrator Display -! procedure. -! ************************************************************* -! - implicit none - class(SurfaceMonitor_t), intent(in) :: self - - write(STD_OUT , '(3X,A10)' , advance = "no") trim(self % monitorName(1 : MONITOR_LENGTH)) - - end subroutine SurfaceMonitor_WriteLabel - - subroutine SurfaceMonitor_WriteValue ( self , bufferLine ) -! -! ************************************************************* -! This subroutine writes the monitor value for the time -! integrator Display procedure. -! ************************************************************* -! - implicit none - class(SurfaceMonitor_t) :: self - integer :: bufferLine - - write(STD_OUT , '(1X,A,1X,ES10.3)' , advance = "no") "|" , self % values ( bufferLine ) - - end subroutine SurfaceMonitor_WriteValue - - subroutine SurfaceMonitor_WriteToFile ( self , iter , t , no_of_lines) -! -! ************************************************************* -! This subroutine writes the buffer to the file. -! ************************************************************* -! - implicit none - class(SurfaceMonitor_t) :: self - integer :: iter(:) - real(kind=RP) :: t(:) - integer :: no_of_lines -! -! --------------- -! Local variables -! --------------- -! - integer :: i - integer :: fID - - if ( MPI_Process % isRoot ) then - - open( newunit = fID , file = trim ( self % fileName ) , action = "write" , access = "append" , status = "old" ) - - do i = 1 , no_of_lines - write( fID , '(I10,2X,ES24.16,2X,ES24.16)' ) iter(i) , t(i) , self % values(i) - - end do - - close ( fID ) - end if - - if ( no_of_lines .ne. 0 ) self % values(1) = self % values(no_of_lines) - - end subroutine SurfaceMonitor_WriteToFile -! - elemental subroutine SurfaceMonitor_Destruct (self) - implicit none - class(SurfaceMonitor_t), intent(inout) :: self - - safedeallocate (self % values) - safedeallocate (self % referenceSurface) - end subroutine SurfaceMonitor_Destruct -! -!////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -! - elemental subroutine SurfaceMonitor_Assign(to, from) - implicit none - class(SurfaceMonitor_t), intent(inout) :: to - type(SurfaceMonitor_t), intent(in) :: from - - if ( from % active) then - to % active = from % active - to % isDimensionless = from % isDimensionless - to % ID = from % ID - to % direction = from % direction - to % marker = from % marker - - safedeallocate(to % referenceSurface) - if ( allocated(from % referenceSurface) ) then - allocate (to % referenceSurface) - to % referenceSurface = from % referenceSurface - end if - - safedeallocate(to % values) - allocate ( to % values( size(from % values) ) ) - to % values = from % values - - to % dynamicPressure = from % dynamicPressure - to % monitorName = from % monitorName - to % fileName = from % fileName - to % variable = from % variable - else - to % active = .FALSE. - end if - - end subroutine SurfaceMonitor_Assign -#endif -end module SurfaceMonitorClass diff --git a/Solver/src/libs/monitors/Makefile b/Solver/src/libs/monitors/Makefile index 2f2f10635..b38152fea 100644 --- a/Solver/src/libs/monitors/Makefile +++ b/Solver/src/libs/monitors/Makefile @@ -88,6 +88,40 @@ LIB = monitors all: lib$(LIB)_ns.a lib$(LIB)_nssa.a lib$(LIB)_ins.a lib$(LIB)_ch.a lib$(LIB)_mu.a lib$(LIB)_caa.a +# nvfortran GPU: strip -fopenmp from files with mixed OMP/ACC directives +# (Probe.f90 and Monitors.f90 keep -fopenmp via the generic rule below) +FFLAGS_NO_OMP := $(filter-out -fopenmp -mp,$(FFLAGS)) + +./build_ns/SurfaceIntegrals.o ./build_ns/VolumeIntegrals.o: ./build_ns/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -I./include_ns $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ns -c $< -o $@ $(MOD_FLAG) ./include_ns + +./build_nssa/SurfaceIntegrals.o ./build_nssa/VolumeIntegrals.o: ./build_nssa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DNAVIERSTOKES -DSPALARTALMARAS -I./include_nssa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_nssa -c $< -o $@ $(MOD_FLAG) ./include_nssa + +./build_ins/SurfaceIntegrals.o ./build_ins/VolumeIntegrals.o: ./build_ins/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DINCNS -I./include_ins $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ins -c $< -o $@ $(MOD_FLAG) ./include_ins + +./build_ch/SurfaceIntegrals.o ./build_ch/VolumeIntegrals.o: ./build_ch/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DCAHNHILLIARD -I./include_ch $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_ch -c $< -o $@ $(MOD_FLAG) ./include_ch + +./build_mu/SurfaceIntegrals.o ./build_mu/VolumeIntegrals.o: ./build_mu/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DMULTIPHASE -DCAHNHILLIARD -I./include_mu $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_mu -c $< -o $@ $(MOD_FLAG) ./include_mu + +./build_caa/SurfaceIntegrals.o ./build_caa/VolumeIntegrals.o: ./build_caa/%.o: %.f90 + @echo + @echo $< + $(FC) $(FFLAGS_NO_OMP) -DACOUSTIC -I./include_caa $(MACROS) $(EXTLIB_INC) $(INCLUDE) $(INCLUDEO) $(INCLUDE)_caa -c $< -o $@ $(MOD_FLAG) ./include_caa + ./build_ns/%.o: %.f90 @echo @echo $<