Enable Zoltan-Based In-Process Subdomain Partitioning

CMakeLists_files.cmake

@@ -50,6 +50,7 @@ list (APPEND MAIN_SOURCE_FILES
   opm/simulators/flow/NonlinearSolverEbos.cpp
   opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.cpp
   opm/simulators/flow/ValidationFunctions.cpp
+  opm/simulators/flow/aspinPartition.cpp
   opm/simulators/linalg/bda/WellContributions.cpp
   opm/simulators/linalg/bda/MultisegmentWellContribution.cpp
   opm/simulators/linalg/ExtractParallelGridInformationToISTL.cpp
@@ -226,6 +227,7 @@ endif()
 # find tests -name '*.cpp' -a ! -wholename '*/not-unit/*' -printf '\t%p\n' | sort
 list (APPEND TEST_SOURCE_FILES
   tests/test_ALQState.cpp
+  tests/test_aspinPartition.cpp
   tests/test_blackoil_amg.cpp
   tests/test_convergenceoutputconfiguration.cpp
   tests/test_convergencereport.cpp
@@ -343,6 +345,7 @@ list (APPEND TEST_DATA_FILES
   tests/include/summary.inc
   tests/include/test1_20x30x10.grdecl
   tests/include/well_vfp.ecl
+  tests/partition.txt
   )
 
 
@@ -362,6 +365,8 @@ list (APPEND PUBLIC_HEADER_FILES
   opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp
   opm/simulators/flow/KeywordValidation.hpp
   opm/simulators/flow/ValidationFunctions.hpp
+  opm/simulators/flow/aspinPartition.hpp
+  opm/simulators/flow/SubDomain.hpp
   opm/core/props/BlackoilPhases.hpp
   opm/core/props/phaseUsageFromDeck.hpp
   opm/core/props/satfunc/RelpermDiagnostics.hpp
@@ -421,6 +426,7 @@ list (APPEND PUBLIC_HEADER_FILES
   opm/simulators/linalg/SmallDenseMatrixUtils.hpp
   opm/simulators/linalg/WellOperators.hpp
   opm/simulators/linalg/WriteSystemMatrixHelper.hpp
+  opm/simulators/linalg/extractMatrix.hpp
   opm/simulators/linalg/findOverlapRowsAndColumns.hpp
   opm/simulators/linalg/getQuasiImpesWeights.hpp
   opm/simulators/linalg/setupPropertyTree.hpp

ebos/eclfluxmodule.hh

@@ -389,14 +389,16 @@ public:
         // of threshold pressure is a quite big hack that only makes sense for ECL
         // datasets. (and even there, its physical justification is quite
         // questionable IMO.)
-        if (std::abs(Toolbox::value(pressureDifference)) > thpres) {
-            if (pressureDifference < 0.0)
-                pressureDifference += thpres;
-            else
-                pressureDifference -= thpres;
-        }
-        else {
-            pressureDifference = 0.0;
+        if (thpres > 0.0) {
+            if (std::abs(Toolbox::value(pressureDifference)) > thpres) {
+                if (pressureDifference < 0.0)
+                    pressureDifference += thpres;
+                else
+                    pressureDifference -= thpres;
+            }
+            else {
+                pressureDifference = 0.0;
+            }
         }
     }
 

ebos/eclgenericcpgridvanguard.cc

@@ -49,7 +49,7 @@
 #include <dune/fem/gridpart/adaptiveleafgridpart.hh>
 #include <dune/fem/gridpart/common/gridpart2gridview.hh>
 #include <ebos/femcpgridcompat.hh>
-#endif //HAVE_DUNE_FEM
+#endif // HAVE_DUNE_FEM
 
 #include <cassert>
 #include <numeric>
@@ -84,6 +84,20 @@ void EclGenericCpGridVanguard<ElementMapper,GridView,Scalar>::releaseEquilGrid()
 }
 
 #if HAVE_MPI
+
+template <class ElementMapper, class GridView, class Scalar>
+std::vector<int>
+EclGenericCpGridVanguard<ElementMapper, GridView, Scalar>::
+partitionCells(const int                                     numDomains,
+               const double                                  imbalanceTolerance,
+               const std::vector<Dune::cpgrid::OpmWellType>& wells) const
+{
+    const auto* faceTrans = static_cast<const double*>(nullptr);
+
+    return this->grid_->
+        zoltanPartitionWithoutScatter(&wells, faceTrans, numDomains, imbalanceTolerance);
+}
+
 template<class ElementMapper, class GridView, class Scalar>
 void EclGenericCpGridVanguard<ElementMapper, GridView, Scalar>::
 doLoadBalance_(const Dune::EdgeWeightMethod            edgeWeightsMethod,

ebos/eclgenericcpgridvanguard.hh

@@ -28,9 +28,12 @@
 #define EWOMS_ECL_CP_GRID_GENERIC_VANGUARD_HH
 
 #include <ebos/eclgenericvanguard.hh>
+
 #include <opm/grid/CpGrid.hpp>
+#include <opm/grid/utility/OpmWellType.hpp>
 
 #include <functional>
+#include <memory>
 #include <optional>
 #include <vector>
 
@@ -115,6 +118,13 @@ public:
         return this->cell_part_;
     }
 
+#if HAVE_MPI
+    std::vector<int>
+    partitionCells(const int                                     numDomains,
+                   const double                                  imbalanceTolerance,
+                   const std::vector<Dune::cpgrid::OpmWellType>& wells) const;
+#endif
+
 protected:
     /*!
      * \brief Distribute the simulation grid over multiple processes

ebos/eclproblem.hh

@@ -1698,6 +1698,9 @@ public:
      */
     bool recycleFirstIterationStorage() const
     {
+        if (this->simulator().timeStepIndex() == 0)
+             return true;
+
         int episodeIdx = this->episodeIndex();
         return !this->drsdtActive_(episodeIdx) &&
                !this->drvdtActive_(episodeIdx) &&