diff --git a/opm/simulators/flow/CpGridVanguard.hpp b/opm/simulators/flow/CpGridVanguard.hpp index 28da0736187..22beeaca183 100644 --- a/opm/simulators/flow/CpGridVanguard.hpp +++ b/opm/simulators/flow/CpGridVanguard.hpp @@ -316,6 +316,12 @@ class CpGridVanguard : public FlowBaseVanguard { return this->zoltanParams_; } + + double zoltanPhgEdgeSizeThreshold() const override + { + return this->zoltanPhgEdgeSizeThreshold_; + } + const std::string& metisParams() const override { return this->metisParams_; diff --git a/opm/simulators/flow/FlowGenericVanguard.cpp b/opm/simulators/flow/FlowGenericVanguard.cpp index 4762ee46c23..b0da78f6498 100644 --- a/opm/simulators/flow/FlowGenericVanguard.cpp +++ b/opm/simulators/flow/FlowGenericVanguard.cpp @@ -124,7 +124,7 @@ FlowGenericVanguard::FlowGenericVanguard(SimulationModelParams&& params) partitionMethod_ = Dune::PartitionMethod(Parameters::Get()); serialPartitioning_ = Parameters::Get(); zoltanParams_ = Parameters::Get(); - + zoltanPhgEdgeSizeThreshold_ = Parameters::Get(); metisParams_ = Parameters::Get(); externalPartitionFile_ = Parameters::Get(); @@ -468,21 +468,26 @@ void FlowGenericVanguard::registerParameters_() Parameters::Register ("Perform partitioning for parallel runs on a single process."); Parameters::Register> - ("Tolerable imbalance of the loadbalancing provided by Zoltan. DEPRECATED: Use --imbalance-tol instead"); + ("Tolerable imbalance of the loadbalancing provided by Zoltan. " + "DEPRECATED: Use --imbalance-tol instead."); Parameters::Register ("Configuration of Zoltan partitioner. " "Valid options are: graph, hypergraph or scotch. " "Alternatively, you can request a configuration to be read " - "from a JSON file by giving the filename here, ending with '.json.' " + "from a JSON file by giving the filename here, with extension '.json'. " "See https://sandialabs.github.io/Zoltan/ug_html/ug.html " "for available Zoltan options."); Parameters::Register> - ("Tolerable imbalance of the loadbalancing (default: 1.1)."); + ("Tolerable imbalance of the loadbalancing."); + Parameters::Register + ("Low-level threshold fraction in the range [0,1] controlling " + "which hypergraph edge to omit. Used if --zoltan-params=\"graph\" " + "or if --zoltan-params=\"hypergraph\"."); Parameters::Register ("Configuration of Metis partitioner. " "You can request a configuration to be read " - "from a JSON file by giving the filename here, ending with '.json.' " - "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf" + "from a JSON file by giving the filename here, with extension '.json'. " + "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf " "for available METIS options."); Parameters::Register ("Name of file from which to load an externally generated " @@ -493,6 +498,7 @@ void FlowGenericVanguard::registerParameters_() Parameters::Hide>(); Parameters::Hide(); + Parameters::Hide(); #endif // HAVE_MPI Parameters::Register diff --git a/opm/simulators/flow/FlowGenericVanguard.hpp b/opm/simulators/flow/FlowGenericVanguard.hpp index 96012d35329..b7a462c462f 100644 --- a/opm/simulators/flow/FlowGenericVanguard.hpp +++ b/opm/simulators/flow/FlowGenericVanguard.hpp @@ -84,6 +84,8 @@ struct SerialPartitioning{ static constexpr bool value = false; }; template struct ZoltanImbalanceTol { static constexpr Scalar value = 1.1; }; +struct ZoltanPhgEdgeSizeThreshold { static constexpr auto value = 0.35; }; + struct ZoltanParams { static constexpr auto value = "graph"; }; } // namespace Opm::Parameters @@ -380,6 +382,7 @@ class FlowGenericVanguard { bool zoltanImbalanceTolSet_; double zoltanImbalanceTol_; + double zoltanPhgEdgeSizeThreshold_; std::string zoltanParams_; std::string metisParams_; diff --git a/opm/simulators/flow/GenericCpGridVanguard.cpp b/opm/simulators/flow/GenericCpGridVanguard.cpp index 233d7531f7d..03bf389aeb0 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.cpp +++ b/opm/simulators/flow/GenericCpGridVanguard.cpp @@ -158,11 +158,20 @@ doLoadBalance_(const Dune::EdgeWeightMethod edgeWeightsMethod, const int numJacobiBlocks, const bool enableEclOutput) { - if ((partitionMethod == Dune::PartitionMethod::zoltan - || partitionMethod == Dune::PartitionMethod::zoltanGoG) && !this->zoltanParams().empty()) - this->grid_->setPartitioningParams(setupZoltanParams(this->zoltanParams())); - if (partitionMethod == Dune::PartitionMethod::metis && !this->metisParams().empty()) - this->grid_->setPartitioningParams(setupMetisParams(this->metisParams())); + if (((partitionMethod == Dune::PartitionMethod::zoltan) || + (partitionMethod == Dune::PartitionMethod::zoltanGoG)) && + !this->zoltanParams().empty()) + { + this->grid_->setPartitioningParams + (setupZoltanParams(this->zoltanParams(), + this->zoltanPhgEdgeSizeThreshold())); + } + else if ((partitionMethod == Dune::PartitionMethod::metis) && + !this->metisParams().empty()) + { + this->grid_->setPartitioningParams + (setupMetisParams(this->metisParams())); + } const auto mpiSize = this->grid_->comm().size(); diff --git a/opm/simulators/flow/GenericCpGridVanguard.hpp b/opm/simulators/flow/GenericCpGridVanguard.hpp index 62c70f9f990..b9042bb9ab8 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.hpp +++ b/opm/simulators/flow/GenericCpGridVanguard.hpp @@ -209,6 +209,7 @@ class GenericCpGridVanguard { protected: virtual const std::string& zoltanParams() const = 0; + virtual double zoltanPhgEdgeSizeThreshold() const = 0; virtual const std::string& metisParams() const = 0; #endif // HAVE_MPI diff --git a/opm/simulators/utils/SetupPartitioningParams.cpp b/opm/simulators/utils/SetupPartitioningParams.cpp index febc176ae2c..31c17c18c2a 100644 --- a/opm/simulators/utils/SetupPartitioningParams.cpp +++ b/opm/simulators/utils/SetupPartitioningParams.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -99,6 +100,18 @@ namespace { namespace { + std::map + applyEdgeSizeThreshold(std::map&& params, + const std::optional& edgeSizeThreshold) + { + if (edgeSizeThreshold.has_value()) { + params.emplace("PHG_EDGE_SIZE_THRESHOLD", + fmt::format("{}", *edgeSizeThreshold)); + } + + return params; + } + std::map zoltanGraphParameters(std::string_view graphPackage) { @@ -108,9 +121,10 @@ namespace { }; } - auto zoltanGraphParameters() + auto zoltanGraphParameters(const std::optional& edgeSizeThreshold) { - return zoltanGraphParameters("PHG"); + return applyEdgeSizeThreshold(zoltanGraphParameters("PHG"), + edgeSizeThreshold); } auto zoltanScotchParameters() @@ -119,11 +133,12 @@ namespace { } std::map - zoltanHyperGraphParameters() + zoltanHyperGraphParameters(const std::optional& edgeSizeThreshold) { - return { + return applyEdgeSizeThreshold({ { "LB_METHOD", "HYPERGRAPH" }, - }; + { "HYPERGRAPH_PACKAGE", "PHG" }, + }, edgeSizeThreshold); } } // Anonymous namespace @@ -131,14 +146,15 @@ namespace { // =========================================================================== std::map -Opm::setupZoltanParams(const std::string& conf) +Opm::setupZoltanParams(const std::string& conf, + const std::optional& edgeSizeThreshold) { if (conf == "graph") { - return zoltanGraphParameters(); + return zoltanGraphParameters(edgeSizeThreshold); } else if (conf == "hypergraph") { - return zoltanHyperGraphParameters(); + return zoltanHyperGraphParameters(edgeSizeThreshold); } else if (conf == "scotch") { diff --git a/opm/simulators/utils/SetupPartitioningParams.hpp b/opm/simulators/utils/SetupPartitioningParams.hpp index b53be695f99..da6ff5f2531 100644 --- a/opm/simulators/utils/SetupPartitioningParams.hpp +++ b/opm/simulators/utils/SetupPartitioningParams.hpp @@ -21,12 +21,47 @@ #define OPM_SETUP_PARTITIONING_PARAMS_HPP #include +#include #include namespace Opm { -std::map setupZoltanParams(const std::string& conf); -std::map setupMetisParams(const std::string& conf); +/// Form collection of Zoltan partitioning parameters from named configuration +/// +/// \param[in] conf Named Zoltan configuration. Must either be the name of a +/// JSON configuration file with the filename extension ".json", or one of +/// the known configuration names +/// +/// -* graph Generates configuration parameters for the "GRAPH" +/// load-balancing method, using the "PHG" graph package. +/// +/// -* hypergraph Generates configuration parameters for the "HYPERGRAPH" +/// load-balancing method. +/// +/// -* scotch Generates configuration parameters for the "GRAPH" +/// load-balancing method, using the "Scotch" graph package. +/// +/// \param[in] edgeSizeThreshold Low-level Zoltan partitioning control +/// parameter for when to omit a hyperedge in a hypergraph. Fraction in the +/// range [0,1] representing a threshold above which to omit discard +/// hyperedges. Used for conf="graph" and conf="hypergraph". Nullopt to +/// use the built-in default value. +/// +/// \return Collection of Zoltan partitioning parameters. +std::map +setupZoltanParams(const std::string& conf, + const std::optional& edgeSizeThreshold = {}); + +/// Form collection of METIS partitioning parameters from named configuration +/// +/// \param[in] conf Named METIS configuration. Must either be the name of a +/// JSON configuration file with the filename extension ".json", or the +/// known configuration name "default" which uses the built-in default +/// partitioning parameters. +/// +/// \return Collection of METIS partitioning parameters. +std::map +setupMetisParams(const std::string& conf); } // namespace Opm