From f16080eb210c42a41abcb7b5fea4a7484ec1424e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A5rd=20Skaflestad?= Date: Tue, 25 Feb 2025 16:52:49 +0100 Subject: [PATCH 1/2] Revise Parameter Descriptions for Partitioning Parameters Slight rewording and splitting long lines for the most part. --- opm/simulators/flow/FlowGenericVanguard.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/opm/simulators/flow/FlowGenericVanguard.cpp b/opm/simulators/flow/FlowGenericVanguard.cpp index 4762ee46c23..2190fc5f5a4 100644 --- a/opm/simulators/flow/FlowGenericVanguard.cpp +++ b/opm/simulators/flow/FlowGenericVanguard.cpp @@ -468,21 +468,22 @@ void FlowGenericVanguard::registerParameters_() Parameters::Register ("Perform partitioning for parallel runs on a single process."); Parameters::Register> - ("Tolerable imbalance of the loadbalancing provided by Zoltan. DEPRECATED: Use --imbalance-tol instead"); + ("Tolerable imbalance of the loadbalancing provided by Zoltan. " + "DEPRECATED: Use --imbalance-tol instead."); Parameters::Register ("Configuration of Zoltan partitioner. " "Valid options are: graph, hypergraph or scotch. " "Alternatively, you can request a configuration to be read " - "from a JSON file by giving the filename here, ending with '.json.' " + "from a JSON file by giving the filename here, with extension '.json'. " "See https://sandialabs.github.io/Zoltan/ug_html/ug.html " "for available Zoltan options."); Parameters::Register> - ("Tolerable imbalance of the loadbalancing (default: 1.1)."); + ("Tolerable imbalance of the loadbalancing."); Parameters::Register ("Configuration of Metis partitioner. " "You can request a configuration to be read " - "from a JSON file by giving the filename here, ending with '.json.' " - "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf" + "from a JSON file by giving the filename here, with extension '.json'. " + "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf " "for available METIS options."); Parameters::Register ("Name of file from which to load an externally generated " From fed4a3bb1c5528d20fa8e32b059d0b1de39f7633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A5rd=20Skaflestad?= Date: Tue, 25 Feb 2025 16:56:24 +0100 Subject: [PATCH 2/2] Make Zoltan's Hyperedge Size Threshold Runtime Controllable This commit introduces a new hidden runtime parameter, ZoltanPhgEdgeSizeThreshold (--zoltan-phg-edge-size-threshold, double, default value = 0.35) which maps to Zoltan's low-level control parameter PHG_EDGE_SIZE_THRESHOLD that controls which hypergraph edges to omit/discard. We add a new parameter to 'setupZoltanParameters()' and thread the command line parameter through as an argument to this function. This is to enable runtime experimentation with this aspect of the partitioning algorithm, but end-users should typically not alter the default value without good reason. --- opm/simulators/flow/CpGridVanguard.hpp | 6 +++ opm/simulators/flow/FlowGenericVanguard.cpp | 7 +++- opm/simulators/flow/FlowGenericVanguard.hpp | 3 ++ opm/simulators/flow/GenericCpGridVanguard.cpp | 19 ++++++--- opm/simulators/flow/GenericCpGridVanguard.hpp | 1 + .../utils/SetupPartitioningParams.cpp | 32 +++++++++++---- .../utils/SetupPartitioningParams.hpp | 39 ++++++++++++++++++- 7 files changed, 91 insertions(+), 16 deletions(-) diff --git a/opm/simulators/flow/CpGridVanguard.hpp b/opm/simulators/flow/CpGridVanguard.hpp index 28da0736187..22beeaca183 100644 --- a/opm/simulators/flow/CpGridVanguard.hpp +++ b/opm/simulators/flow/CpGridVanguard.hpp @@ -316,6 +316,12 @@ class CpGridVanguard : public FlowBaseVanguard { return this->zoltanParams_; } + + double zoltanPhgEdgeSizeThreshold() const override + { + return this->zoltanPhgEdgeSizeThreshold_; + } + const std::string& metisParams() const override { return this->metisParams_; diff --git a/opm/simulators/flow/FlowGenericVanguard.cpp b/opm/simulators/flow/FlowGenericVanguard.cpp index 2190fc5f5a4..b0da78f6498 100644 --- a/opm/simulators/flow/FlowGenericVanguard.cpp +++ b/opm/simulators/flow/FlowGenericVanguard.cpp @@ -124,7 +124,7 @@ FlowGenericVanguard::FlowGenericVanguard(SimulationModelParams&& params) partitionMethod_ = Dune::PartitionMethod(Parameters::Get()); serialPartitioning_ = Parameters::Get(); zoltanParams_ = Parameters::Get(); - + zoltanPhgEdgeSizeThreshold_ = Parameters::Get(); metisParams_ = Parameters::Get(); externalPartitionFile_ = Parameters::Get(); @@ -479,6 +479,10 @@ void FlowGenericVanguard::registerParameters_() "for available Zoltan options."); Parameters::Register> ("Tolerable imbalance of the loadbalancing."); + Parameters::Register + ("Low-level threshold fraction in the range [0,1] controlling " + "which hypergraph edge to omit. Used if --zoltan-params=\"graph\" " + "or if --zoltan-params=\"hypergraph\"."); Parameters::Register ("Configuration of Metis partitioner. " "You can request a configuration to be read " @@ -494,6 +498,7 @@ void FlowGenericVanguard::registerParameters_() Parameters::Hide>(); Parameters::Hide(); + Parameters::Hide(); #endif // HAVE_MPI Parameters::Register diff --git a/opm/simulators/flow/FlowGenericVanguard.hpp b/opm/simulators/flow/FlowGenericVanguard.hpp index 96012d35329..b7a462c462f 100644 --- a/opm/simulators/flow/FlowGenericVanguard.hpp +++ b/opm/simulators/flow/FlowGenericVanguard.hpp @@ -84,6 +84,8 @@ struct SerialPartitioning{ static constexpr bool value = false; }; template struct ZoltanImbalanceTol { static constexpr Scalar value = 1.1; }; +struct ZoltanPhgEdgeSizeThreshold { static constexpr auto value = 0.35; }; + struct ZoltanParams { static constexpr auto value = "graph"; }; } // namespace Opm::Parameters @@ -380,6 +382,7 @@ class FlowGenericVanguard { bool zoltanImbalanceTolSet_; double zoltanImbalanceTol_; + double zoltanPhgEdgeSizeThreshold_; std::string zoltanParams_; std::string metisParams_; diff --git a/opm/simulators/flow/GenericCpGridVanguard.cpp b/opm/simulators/flow/GenericCpGridVanguard.cpp index 233d7531f7d..03bf389aeb0 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.cpp +++ b/opm/simulators/flow/GenericCpGridVanguard.cpp @@ -158,11 +158,20 @@ doLoadBalance_(const Dune::EdgeWeightMethod edgeWeightsMethod, const int numJacobiBlocks, const bool enableEclOutput) { - if ((partitionMethod == Dune::PartitionMethod::zoltan - || partitionMethod == Dune::PartitionMethod::zoltanGoG) && !this->zoltanParams().empty()) - this->grid_->setPartitioningParams(setupZoltanParams(this->zoltanParams())); - if (partitionMethod == Dune::PartitionMethod::metis && !this->metisParams().empty()) - this->grid_->setPartitioningParams(setupMetisParams(this->metisParams())); + if (((partitionMethod == Dune::PartitionMethod::zoltan) || + (partitionMethod == Dune::PartitionMethod::zoltanGoG)) && + !this->zoltanParams().empty()) + { + this->grid_->setPartitioningParams + (setupZoltanParams(this->zoltanParams(), + this->zoltanPhgEdgeSizeThreshold())); + } + else if ((partitionMethod == Dune::PartitionMethod::metis) && + !this->metisParams().empty()) + { + this->grid_->setPartitioningParams + (setupMetisParams(this->metisParams())); + } const auto mpiSize = this->grid_->comm().size(); diff --git a/opm/simulators/flow/GenericCpGridVanguard.hpp b/opm/simulators/flow/GenericCpGridVanguard.hpp index 62c70f9f990..b9042bb9ab8 100644 --- a/opm/simulators/flow/GenericCpGridVanguard.hpp +++ b/opm/simulators/flow/GenericCpGridVanguard.hpp @@ -209,6 +209,7 @@ class GenericCpGridVanguard { protected: virtual const std::string& zoltanParams() const = 0; + virtual double zoltanPhgEdgeSizeThreshold() const = 0; virtual const std::string& metisParams() const = 0; #endif // HAVE_MPI diff --git a/opm/simulators/utils/SetupPartitioningParams.cpp b/opm/simulators/utils/SetupPartitioningParams.cpp index febc176ae2c..31c17c18c2a 100644 --- a/opm/simulators/utils/SetupPartitioningParams.cpp +++ b/opm/simulators/utils/SetupPartitioningParams.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -99,6 +100,18 @@ namespace { namespace { + std::map + applyEdgeSizeThreshold(std::map&& params, + const std::optional& edgeSizeThreshold) + { + if (edgeSizeThreshold.has_value()) { + params.emplace("PHG_EDGE_SIZE_THRESHOLD", + fmt::format("{}", *edgeSizeThreshold)); + } + + return params; + } + std::map zoltanGraphParameters(std::string_view graphPackage) { @@ -108,9 +121,10 @@ namespace { }; } - auto zoltanGraphParameters() + auto zoltanGraphParameters(const std::optional& edgeSizeThreshold) { - return zoltanGraphParameters("PHG"); + return applyEdgeSizeThreshold(zoltanGraphParameters("PHG"), + edgeSizeThreshold); } auto zoltanScotchParameters() @@ -119,11 +133,12 @@ namespace { } std::map - zoltanHyperGraphParameters() + zoltanHyperGraphParameters(const std::optional& edgeSizeThreshold) { - return { + return applyEdgeSizeThreshold({ { "LB_METHOD", "HYPERGRAPH" }, - }; + { "HYPERGRAPH_PACKAGE", "PHG" }, + }, edgeSizeThreshold); } } // Anonymous namespace @@ -131,14 +146,15 @@ namespace { // =========================================================================== std::map -Opm::setupZoltanParams(const std::string& conf) +Opm::setupZoltanParams(const std::string& conf, + const std::optional& edgeSizeThreshold) { if (conf == "graph") { - return zoltanGraphParameters(); + return zoltanGraphParameters(edgeSizeThreshold); } else if (conf == "hypergraph") { - return zoltanHyperGraphParameters(); + return zoltanHyperGraphParameters(edgeSizeThreshold); } else if (conf == "scotch") { diff --git a/opm/simulators/utils/SetupPartitioningParams.hpp b/opm/simulators/utils/SetupPartitioningParams.hpp index b53be695f99..da6ff5f2531 100644 --- a/opm/simulators/utils/SetupPartitioningParams.hpp +++ b/opm/simulators/utils/SetupPartitioningParams.hpp @@ -21,12 +21,47 @@ #define OPM_SETUP_PARTITIONING_PARAMS_HPP #include +#include #include namespace Opm { -std::map setupZoltanParams(const std::string& conf); -std::map setupMetisParams(const std::string& conf); +/// Form collection of Zoltan partitioning parameters from named configuration +/// +/// \param[in] conf Named Zoltan configuration. Must either be the name of a +/// JSON configuration file with the filename extension ".json", or one of +/// the known configuration names +/// +/// -* graph Generates configuration parameters for the "GRAPH" +/// load-balancing method, using the "PHG" graph package. +/// +/// -* hypergraph Generates configuration parameters for the "HYPERGRAPH" +/// load-balancing method. +/// +/// -* scotch Generates configuration parameters for the "GRAPH" +/// load-balancing method, using the "Scotch" graph package. +/// +/// \param[in] edgeSizeThreshold Low-level Zoltan partitioning control +/// parameter for when to omit a hyperedge in a hypergraph. Fraction in the +/// range [0,1] representing a threshold above which to omit discard +/// hyperedges. Used for conf="graph" and conf="hypergraph". Nullopt to +/// use the built-in default value. +/// +/// \return Collection of Zoltan partitioning parameters. +std::map +setupZoltanParams(const std::string& conf, + const std::optional& edgeSizeThreshold = {}); + +/// Form collection of METIS partitioning parameters from named configuration +/// +/// \param[in] conf Named METIS configuration. Must either be the name of a +/// JSON configuration file with the filename extension ".json", or the +/// known configuration name "default" which uses the built-in default +/// partitioning parameters. +/// +/// \return Collection of METIS partitioning parameters. +std::map +setupMetisParams(const std::string& conf); } // namespace Opm