feat(dist): active set (#55)

* refactor: implement reified() for AbstractDistributedGraph, general cleanup * refactor: implement reified() for AbstractDistributedGraph, general cleanup * . * feat: enable active set strategy for distributed LP * fix: bad ghost graph init might lead to deadlock * feat: count skipped nodes in debug output * fix: deadlock * fix: crash due to missing gain cache init
KaHIP · Dec 26, 2024 · 6cabccd · 6cabccd
1 parent 52b9708
commit 6cabccd
Show file tree

Hide file tree

Showing 9 changed files with 162 additions and 32 deletions.
diff --git a/kaminpar-cli/dkaminpar_arguments.cc b/kaminpar-cli/dkaminpar_arguments.cc
@@ -13,7 +13,9 @@
 #include "kaminpar-dist/context_io.h"
 
 namespace kaminpar::dist {
+
 namespace {
+
 void create_chunks_options(CLI::Option_group *cli, const std::string &prefix, ChunksContext &ctx) {
   cli->add_option(
          prefix + "-total-chunks",
@@ -40,6 +42,7 @@ void create_chunks_options(CLI::Option_group *cli, const std::string &prefix, Ch
   )
       ->capture_default_str();
 }
+
 } // namespace
 
 void create_all_options(CLI::App *app, Context &ctx) {
@@ -514,6 +517,14 @@ CLI::Option_group *create_global_lp_coarsening_options(CLI::App *app, Context &c
   lp->add_flag("--c-glp-prevent-cyclic-moves", ctx.coarsening.global_lp.prevent_cyclic_moves);
   lp->add_flag("--c-glp-enforce-legacy-weight", ctx.coarsening.global_lp.enforce_legacy_weight);
 
+  lp->add_option("--c-glp-active-set", ctx.coarsening.global_lp.active_set_strategy)
+      ->transform(CLI::CheckedTransformer(get_active_set_strategies()).description(""))
+      ->description(R"(Determines the active set strategy:
+  - none:   Do not use an active set
+  - local:  Only for non-interface nodes
+  - global: Full active set strategy (requires ghost graph)")
+      ->capture_default_str();
+
   return lp;
 }
 
@@ -641,4 +652,5 @@ CLI::Option_group *create_mtkahypar_refinement_options(CLI::App *app, Context &c
 
   return mtkahypar;
 }
+
 } // namespace kaminpar::dist
diff --git a/kaminpar-cli/dkaminpar_arguments.h b/kaminpar-cli/dkaminpar_arguments.h
@@ -14,6 +14,7 @@
 #include "kaminpar-dist/context.h"
 
 namespace kaminpar::dist {
+
 void create_all_options(CLI::App *app, Context &ctx);
 
 CLI::Option_group *create_partitioning_options(CLI::App *app, Context &ctx);
@@ -45,4 +46,5 @@ CLI::Option_group *create_local_lp_coarsening_options(CLI::App *app, Context &ct
 CLI::Option_group *create_global_lp_coarsening_options(CLI::App *app, Context &ctx);
 
 CLI::Option_group *create_hem_coarsening_options(CLI::App *app, Context &ctx);
+
 } // namespace kaminpar::dist
diff --git a/kaminpar-dist/coarsening/clustering/lp/global_lp_clusterer.cc b/kaminpar-dist/coarsening/clustering/lp/global_lp_clusterer.cc
@@ -21,16 +21,21 @@ namespace kaminpar::dist {
 
 namespace {
 
+SET_DEBUG(true);
+
+}
+
+namespace {
+
 struct GlobalLPClusteringConfig : public LabelPropagationConfig {
   using RatingMap = ::kaminpar::RatingMap<EdgeWeight, GlobalNodeID, rm_backyard::Sparsehash>;
 
   using ClusterID = GlobalNodeID;
   using ClusterWeight = GlobalNodeWeight;
 
-  static constexpr bool kTrackClusterCount = false;         // NOLINT
-  static constexpr bool kUseTwoHopClustering = false;       // NOLINT
-  static constexpr bool kUseActiveSetStrategy = false;      // NOLINT
-  static constexpr bool kUseLocalActiveSetStrategy = false; // NOLINT
+  static constexpr bool kTrackClusterCount = false;   // NOLINT
+  static constexpr bool kUseTwoHopClustering = false; // NOLINT
+  static constexpr bool kUseActiveSetStrategy = true; // NOLINT
 };
 
 } // namespace
@@ -51,8 +56,6 @@ class GlobalLPClusteringImpl final : public ChunkRandomdLabelPropagation<
                                          GlobalLPClusteringConfig,
                                          Graph>,
                                      public NonatomicClusterVectorRef<NodeID, GlobalNodeID> {
-  SET_DEBUG(false);
-
   using Base =
       ChunkRandomdLabelPropagation<GlobalLPClusteringImpl<Graph>, GlobalLPClusteringConfig, Graph>;
   using ClusterBase = NonatomicClusterVectorRef<NodeID, GlobalNodeID>;
@@ -70,6 +73,13 @@ class GlobalLPClusteringImpl final : public ChunkRandomdLabelPropagation<
     set_max_num_iterations(_c_ctx.global_lp.num_iterations);
     Base::set_max_degree(_c_ctx.global_lp.active_high_degree_threshold);
     Base::set_max_num_neighbors(_c_ctx.global_lp.max_num_neighbors);
+
+    if (_c_ctx.global_lp.active_set_strategy == ActiveSetStrategy::LOCAL) {
+      Base::enable_local_active_set();
+    }
+    if (_c_ctx.global_lp.active_set_strategy == ActiveSetStrategy::GLOBAL) {
+      Base::enable_active_set();
+    }
   }
 
   void setup(GlobalLPClusteringMemoryContext &memory_context) {
@@ -107,6 +117,11 @@ class GlobalLPClusteringImpl final : public ChunkRandomdLabelPropagation<
   void initialize(const Graph &graph) {
     _graph = &graph;
 
+    if (_c_ctx.global_lp.active_set_strategy == ActiveSetStrategy::GLOBAL) {
+      // Dummy access to initialize the ghost graph
+      _graph->ghost_graph();
+    }
+
     START_TIMER("Initialize high-degree node info");
     if (_passive_high_degree_threshold > 0) {
       SCOPED_HEAP_PROFILER("Initialize high-degree node info");
@@ -153,6 +168,23 @@ class GlobalLPClusteringImpl final : public ChunkRandomdLabelPropagation<
         const auto [from, to] = math::compute_local_range<NodeID>(_graph->n(), num_chunks, chunk);
         global_num_moved_nodes += process_chunk(from, to);
       }
+
+      if constexpr (kDebug) {
+        GlobalNodeID global_num_skipped_nodes = Base::_num_skipped_nodes_ets.combine(std::plus{});
+        MPI_Allreduce(
+            MPI_IN_PLACE,
+            &global_num_skipped_nodes,
+            1,
+            mpi::type::get<GlobalNodeID>(),
+            MPI_SUM,
+            _graph->communicator()
+        );
+
+        DBG0 << "Iteration " << iteration << ": " << global_num_moved_nodes << " nodes moved, "
+             << global_num_skipped_nodes << " nodes skipped";
+        Base::_num_skipped_nodes_ets.clear();
+      }
+
       if (global_num_moved_nodes == 0) {
         break;
       }
@@ -350,7 +382,7 @@ class GlobalLPClusteringImpl final : public ChunkRandomdLabelPropagation<
     TIMER_BARRIER(_graph->communicator());
 
     const NodeID local_num_moved_nodes = TIMED_SCOPE("Local work") {
-      return Base::perform_iteration(from, to);
+      return Base::perform_iteration(from, to, kDebug);
     };
 
     const GlobalNodeID global_num_moved_nodes =
@@ -598,6 +630,8 @@ class GlobalLPClusteringImpl final : public ChunkRandomdLabelPropagation<
                   weight_delta_handle.find(new_gcluster + 1) == weight_delta_handle.end()) {
                 change_cluster_weight(new_gcluster, weight, false);
               }
+
+              Base::activate_neighbors_of_ghost_node(lnode);
             }
           });
         }

diff --git a/kaminpar-dist/context_io.cc b/kaminpar-dist/context_io.cc
@@ -303,6 +303,27 @@ std::unordered_map<std::string, GainCacheStrategy> get_gain_cache_strategies() {
   };
 }
 
+std::ostream &operator<<(std::ostream &out, ActiveSetStrategy strategy) {
+  switch (strategy) {
+  case ActiveSetStrategy::NONE:
+    return out << "none";
+  case ActiveSetStrategy::LOCAL:
+    return out << "local";
+  case ActiveSetStrategy::GLOBAL:
+    return out << "global";
+  }
+
+  return out << "<invalid>";
+}
+
+std::unordered_map<std::string, ActiveSetStrategy> get_active_set_strategies() {
+  return {
+      {"none", ActiveSetStrategy::NONE},
+      {"local", ActiveSetStrategy::LOCAL},
+      {"global", ActiveSetStrategy::GLOBAL},
+  };
+}
+
 void print(const Context &ctx, const bool root, std::ostream &out, MPI_Comm comm) {
   if (root) {
     out << "Seed:                         " << Random::get_seed() << "\n";
@@ -494,9 +515,7 @@ void print(const CoarseningContext &ctx, const ParallelContext &parallel, std::o
           << " (passive), " << ctx.global_lp.active_high_degree_threshold << " (active)\n";
       out << "  Max degree:                 " << ctx.global_lp.max_num_neighbors << "\n";
       print(ctx.global_lp.chunks, parallel, out);
-      out << "  Active set:                 "
-          << (ctx.global_clustering_algorithm == ClusteringAlgorithm::GLOBAL_LP ? "no" : "yes")
-          << "\n";
+      out << "  Active set:                 " << ctx.global_lp.active_set_strategy << "\n";
       out << "  Cluster weights:            "
           << (ctx.global_lp.sync_cluster_weights ? "sync" : "no-sync") << "+"
           << (ctx.global_lp.enforce_cluster_weights ? "enforce" : "no-enforce") << " "

diff --git a/kaminpar-dist/context_io.h b/kaminpar-dist/context_io.h
@@ -24,6 +24,7 @@ std::ostream &operator<<(std::ostream &out, GraphOrdering ordering);
 std::ostream &operator<<(std::ostream &out, ClusterSizeStrategy strategy);
 std::ostream &operator<<(std::ostream &out, ClusterStrategy strategy);
 std::ostream &operator<<(std::ostream &out, GainCacheStrategy strategy);
+std::ostream &operator<<(std::ostream &out, ActiveSetStrategy strategy);
 
 std::unordered_map<std::string, PartitioningMode> get_partitioning_modes();
 std::unordered_map<std::string, ClusteringAlgorithm> get_clustering_algorithms();
@@ -37,6 +38,7 @@ std::unordered_map<std::string, GraphDistribution> get_graph_distributions();
 std::unordered_map<std::string, ClusterSizeStrategy> get_move_set_size_strategies();
 std::unordered_map<std::string, ClusterStrategy> get_move_set_strategies();
 std::unordered_map<std::string, GainCacheStrategy> get_gain_cache_strategies();
+std::unordered_map<std::string, ActiveSetStrategy> get_active_set_strategies();
 
 std::string get_refinement_algorithms_description();
 std::string get_balancing_algorithms_description();