From 0de68bd8a17e0d4addc2c22933868e887aabc5be Mon Sep 17 00:00:00 2001
From: Kevin Huck <khuck@cs.uoregon.edu>
Date: Mon, 18 Nov 2019 09:33:26 -0700
Subject: [PATCH] Making policy shutdown more robust Adding deregistering of
 periodic policies.

---
 src/apex/apex_policies.cpp                            | 11 ++++++++++-
 src/apex/policy_handler.cpp                           |  4 ++++
 .../testThrottlingActiveHarmony.c                     |  5 +++--
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/apex/apex_policies.cpp b/src/apex/apex_policies.cpp
index c8743808..b3d1b3a8 100644
--- a/src/apex/apex_policies.cpp
+++ b/src/apex/apex_policies.cpp
@@ -44,6 +44,9 @@ static unordered_map<apex_tuning_session_handle,
     shared_ptr<apex_tuning_session>> session_map;
 static std::atomic<apex_tuning_session_handle> next_handle{1};
 
+apex_policy_handle * power_throttling_policy_handle;
+apex_policy_handle * common_setup_timer_throttling_handle;
+
 static shared_ptr<apex_tuning_session> get_session(
     const apex_tuning_session_handle & h) {
   apex::read_lock_type l(session_map_mutex);
@@ -926,7 +929,8 @@ inline int __setup_power_cap_throttling()
             << thread_cap_tuning_session->min_watts << " max watts: "
             << thread_cap_tuning_session->max_watts << endl;
       }
-      apex::register_periodic_policy(apex::apex_options::throttle_energy_period(),
+      power_throttling_policy_handle =
+        apex::register_periodic_policy(apex::apex_options::throttle_energy_period(),
         apex_power_throttling_policy);
       // get an initial power reading
       apex::current_power_high();
@@ -1315,13 +1319,16 @@ inline int __common_setup_timer_throttling(apex_optimization_criteria_t
             thread_cap_tuning_session->cap_data_open = true;
         }
         if (method == APEX_SIMPLE_HYSTERESIS) {
+            common_setup_timer_throttling_handle =
             apex::register_periodic_policy(update_interval,
             apex_throughput_throttling_policy);
         } else if (method == APEX_DISCRETE_HILL_CLIMBING) {
+            common_setup_timer_throttling_handle =
             apex::register_periodic_policy(update_interval,
             apex_throughput_throttling_dhc_policy);
         } else if (method == APEX_ACTIVE_HARMONY) {
             __apex_active_harmony_setup(thread_cap_tuning_session);
+            common_setup_timer_throttling_handle =
             apex::register_periodic_policy(update_interval,
             apex_throughput_throttling_ah_policy);
         }
@@ -1501,6 +1508,8 @@ inline int __shutdown_throttling(void)
         free(thread_cap_tuning_session->observations);
         delete thread_cap_tuning_session;
         thread_cap_tuning_session = nullptr;
+        apex::deregister_policy(power_throttling_policy_handle);
+        apex::deregister_policy(common_setup_timer_throttling_handle);
         return APEX_NOERROR;
     } else {
         return APEX_ERROR;
diff --git a/src/apex/policy_handler.cpp b/src/apex/policy_handler.cpp
index d0d9cd6a..cefeda07 100644
--- a/src/apex/policy_handler.cpp
+++ b/src/apex/policy_handler.cpp
@@ -203,6 +203,9 @@ namespace apex {
     }
 
     int policy_handler::deregister_policy(apex_policy_handle * handle) {
+        if (handle == nullptr) {
+            return APEX_NOERROR;
+        }
         // save the old policy setting
         bool old_policy_setting = apex_options::use_policy();
         // prevent policies from iterating - kind of like a lock, but faster.
@@ -406,6 +409,7 @@ namespace apex {
                 break;
             }
         }
+        handle = nullptr;
         apex_options::use_policy(old_policy_setting);
         return APEX_NOERROR;
     }
diff --git a/src/examples/ThrottlingActiveHarmony/testThrottlingActiveHarmony.c b/src/examples/ThrottlingActiveHarmony/testThrottlingActiveHarmony.c
index a0b92b3d..7ddada9a 100644
--- a/src/examples/ThrottlingActiveHarmony/testThrottlingActiveHarmony.c
+++ b/src/examples/ThrottlingActiveHarmony/testThrottlingActiveHarmony.c
@@ -10,7 +10,7 @@
 #define MAX(a,b) ((a) > (b) ? a : b)
 #define MIN(a,b) ((a) < (b) ? a : b)
 
-#define ITERATIONS 1000
+#define ITERATIONS 10000
 #define SLEEPY_TIME 10000 // 10,000
 
 int num_threads = 1;
@@ -96,9 +96,10 @@ int main(int argc, char **argv)
   }
   apex_stop(p);
   int final_cap = apex_get_thread_cap();
-  if (final_cap < original_cap) {
+  if (final_cap <= original_cap) {
       printf ("Test passed.\n");
   }
+  apex_shutdown_throttling();
   apex_finalize();
   free(thread);
   return(0);