diff --git a/src/apex/apex.cpp b/src/apex/apex.cpp index c5b632ea..0fee121f 100644 --- a/src/apex/apex.cpp +++ b/src/apex/apex.cpp @@ -1386,7 +1386,15 @@ void finalize() // FIRST, stop the top level timer, while the infrastructure is still // functioning. if (top_level_timer != nullptr) { stop(top_level_timer); } - instance->the_profiler_listener->stop_main_timer(); + // if not done already... + shutdown_throttling(); // stop thread scheduler policies + stop_all_async_threads(); // stop OS/HW monitoring + // notify all listeners that we are going to stop soon + for (unsigned int i = 0 ; i < instance->listeners.size() ; i++) { + instance->listeners[i]->on_pre_shutdown(); + } + //instance->the_profiler_listener->stop_main_timer(); + /* This could take a while */ #ifdef APEX_WITH_CUDA flushTrace(); #endif @@ -1394,9 +1402,6 @@ void finalize() apex_options::suspend(true); // now, process all output dump(false); - // if not done already... - shutdown_throttling(); - stop_all_async_threads(); exit_thread(); if (!_measurement_stopped) { diff --git a/src/apex/concurrency_handler.hpp b/src/apex/concurrency_handler.hpp index 3a797440..4a52b241 100644 --- a/src/apex/concurrency_handler.hpp +++ b/src/apex/concurrency_handler.hpp @@ -58,6 +58,7 @@ class concurrency_handler : public handler, public event_listener { void on_startup(startup_event_data &data) { APEX_UNUSED(data); }; void on_dump(dump_event_data &data); void on_reset(task_identifier * id); + void on_pre_shutdown(void) { } ; void on_shutdown(shutdown_event_data &data); void on_new_node(node_event_data &data) { APEX_UNUSED(data); }; void on_new_thread(new_thread_event_data &data); diff --git a/src/apex/event_listener.hpp b/src/apex/event_listener.hpp index c21d89f7..493e5105 100644 --- a/src/apex/event_listener.hpp +++ b/src/apex/event_listener.hpp @@ -113,6 +113,7 @@ class event_listener virtual ~event_listener() {}; // all methods in the interface that a handler has to override virtual void on_startup(startup_event_data &data) = 0; + virtual void on_pre_shutdown(void) = 0; virtual void on_shutdown(shutdown_event_data &data) = 0; virtual void on_dump(dump_event_data &data) = 0; virtual void on_reset(task_identifier * id) = 0; diff --git a/src/apex/otf2_listener.cpp b/src/apex/otf2_listener.cpp index 03e64e7d..c83fd381 100644 --- a/src/apex/otf2_listener.cpp +++ b/src/apex/otf2_listener.cpp @@ -108,6 +108,7 @@ namespace apex { }; uint64_t otf2_listener::globalOffset(0); + uint64_t otf2_listener::saved_end_timestamp(0); const std::string otf2_listener::empty(""); int otf2_listener::my_saved_node_id(0); int otf2_listener::my_saved_node_count(1); @@ -773,7 +774,7 @@ namespace apex { void otf2_listener::write_clock_properties(void) { /* write the clock properties */ uint64_t ticks_per_second = 1e9; - uint64_t traceLength = get_time(); + uint64_t traceLength = saved_end_timestamp; OTF2_GlobalDefWriter_WriteClockProperties( global_def_writer, ticks_per_second, 0 /* start */, traceLength /* length */ ); } @@ -854,6 +855,12 @@ namespace apex { return; } + /* Before shutdown, take a timestamp in case the shutdown process + * takes a really long time, we don't want to confuse the user */ + void otf2_listener::on_pre_shutdown(void) { + saved_end_timestamp = get_time(); + } + /* At shutdown, we need to reduce all the global information, * and write out the global definitions - strings, regions, * locations, communicators, groups, metrics, etc. diff --git a/src/apex/otf2_listener.hpp b/src/apex/otf2_listener.hpp index b3293d94..c03b44d5 100644 --- a/src/apex/otf2_listener.hpp +++ b/src/apex/otf2_listener.hpp @@ -40,6 +40,9 @@ namespace apex { /* The global offset is referenced from the get_time static function, * so it needs to be static itself. */ static uint64_t globalOffset; + /* We take an end timestamp before shutdown, because we don't + * want to confuse the user with a long trace time */ + static uint64_t saved_end_timestamp; /* All OTF2 callback functions have to be declared static, so that they * can be registered with the OTF2 library */ static OTF2_TimeStamp get_time( void ) { @@ -213,6 +216,7 @@ namespace apex { void on_dump(dump_event_data &data); void on_reset(task_identifier * id) { APEX_UNUSED(id); }; + void on_pre_shutdown(void); void on_shutdown(shutdown_event_data &data); void on_new_node(node_event_data &data); void on_new_thread(new_thread_event_data &data); diff --git a/src/apex/policy_handler.hpp b/src/apex/policy_handler.hpp index bd438c92..82fa57eb 100644 --- a/src/apex/policy_handler.hpp +++ b/src/apex/policy_handler.hpp @@ -93,6 +93,7 @@ class policy_handler : public handler, public event_listener void on_dump(dump_event_data &data); void on_reset(task_identifier * id) { APEX_UNUSED(id); }; + void on_pre_shutdown(void) {}; void on_shutdown(shutdown_event_data &data); void on_new_node(node_event_data &data); void on_new_thread(new_thread_event_data &data); diff --git a/src/apex/profiler_listener.cpp b/src/apex/profiler_listener.cpp index f702e9d3..07610cd9 100644 --- a/src/apex/profiler_listener.cpp +++ b/src/apex/profiler_listener.cpp @@ -1755,6 +1755,10 @@ if (rc != 0) cout << "PAPI error! " << name << ": " << PAPI_strerror(rc) << endl } } + void profiler_listener::on_pre_shutdown(void) { + stop_main_timer(); + } + void profiler_listener::push_profiler_public(std::shared_ptr &p) { push_profiler(0, p); } diff --git a/src/apex/profiler_listener.hpp b/src/apex/profiler_listener.hpp index 724f013b..50f26523 100644 --- a/src/apex/profiler_listener.hpp +++ b/src/apex/profiler_listener.hpp @@ -206,6 +206,7 @@ class profiler_listener : public event_listener { void on_startup(startup_event_data &data); void on_dump(dump_event_data &data); void on_reset(task_identifier * id); + void on_pre_shutdown(void); void on_shutdown(shutdown_event_data &data); void on_new_node(node_event_data &data); void on_new_thread(new_thread_event_data &data); diff --git a/src/apex/tau_listener.hpp b/src/apex/tau_listener.hpp index 7637c9be..1888c430 100644 --- a/src/apex/tau_listener.hpp +++ b/src/apex/tau_listener.hpp @@ -26,6 +26,7 @@ class tau_listener : public event_listener { void on_dump(dump_event_data &data); void on_reset(task_identifier * id) { APEX_UNUSED(id); }; + void on_pre_shutdown(void) {}; void on_shutdown(shutdown_event_data &data); void on_new_node(node_event_data &data); void on_new_thread(new_thread_event_data &data); diff --git a/src/apex/trace_event_listener.cpp b/src/apex/trace_event_listener.cpp index b81e6ffd..2ada5d75 100644 --- a/src/apex/trace_event_listener.cpp +++ b/src/apex/trace_event_listener.cpp @@ -57,6 +57,10 @@ void trace_event_listener::on_dump(dump_event_data &data) { return; } +void trace_event_listener::on_pre_shutdown(void) { + end_trace_time(); +} + void trace_event_listener::on_shutdown(shutdown_event_data &data) { APEX_UNUSED(data); if (!_terminate) { diff --git a/src/apex/trace_event_listener.hpp b/src/apex/trace_event_listener.hpp index f174b8b8..2b6248f5 100644 --- a/src/apex/trace_event_listener.hpp +++ b/src/apex/trace_event_listener.hpp @@ -26,6 +26,7 @@ class trace_event_listener : public event_listener { void on_dump(dump_event_data &data); void on_reset(task_identifier * id) { APEX_UNUSED(id); }; + void on_pre_shutdown(void); void on_shutdown(shutdown_event_data &data); void on_new_node(node_event_data &data); void on_new_thread(new_thread_event_data &data);