Fixing issue #135

When tracking CPU/GPU activity, the memory allocation counters should
be associated with the thread making the call, when writing to OTF2
traces.  This change allows for an optional argument to the
apex::sample_value call that indicates whether the counter is assocaited
with the specific thread or the process as a whole (the default).

src/apex/activity_trace_async.cpp

@@ -338,14 +338,14 @@ void store_profiler_data(const std::string &name, uint32_t correlationId,
 void store_sync_counter_data(const char * name, const std::string& context,
     double value, bool force = false) {
     if (name == nullptr) {
-        apex::sample_value(context, value);
+        apex::sample_value(context, value, true);
     } else {
         std::stringstream ss;
         ss << name;
         if (apex::apex_options::use_cuda_kernel_details() || force) {
             ss << ": " << context;
         }
-        apex::sample_value(ss.str(), value);
+        apex::sample_value(ss.str(), value, true);
     }
 }
 

src/apex/apex.cpp

@@ -1070,7 +1070,7 @@ void yield(std::shared_ptr<task_wrapper> tt_ptr)
     tt_ptr->prof = nullptr;
 }
 
-void sample_value(const std::string &name, double value)
+void sample_value(const std::string &name, double value, bool threaded)
 {
     // if APEX is disabled, do nothing.
     if (apex_options::disable() == true) { return; }
@@ -1103,7 +1103,7 @@ void sample_value(const std::string &name, double value)
             }
         }
     }
-    sample_value_event_data data(tid, name, value);
+    sample_value_event_data data(tid, name, value, threaded);
     if (_notify_listeners) {
         //read_lock_type l(instance->listener_mutex);
         for (unsigned int i = 0 ; i < instance->listeners.size() ; i++) {
@@ -2010,9 +2010,9 @@ extern "C" {
         yield(reinterpret_cast<profiler*>(the_profiler));
     }
 
-    void apex_sample_value(const char * name, double value) {
+    void apex_sample_value(const char * name, double value, bool threaded) {
         string tmp(name);
-        sample_value(tmp, value);
+        sample_value(tmp, value, threaded);
     }
 
     void apex_new_task(apex_profiler_type type, void * identifier,

src/apex/apex_api.hpp

@@ -332,9 +332,10 @@ APEX_EXPORT void set_state(apex_thread_state state);
 
  \param name The name of the sampled value
  \param value The sampled value
+ \param threaded Whether this is a per-thread value, or process-wide
  \return No return value.
  */
-APEX_EXPORT void sample_value(const std::string &name, double value);
+APEX_EXPORT void sample_value(const std::string &name, double value, bool threaded = false);
 
 /**
  \brief Create a new task (dependency).

src/apex/event_listener.cpp

@@ -46,12 +46,13 @@ node_event_data::node_event_data(int node_id, int thread_id) {
 }
 
 sample_value_event_data::sample_value_event_data(int thread_id,
-    string counter_name, double counter_value) {
+    string counter_name, double counter_value, bool threaded) {
   this->event_type_ = APEX_SAMPLE_VALUE;
   this->is_counter = true;
   this->thread_id = thread_id;
   this->counter_name = new string(counter_name);
   this->counter_value = counter_value;
+  this->is_threaded = threaded;
 }
 
 sample_value_event_data::~sample_value_event_data() {

src/apex/event_listener.hpp

@@ -63,8 +63,9 @@ class sample_value_event_data : public event_data {
 public:
   std::string * counter_name;
   double counter_value;
+  bool is_threaded;
   bool is_counter;
-  sample_value_event_data(int thread_id, std::string counter_name, double counter_value);
+  sample_value_event_data(int thread_id, std::string counter_name, double counter_value, bool threaded);
   ~sample_value_event_data();
 };
 

src/apex/otf2_listener.cpp

@@ -1344,6 +1344,11 @@ namespace apex {
         read_lock_type lock(_archive_mutex);
         // not likely, but just in case...
         if (_terminate) { return; }
+
+        OTF2_EvtWriter* local_evt_writer = comm_evt_writer;
+        if (data.is_threaded) {
+            local_evt_writer = getEvtWriter(true);
+        }
         // create a union for storing the value
         OTF2_MetricValue omv[1];
         omv[0].floating_point = data.counter_value;
@@ -1360,8 +1365,8 @@ namespace apex {
             // that time stamps are monotonically increasing. :(
             uint64_t stamp = get_time();
             // write our counter into the event stream
-            if (comm_evt_writer != nullptr) {
-                OTF2_EC(OTF2_EvtWriter_Metric( comm_evt_writer, nullptr, stamp,
+            if (local_evt_writer != nullptr) {
+                OTF2_EC(OTF2_EvtWriter_Metric( local_evt_writer, nullptr, stamp,
                 idx, 1, omt, omv ));
             }
         }