Fixing bug #134.

Changing from pthread_setaffinity_np() to sched_get/setaffinity()
UO-OACISS · Mar 11, 2021 · 10ca3bf · 10ca3bf
1 parent e905352
commit 10ca3bf
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 50 deletions.
diff --git a/src/apex/utils.cpp b/src/apex/utils.cpp
@@ -15,7 +15,11 @@
 #endif
 // for setting thread affinity
 #if !defined(__APPLE__) && !defined(_MSC_VER)
-#include <pthread.h>
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <sched.h>
+#include <unistd.h>
 #endif
 #include <stdio.h>
 #include <stdlib.h>
@@ -30,6 +34,7 @@
 #include <iostream>
 #include <string>
 #include <vector>
+#include <mutex>
 
 namespace apex {
 
@@ -106,33 +111,18 @@ std::string* demangle(const std::string& timer_name) {
 
 void set_thread_affinity(int core) {
 #if !defined(__APPLE__) && !defined(_MSC_VER) && !defined(APEX_HAVE_HPX)
-    int s;
     cpu_set_t cpuset;
-    pthread_t thread;
-
-    thread = pthread_self();
-
-    /* Set affinity mask to include CPUs 0 to 7 */
-
-    CPU_ZERO(&cpuset);
-    CPU_SET(core, &cpuset);
-
-    s = pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
-    if (s != 0) handle_error_en(s, "pthread_setaffinity_np");
+    cpu_set_t mask;
 
-    /* Check the actual affinity mask assigned to the thread */
-
-    /*
-    s = pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
-    if (s != 0) handle_error_en(s, "pthread_getaffinity_np");
-
-    printf("Set returned by pthread_getaffinity_np() contained:\n");
-    for (j = 0; j < CPU_SETSIZE; j++) {
-        if (CPU_ISSET(j, &cpuset)) {
-            printf("    CPU %d\n", j);
-        }
+    if (sched_getaffinity(0, sizeof(cpu_set_t), &mask) == -1) {
+        perror("sched_getaffinity");
+        return;
+    }
+    if (CPU_ISSET(core, &mask)) {
+        CPU_ZERO(&cpuset);
+        CPU_SET(core, &cpuset);
+        sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
     }
-    */
 #else
     APEX_UNUSED(core);
 #endif
@@ -141,34 +131,28 @@ void set_thread_affinity(int core) {
 
 void set_thread_affinity(void) {
 #if !defined(__APPLE__) && !defined(_MSC_VER) && !defined(APEX_HAVE_HPX)
-    int s, j;
     cpu_set_t cpuset;
-    pthread_t thread;
-
-    thread = pthread_self();
-
-    /* Set affinity mask to include CPUs 0 to 7 */
-
-    CPU_ZERO(&cpuset);
-    j = my_hardware_concurrency() - 1;
-    CPU_SET(j, &cpuset);
-
-    s = pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
-    if (s != 0) handle_error_en(s, "pthread_setaffinity_np");
-
-    /* Check the actual affinity mask assigned to the thread */
-
-    /*
-    s = pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
-    if (s != 0) handle_error_en(s, "pthread_getaffinity_np");
-
-    printf("Set returned by pthread_getaffinity_np() contained:\n");
-    for (j = 0; j < CPU_SETSIZE; j++) {
-        if (CPU_ISSET(j, &cpuset)) {
-            printf("    CPU %d\n", j);
+    cpu_set_t mask;
+
+    // only let one thread in here at a time
+    std::mutex _mutex;
+    std::unique_lock<std::mutex> l(_mutex);
+    static unsigned int last_assigned = my_hardware_concurrency();
+
+    if (sched_getaffinity(0, sizeof(cpu_set_t), &mask) == -1) {
+        perror("sched_getaffinity");
+        return;
+    }
+    unsigned int j = my_hardware_concurrency() - 1;
+    for (unsigned int i = j ; i > 0 ; i--) {
+        if (CPU_ISSET(i, &mask) && i < last_assigned) {
+            CPU_ZERO(&cpuset);
+            CPU_SET(i, &cpuset);
+            sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
+            last_assigned = i;
+            break;
         }
     }
-    */
 #endif
     return;
 }

diff --git a/src/examples/PeriodicPlugin/CMakeLists.txt b/src/examples/PeriodicPlugin/CMakeLists.txt
@@ -16,6 +16,7 @@ set_target_properties(apex_periodic_policy PROPERTIES OUTPUT_NAME apex_periodic_
 
 add_executable (periodic_policy_test policy_test.cpp)
 add_dependencies (periodic_policy_test apex)
+target_link_libraries (periodic_policy_test ${LIBS})
 
 INSTALL(TARGETS apex_periodic_policy periodic_policy_test
     RUNTIME DESTINATION bin