Not enable peer access in case of the GPUs are located over QPI

BVLC · Nov 12, 2015 · d54ae85 · d54ae85
1 parent 19028e7
commit d54ae85
Showing 1 changed file with 22 additions and 2 deletions.
diff --git a/src/caffe/parallel.cpp b/src/caffe/parallel.cpp
@@ -7,6 +7,7 @@
 #include <sys/mman.h>
 #include <sys/stat.h>
 
+#include <algorithm>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -232,7 +233,19 @@ P2PSync<Dtype>::P2PSync(shared_ptr<Solver<Dtype> > root_solver,
     int access;
     CUDA_CHECK(cudaDeviceCanAccessPeer(&access, self, peer));
     if (access) {
-      CUDA_CHECK(cudaDeviceEnablePeerAccess(peer, 0));
+      cudaDeviceProp a, b;
+      CUDA_CHECK(cudaGetDeviceProperties(&a, self));
+      CUDA_CHECK(cudaGetDeviceProperties(&b, peer));
+      const int pci_bus_id_offset = 0x80;
+      if (std::max(a.pciBusID, b.pciBusID) < pci_bus_id_offset ||
+          std::min(a.pciBusID, b.pciBusID) >= pci_bus_id_offset) {
+        CUDA_CHECK(cudaDeviceEnablePeerAccess(peer, 0));
+      } else {
+        LOG(INFO) << "This will result in poor memcpy performance over QPI, "
+                  << "if enables peer to peer access from GPU "
+                  << self << " (pciBusID " << a.pciBusID << ") to GPU "
+                  << peer << " (pciBusID " << b.pciBusID << ")";
+      }
     } else {
       LOG(INFO)<< "GPU " << self << " does not have p2p access to GPU " << peer;
     }
@@ -262,7 +275,14 @@ P2PSync<Dtype>::~P2PSync() {
     int access;
     CUDA_CHECK(cudaDeviceCanAccessPeer(&access, self, peer));
     if (access) {
-      CUDA_CHECK(cudaDeviceDisablePeerAccess(peer));
+      cudaDeviceProp a, b;
+      CUDA_CHECK(cudaGetDeviceProperties(&a, self));
+      CUDA_CHECK(cudaGetDeviceProperties(&b, peer));
+      const int pci_bus_id_offset = 0x80;
+      if (std::max(a.pciBusID, b.pciBusID) < pci_bus_id_offset ||
+          std::min(a.pciBusID, b.pciBusID) >= pci_bus_id_offset) {
+        CUDA_CHECK(cudaDeviceDisablePeerAccess(peer));
+      }
     }
   }