Not enable peer access in case of the GPUs are located over QPI

BVLC · Nov 12, 2015 · 065d9f5 · 065d9f5
1 parent 19028e7
commit 065d9f5
Showing 1 changed file with 20 additions and 2 deletions.
diff --git a/src/caffe/parallel.cpp b/src/caffe/parallel.cpp
@@ -232,7 +232,18 @@ P2PSync<Dtype>::P2PSync(shared_ptr<Solver<Dtype> > root_solver,
     int access;
     CUDA_CHECK(cudaDeviceCanAccessPeer(&access, self, peer));
     if (access) {
-      CUDA_CHECK(cudaDeviceEnablePeerAccess(peer, 0));
+      cudaDeviceProp a, b;
+      CUDA_CHECK(cudaGetDeviceProperties(&a, self));
+      CUDA_CHECK(cudaGetDeviceProperties(&b, peer));
+      const int pci_bus_id_offset = 0x80;
+      if (std::max(a.pciBusID, b.pciBusID) < pci_bus_id_offset ||
+          std::min(a.pciBusID, b.pciBusID) >= pci_bus_id_offset) {
+        CUDA_CHECK(cudaDeviceEnablePeerAccess(peer, 0));
+      } else {
+        LOG(INFO) << "This will result in poor memcpy performance over QPI, if enables p2p access"
+                  << " from GPU " << self << " (pciBusID " << a.pciBusID << ")"
+                  << " to GPU " << peer << " (pciBusID " << b.pciBusID << ")";
+      }
     } else {
       LOG(INFO)<< "GPU " << self << " does not have p2p access to GPU " << peer;
     }
@@ -262,7 +273,14 @@ P2PSync<Dtype>::~P2PSync() {
     int access;
     CUDA_CHECK(cudaDeviceCanAccessPeer(&access, self, peer));
     if (access) {
-      CUDA_CHECK(cudaDeviceDisablePeerAccess(peer));
+      cudaDeviceProp a, b;
+      CUDA_CHECK(cudaGetDeviceProperties(&a, self));
+      CUDA_CHECK(cudaGetDeviceProperties(&b, peer));
+      const int pci_bus_id_offset = 0x80;
+      if (std::max(a.pciBusID, b.pciBusID) < pci_bus_id_offset ||
+          std::min(a.pciBusID, b.pciBusID) >= pci_bus_id_offset) {
+        CUDA_CHECK(cudaDeviceDisablePeerAccess(peer));
+      }
     }
   }