diff --git a/public/worknotes/404.html b/public/worknotes/404.html
index e95aaa3..3abe2c2 100644
--- a/public/worknotes/404.html
+++ b/public/worknotes/404.html
@@ -15,7 +15,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/categories/index.html b/public/worknotes/categories/index.html
index 0e6e983..d13d7f3 100644
--- a/public/worknotes/categories/index.html
+++ b/public/worknotes/categories/index.html
@@ -14,7 +14,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/categories/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/docs/caf/index.html b/public/worknotes/docs/caf/index.html
index b4bfc7d..668a112 100644
--- a/public/worknotes/docs/caf/index.html
+++ b/public/worknotes/docs/caf/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cfdem/cavitycfdem/index.html b/public/worknotes/docs/cfdem/cavitycfdem/index.html
index fa6c5e9..ef38c0d 100644
--- a/public/worknotes/docs/cfdem/cavitycfdem/index.html
+++ b/public/worknotes/docs/cfdem/cavitycfdem/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cfdem/index.html b/public/worknotes/docs/cfdem/index.html
index e929d83..a6ef446 100644
--- a/public/worknotes/docs/cfdem/index.html
+++ b/public/worknotes/docs/cfdem/index.html
@@ -14,7 +14,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/docs/cfdem/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/docs/cfdem/liggghts-intel-comp/index.html b/public/worknotes/docs/cfdem/liggghts-intel-comp/index.html
index 7bbe153..203c475 100644
--- a/public/worknotes/docs/cfdem/liggghts-intel-comp/index.html
+++ b/public/worknotes/docs/cfdem/liggghts-intel-comp/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cfdem/snappyhexmesh/index.html b/public/worknotes/docs/cfdem/snappyhexmesh/index.html
index 032ba23..81ece36 100644
--- a/public/worknotes/docs/cfdem/snappyhexmesh/index.html
+++ b/public/worknotes/docs/cfdem/snappyhexmesh/index.html
@@ -16,7 +16,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/cuda-wsl/index.html b/public/worknotes/docs/cuda/cuda-wsl/index.html
index d948c6b..00fb3cd 100644
--- a/public/worknotes/docs/cuda/cuda-wsl/index.html
+++ b/public/worknotes/docs/cuda/cuda-wsl/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/hidden/index.html b/public/worknotes/docs/cuda/hidden/index.html
index 3e01cec..2b9bdfc 100644
--- a/public/worknotes/docs/cuda/hidden/index.html
+++ b/public/worknotes/docs/cuda/hidden/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/index.html b/public/worknotes/docs/cuda/index.html
index 0e3753d..2714ab2 100644
--- a/public/worknotes/docs/cuda/index.html
+++ b/public/worknotes/docs/cuda/index.html
@@ -14,7 +14,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/docs/cuda/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/docs/cuda/reference-codes/MemCpy/index.html b/public/worknotes/docs/cuda/reference-codes/MemCpy/index.html
index 978beda..a127e2a 100644
--- a/public/worknotes/docs/cuda/reference-codes/MemCpy/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/MemCpy/index.html
@@ -22,7 +22,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/bandwidthTest/index.html b/public/worknotes/docs/cuda/reference-codes/bandwidthTest/index.html
index 34960e5..2193d7f 100644
--- a/public/worknotes/docs/cuda/reference-codes/bandwidthTest/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/bandwidthTest/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/checkP2pAccess/index.html b/public/worknotes/docs/cuda/reference-codes/checkP2pAccess/index.html
index 3fb7eb4..3bfe8c2 100644
--- a/public/worknotes/docs/cuda/reference-codes/checkP2pAccess/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/checkP2pAccess/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/deviceQuery/index.html b/public/worknotes/docs/cuda/reference-codes/deviceQuery/index.html
index 1820111..175491c 100644
--- a/public/worknotes/docs/cuda/reference-codes/deviceQuery/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/deviceQuery/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/directTransfer/index.html b/public/worknotes/docs/cuda/reference-codes/directTransfer/index.html
index 0de8c9b..6cb9f75 100644
--- a/public/worknotes/docs/cuda/reference-codes/directTransfer/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/directTransfer/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/error/index.html b/public/worknotes/docs/cuda/reference-codes/error/index.html
index 38c028a..1f30e7f 100644
--- a/public/worknotes/docs/cuda/reference-codes/error/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/error/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/index.html b/public/worknotes/docs/cuda/reference-codes/index.html
index d5623c2..a14c4b7 100644
--- a/public/worknotes/docs/cuda/reference-codes/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/index.html
@@ -15,7 +15,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/docs/cuda/reference-codes/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/docs/cuda/reference-codes/limitingFactor/index.html b/public/worknotes/docs/cuda/reference-codes/limitingFactor/index.html
index 25721b2..b3315b9 100644
--- a/public/worknotes/docs/cuda/reference-codes/limitingFactor/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/limitingFactor/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/mpiDeviceUtil/index.html b/public/worknotes/docs/cuda/reference-codes/mpiDeviceUtil/index.html
index 0299c31..bb5168f 100644
--- a/public/worknotes/docs/cuda/reference-codes/mpiDeviceUtil/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/mpiDeviceUtil/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/mpiDevices/index.html b/public/worknotes/docs/cuda/reference-codes/mpiDevices/index.html
index 93a16c2..4dd3fc2 100644
--- a/public/worknotes/docs/cuda/reference-codes/mpiDevices/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/mpiDevices/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/offsetNStride/index.html b/public/worknotes/docs/cuda/reference-codes/offsetNStride/index.html
index 4881c7f..713496f 100644
--- a/public/worknotes/docs/cuda/reference-codes/offsetNStride/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/offsetNStride/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/p2pBandwidth/index.html b/public/worknotes/docs/cuda/reference-codes/p2pBandwidth/index.html
index ccc0147..3225a7e 100644
--- a/public/worknotes/docs/cuda/reference-codes/p2pBandwidth/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/p2pBandwidth/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/peakBandwidth/index.html b/public/worknotes/docs/cuda/reference-codes/peakBandwidth/index.html
index fcce611..fe77fc4 100644
--- a/public/worknotes/docs/cuda/reference-codes/peakBandwidth/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/peakBandwidth/index.html
@@ -22,7 +22,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/precision_m/index.html b/public/worknotes/docs/cuda/reference-codes/precision_m/index.html
index 520e464..4baaa6a 100644
--- a/public/worknotes/docs/cuda/reference-codes/precision_m/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/precision_m/index.html
@@ -16,7 +16,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/sharedExample/index.html b/public/worknotes/docs/cuda/reference-codes/sharedExample/index.html
index 028a301..b7242f5 100644
--- a/public/worknotes/docs/cuda/reference-codes/sharedExample/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/sharedExample/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/strideTexture/index.html b/public/worknotes/docs/cuda/reference-codes/strideTexture/index.html
index 0eaaf52..8011142 100644
--- a/public/worknotes/docs/cuda/reference-codes/strideTexture/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/strideTexture/index.html
@@ -22,7 +22,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/testAsync/index.html b/public/worknotes/docs/cuda/reference-codes/testAsync/index.html
index 612affa..f3f89d6 100644
--- a/public/worknotes/docs/cuda/reference-codes/testAsync/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/testAsync/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/cuda/reference-codes/transposeMPI/index.html b/public/worknotes/docs/cuda/reference-codes/transposeMPI/index.html
index 32f7b33..3855a8b 100644
--- a/public/worknotes/docs/cuda/reference-codes/transposeMPI/index.html
+++ b/public/worknotes/docs/cuda/reference-codes/transposeMPI/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/f-cpp/index.html b/public/worknotes/docs/f-cpp/index.html
index 94d51e3..16359c5 100644
--- a/public/worknotes/docs/f-cpp/index.html
+++ b/public/worknotes/docs/f-cpp/index.html
@@ -16,7 +16,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/index.html b/public/worknotes/docs/index.html
index fbbc74f..65ca6e9 100644
--- a/public/worknotes/docs/index.html
+++ b/public/worknotes/docs/index.html
@@ -14,7 +14,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/docs/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/docs/manual-vectorization/addition/index.html b/public/worknotes/docs/manual-vectorization/addition/index.html
index 33b37ac..d1f817c 100644
--- a/public/worknotes/docs/manual-vectorization/addition/index.html
+++ b/public/worknotes/docs/manual-vectorization/addition/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/manual-vectorization/cll-avx512/index.html b/public/worknotes/docs/manual-vectorization/cll-avx512/index.html
index a469faf..d4b4f26 100644
--- a/public/worknotes/docs/manual-vectorization/cll-avx512/index.html
+++ b/public/worknotes/docs/manual-vectorization/cll-avx512/index.html
@@ -16,7 +16,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/manual-vectorization/faster-sumreduce/index.html b/public/worknotes/docs/manual-vectorization/faster-sumreduce/index.html
index 6bfb4e9..532f09d 100644
--- a/public/worknotes/docs/manual-vectorization/faster-sumreduce/index.html
+++ b/public/worknotes/docs/manual-vectorization/faster-sumreduce/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/manual-vectorization/index.html b/public/worknotes/docs/manual-vectorization/index.html
index 14f9859..90d85ef 100644
--- a/public/worknotes/docs/manual-vectorization/index.html
+++ b/public/worknotes/docs/manual-vectorization/index.html
@@ -14,7 +14,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/docs/manual-vectorization/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/docs/manual-vectorization/motivation/index.html b/public/worknotes/docs/manual-vectorization/motivation/index.html
index e0fccea..1fd1e93 100644
--- a/public/worknotes/docs/manual-vectorization/motivation/index.html
+++ b/public/worknotes/docs/manual-vectorization/motivation/index.html
@@ -16,7 +16,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/manual-vectorization/sumreduce/index.html b/public/worknotes/docs/manual-vectorization/sumreduce/index.html
index 83f9366..1479191 100644
--- a/public/worknotes/docs/manual-vectorization/sumreduce/index.html
+++ b/public/worknotes/docs/manual-vectorization/sumreduce/index.html
@@ -16,7 +16,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/manual-vectorization/sweep/index.html b/public/worknotes/docs/manual-vectorization/sweep/index.html
index c63d59f..34d8d89 100644
--- a/public/worknotes/docs/manual-vectorization/sweep/index.html
+++ b/public/worknotes/docs/manual-vectorization/sweep/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/pi/index.html b/public/worknotes/docs/pi/index.html
index 7dfd7c0..07682ff 100644
--- a/public/worknotes/docs/pi/index.html
+++ b/public/worknotes/docs/pi/index.html
@@ -14,7 +14,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/docs/pi/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/docs/pi/slurm-cluster/index.html b/public/worknotes/docs/pi/slurm-cluster/index.html
index 8855f3c..37e0945 100644
--- a/public/worknotes/docs/pi/slurm-cluster/index.html
+++ b/public/worknotes/docs/pi/slurm-cluster/index.html
@@ -24,7 +24,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/pi/trim-jetsonnano/index.html b/public/worknotes/docs/pi/trim-jetsonnano/index.html
index 739cdce..2e1f565 100644
--- a/public/worknotes/docs/pi/trim-jetsonnano/index.html
+++ b/public/worknotes/docs/pi/trim-jetsonnano/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/pi/webserver/index.html b/public/worknotes/docs/pi/webserver/index.html
index faba8c6..76b3268 100644
--- a/public/worknotes/docs/pi/webserver/index.html
+++ b/public/worknotes/docs/pi/webserver/index.html
@@ -20,7 +20,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-search/index.html b/public/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-search/index.html
index c36b080..fab78d5 100644
--- a/public/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-search/index.html
+++ b/public/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-search/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
@@ -784,7 +784,8 @@ <h2 id="code-fortran">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">   </span><span style="color:#8f5902;font-style:italic">!---------------------------------------------------------------------------
 </span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">   </span><span style="color:#204a87;font-weight:bold">function</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">coordsToCell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">xi</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">result</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! function to convert coordinates to grid cells
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">implicit</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">none</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">real</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">f</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">intent</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">in</span><span style="color:#000;font-weight:bold">)</span><span style="color:#204a87;font-weight:bold">::</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">xi</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">integer</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">intent</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">in</span><span style="color:#000;font-weight:bold">)</span><span style="color:#204a87;font-weight:bold">::</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#f8f8f8;text-decoration:underline">
@@ -806,21 +807,25 @@ <h2 id="code-fortran">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">real</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">f</span><span style="color:#000;font-weight:bold">)</span><span style="color:#204a87;font-weight:bold">::</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">r2</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">integer</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">allocatable</span><span style="color:#204a87;font-weight:bold">::</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cells</span><span style="color:#000;font-weight:bold">(:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">:</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">minval</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! determine grid min-max coordinates, with concessions made for dim=2 case
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">:</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">minval</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">if</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">==</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">0.d0</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">:</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">maxval</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">if</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">==</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">0.d0</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2.d0</span><span style="color:#ce5c00;font-weight:bold">*</span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! creating buffer layers
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2.d0</span><span style="color:#ce5c00;font-weight:bold">*</span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">+</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2.d0</span><span style="color:#ce5c00;font-weight:bold">*</span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">int</span><span style="color:#000;font-weight:bold">((</span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:))</span><span style="color:#ce5c00;font-weight:bold">/</span><span style="color:#000">cutoff</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">+</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">+</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#ce5c00;font-weight:bold">*</span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">allocate</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">&amp;</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! initializing grid arrays
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">allocate</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">&amp;</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">                </span><span style="color:#000">cells</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">maxpercell</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)))</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">0</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! populating grid with point information
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#000">icell</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">coordsToCell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">(:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">))</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">&amp;</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">            </span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">))</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">+</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#f8f8f8;text-decoration:underline">
@@ -835,7 +840,8 @@ <h2 id="code-fortran">
 </span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#000">cells</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">))</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">end</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">npairs</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">0</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! perform pair search
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">npairs</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">0</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">coordsToCell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">(:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">xi</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#f8f8f8;text-decoration:underline">
diff --git a/public/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search/index.html b/public/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search/index.html
index b2e9b74..fcb57a1 100644
--- a/public/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search/index.html
+++ b/public/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
@@ -842,21 +842,24 @@ <h2 id="code-fortran">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">                                                         </span><span style="color:#0000cf;font-weight:bold">0</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline">  </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">&amp;</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">                                                         </span><span style="color:#0000cf;font-weight:bold">0</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline">  </span><span style="color:#0000cf;font-weight:bold">0</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">],</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">[</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">13</span><span style="color:#000;font-weight:bold">])</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">:</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">minval</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! determine grid min-max coordinates, with concessions made for dim=2 case
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">:</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">minval</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">if</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">==</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">0.d0</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">:</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">maxval</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">if</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">==</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">0.d0</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2.d0</span><span style="color:#ce5c00;font-weight:bold">*</span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! creating buffer layers
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2.d0</span><span style="color:#ce5c00;font-weight:bold">*</span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">+</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">2.d0</span><span style="color:#ce5c00;font-weight:bold">*</span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">int</span><span style="color:#000;font-weight:bold">((</span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">-</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">(:))</span><span style="color:#ce5c00;font-weight:bold">/</span><span style="color:#000">cutoff</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">+</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">+</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(:)</span><span style="color:#ce5c00;font-weight:bold">*</span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">allocate</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">&amp;</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! initializing grid arrays
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">allocate</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">&amp;</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">                </span><span style="color:#000">cells</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">maxpercell</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">ngridx</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">)))</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">:)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">0</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! populating grid with point information
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#000">icell</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">coordsToCell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">(:,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">i</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">minx</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">))</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000;font-weight:bold">&amp;</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">            </span><span style="color:#000">pincell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">2</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">icell</span><span style="color:#000;font-weight:bold">(</span><span style="color:#0000cf;font-weight:bold">3</span><span style="color:#000;font-weight:bold">))</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">+</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#0000cf;font-weight:bold">1</span><span style="color:#f8f8f8;text-decoration:underline">
diff --git a/public/worknotes/docs/useful/fixed-cutoff-direct-pair-search/index.html b/public/worknotes/docs/useful/fixed-cutoff-direct-pair-search/index.html
index 269d9fa..16f9075 100644
--- a/public/worknotes/docs/useful/fixed-cutoff-direct-pair-search/index.html
+++ b/public/worknotes/docs/useful/fixed-cutoff-direct-pair-search/index.html
@@ -18,7 +18,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
@@ -779,7 +779,9 @@ <h2 id="code-fortran">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">   </span><span style="color:#8f5902;font-style:italic">!---------------------------------------------------------------------------
 </span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">   </span><span style="color:#204a87;font-weight:bold">subroutine</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">dsearch_dp</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxnpair</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npairs</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">pair_i</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">pair_j</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! subroutine to find pairs of points using direct search. Output is 
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! two lists describing which points are in the pair
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">implicit</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">none</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">integer</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">intent</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">in</span><span style="color:#000;font-weight:bold">)</span><span style="color:#204a87;font-weight:bold">::</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxnpair</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">real</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">f</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">intent</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">in</span><span style="color:#000;font-weight:bold">)</span><span style="color:#204a87;font-weight:bold">::</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
@@ -804,7 +806,9 @@ <h2 id="code-fortran">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">   </span><span style="color:#8f5902;font-style:italic">!---------------------------------------------------------------------------
 </span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">   </span><span style="color:#204a87;font-weight:bold">subroutine</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">dsearch_compact_dp</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxnpair</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npairs</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">endpos</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">pair_j</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! subroutine to find pairs using direct search. Output is two lists. One
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#8f5902;font-style:italic">! list describes the last pair with point `i` as the &#34;left-sided&#34; point.
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">implicit</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">none</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">integer</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">intent</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">in</span><span style="color:#000;font-weight:bold">)</span><span style="color:#204a87;font-weight:bold">::</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">maxnpair</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">real</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">f</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">intent</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">in</span><span style="color:#000;font-weight:bold">)</span><span style="color:#204a87;font-weight:bold">::</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">x</span><span style="color:#000;font-weight:bold">(</span><span style="color:#204a87">dim</span><span style="color:#000;font-weight:bold">,</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npoints</span><span style="color:#000;font-weight:bold">),</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">cutoff</span><span style="color:#f8f8f8;text-decoration:underline">
@@ -822,7 +826,8 @@ <h2 id="code-fortran">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">               </span><span style="color:#000">pair_j</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">npairs</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">j</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">            </span><span style="color:#204a87;font-weight:bold">end</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">if</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#204a87;font-weight:bold">end</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline">
-</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#000">endpos</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">i</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npairs</span><span style="color:#f8f8f8;text-decoration:underline">
+</span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#8f5902;font-style:italic">! update end position when done with point i
+</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"></span><span style="color:#f8f8f8;text-decoration:underline">         </span><span style="color:#000">endpos</span><span style="color:#000;font-weight:bold">(</span><span style="color:#000">i</span><span style="color:#000;font-weight:bold">)</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#ce5c00;font-weight:bold">=</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">npairs</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">      </span><span style="color:#204a87;font-weight:bold">end</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">do</span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">
 </span></span></span><span style="display:flex;"><span><span style="color:#f8f8f8;text-decoration:underline">   </span><span style="color:#204a87;font-weight:bold">end</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#204a87;font-weight:bold">subroutine</span><span style="color:#f8f8f8;text-decoration:underline"> </span><span style="color:#000">dsearch_compact_dp</span><span style="color:#f8f8f8;text-decoration:underline">
diff --git a/public/worknotes/docs/useful/grid-rows-spatial-hashing/index.html b/public/worknotes/docs/useful/grid-rows-spatial-hashing/index.html
index 4a470d1..c3c78f1 100644
--- a/public/worknotes/docs/useful/grid-rows-spatial-hashing/index.html
+++ b/public/worknotes/docs/useful/grid-rows-spatial-hashing/index.html
@@ -10,13 +10,13 @@
 <meta property="og:type" content="article" />
 <meta property="og:url" content="/worknotes/docs/useful/grid-rows-spatial-hashing/" /><meta property="article:section" content="docs" />
 
-<meta property="article:modified_time" content="2023-03-18T01:36:43+11:00" />
+<meta property="article:modified_time" content="2023-09-10T15:17:43+10:00" />
 <title>grid dimension hashing | Ed&#39;s Space - Notes</title>
 <link rel="manifest" href="/worknotes/manifest.json">
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
@@ -800,7 +800,7 @@ <h2 id="code-fortran-and-c">
   <a class="anchor" href="#code-fortran-and-c">#</a>
 </h2>
 <p>Save the C++ file as <code>sort.cpp</code>, and Fortran code as <code>cll3.F90</code>. Compile the code with</p>
-<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-bash" data-lang="bash"><span style="display:flex;"><span>g++ -c sort.cpp
+<div class="highlight"><pre tabindex="0" style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-bash" data-lang="bash"><span style="display:flex;"><span>g++ -c sort.cpp
 </span></span><span style="display:flex;"><span>gfortran -o cll3.x cll3.F90 sort.o -lstdc++
 </span></span></code></pre></div><div class="highlight"><pre tabindex="0" style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-cpp" data-lang="cpp"><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic">// sort.cpp
 </span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic">// C++ source code containing STL sort - modified for sort by indices
@@ -1084,9 +1084,9 @@ <h2 id="code-fortran-and-c">
   <div class="flex flex-wrap justify-between">
 
 
-  <div><a class="flex align-center" href="https://github.com/edoyango/personal-webpage/commit/7daac53108f4650eb77a9816344969fd8e7c434d" title='Last modified by Edward Yang | March 18, 2023' target="_blank" rel="noopener">
+  <div><a class="flex align-center" href="https://github.com/edoyango/personal-webpage/commit/cfeb5a0ba27ba967fada99387584dcdb0c66198d" title='Last modified by Edward Yang | September 10, 2023' target="_blank" rel="noopener">
       <img src="/worknotes/svg/calendar.svg" class="book-icon" alt="Calendar" />
-      <span>March 18, 2023</span>
+      <span>September 10, 2023</span>
     </a>
   </div>
 
diff --git a/public/worknotes/docs/useful/index.html b/public/worknotes/docs/useful/index.html
index c5e96e7..fbb5af6 100644
--- a/public/worknotes/docs/useful/index.html
+++ b/public/worknotes/docs/useful/index.html
@@ -14,7 +14,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/docs/useful/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/docs/useful/z-curve-spatial-hashing/index.html b/public/worknotes/docs/useful/z-curve-spatial-hashing/index.html
index 3c229cd..eee55a5 100644
--- a/public/worknotes/docs/useful/z-curve-spatial-hashing/index.html
+++ b/public/worknotes/docs/useful/z-curve-spatial-hashing/index.html
@@ -16,7 +16,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <!--
diff --git a/public/worknotes/en.search-data.min.bfd72454f23cce5f215a08402e127f32c4fb44ac8cdb9e40d8d0c733fd85a1b6.json b/public/worknotes/en.search-data.min.bfd72454f23cce5f215a08402e127f32c4fb44ac8cdb9e40d8d0c733fd85a1b6.json
new file mode 100644
index 0000000..6171915
--- /dev/null
+++ b/public/worknotes/en.search-data.min.bfd72454f23cce5f215a08402e127f32c4fb44ac8cdb9e40d8d0c733fd85a1b6.json
@@ -0,0 +1 @@
+[{"id":0,"href":"/worknotes/docs/f-cpp/","title":"A Basic Comparison of C++ vs Fortran","section":"Docs","content":" A Basic Comparison of C++ vs Fortran # I\u0026rsquo;m an avid user of Fortran and this is pretty well known in my team. I\u0026rsquo;m not particularly evangalistic about using Fortran, but I do feel it has its place in modern programming, despite the fact that it\u0026rsquo;s one of the oldest programming languages out there. This has kind of been echoed by other Fortran users. For example, in Matsuoka et al. (2023), They declare that \u0026ldquo;Fortran is dead, long live the DSL!\u0026rdquo; is a HPC myth. They go on to explain that the high performance of Fortran is not easily reproduced even in languages like C.\nThe first author, Satoshi Matsuoka, is pretty important in the HPC world currently. He is heavily involved in the HPC community as a researcher as well as leading the development of multiple supercomputers. One of these supercomputers, Fugaku, took the first spot of IO500 upon debut, which notably chose to use custom-built CPUs Sato et al. (2022) based on the applications that they intended to run. So, while Matsuoka et al. (2023) is not particularly scientific in how they declare their myths, what it says does carry weight due to the author\u0026rsquo;s background and experience.\nMy colleague was (and still is) pretty skeptical about the performance advantages of Fortran over other languages (in particular C++). I hadn\u0026rsquo;t been able to erase his skepticism due to my naivety on the subject. I had \u0026ldquo;grown up with\u0026rdquo; (in a programming sense) Fortran, and had only played with C++. So I figured, that if I was to continue to use Fortran, I might as well be aware of how it compares to other languages.\nHere, I use the \u0026ldquo;direct search\u0026rdquo; method of finding pairs of points that lay within a certain distance of eachother. I use pair search algorithms within the context of SPH, where each of these pairs have a kernel weight associated with it. So the algorithm shown here will also calculating this value for each pair.\nThe C++ code I\u0026rsquo;m writing here is supposed to be from the perspective of a beginner. Firstly because I am a beginner C++ coder, but also because this exercise is supposed to help beginners decide why they want to choose Fortran over C++ when starting a new project. This hypothetical beginners knowledge of C++ will come from the Learn C++ website, as it is often recommended for beginners.\nHardware # Everything on this page is run on the following platform:\nAn Inspiron 7501 with i7-10750H CPU 24GB (16GB + 8GB) @ 2933MHz RAM Windows 10 WSL2 Ubuntu 20.04 The g++/gfortran compilers are used here (version 9.4.0).\nTesting scheme # Each version of the code is compiled multiple times with different flags for optimisations:\nno flags i.e., default optimisation -O0 (no optimisation) -O3 (aggressive optimisations) -fstrict-aliasing (strict aliasing - only for C++ code) the strict aliasing optimisation was added because variables in Fortran are strictly \u0026ldquo;aliased\u0026rdquo;, whereas this is not the case for C++. This apparently allows for optimisations to be made when compiling all Fortran code. See thisstackoverflow discussion for more details.\nThe Fortran code # Without going into too much detail, the dsearch subroutine is really just a nested loop that loops over the points twice to find which are within the cutoff distance (scale_k*hsml) of eachother. The loop is written to take advantage of symmetry. Consequently, the final list of pairs (pair_i, and pair_j), are unique i.e., given a pair (i, j), the reversed pair (j, i) doesn\u0026rsquo;t exist.\nNote that the points\u0026rsquo; positions are generated randomly rather than deterministicly, so that data accesses are random.\nmodule kernel_and_consts use iso_fortran_env implicit none ! constants integer, parameter:: dims = 3 real(real64), parameter:: scale_k = 2._real64 real(real64), parameter:: pi = 3.141592653589793_real64 contains ! -------------------------------------------------------------------------- real(real64) function system_clock_timer() ! higher-precision timing function than Fortran\u0026#39;s inbuilt CPU_TIME use, intrinsic:: ISO_FORTRAN_ENV, only: int64 integer(int64):: c, cr call SYSTEM_CLOCK(c, cr) system_clock_timer = dble(c)/dble(cr) end function system_clock_timer ! -------------------------------------------------------------------------- real(real64) function kernel(r, hsml) ! scalar function that calculates the kernel weight given a distance real(real64), intent(in):: r, hsml real(real64):: q, factor factor = 21._real64/(256._real64*pi*hsml*hsml*hsml) kernel = factor*dim(scale_k, q)**4*(2._real64*q + 1._real64) end function kernel ! -------------------------------------------------------------------------- subroutine dsearch(x, ntotal, hsml, pair_i, pair_j, w, niac) ! the direct pair search algorithm real(real64), intent(in):: x(dims, ntotal), hsml integer, intent(in):: ntotal integer, intent(out):: pair_i(:), pair_j(:), niac real(real64), intent(out):: w(:) integer:: i, j real(real64):: r niac = 0 do i = 1, ntotal-1 do j = i+1, ntotal r = sum((x(:, i)-x(:, j))**2) if (r \u0026lt; hsml*hsml*scale_k*scale_k) then r = sqrt(r) niac = niac + 1 pair_i(niac) = i pair_j(niac) = j w(niac) = kernel(r, hsml) end if end do end do end subroutine dsearch end module kernel_and_consts ! ------------------------------------------------------------------------------ program main use kernel_and_consts implicit none integer, parameter:: nx(4) = [10, 20, 30, 40] integer:: ntotal, maxinter, i, j, n, niac real(real64):: dx, hsml, tstart, tstop real(real64), allocatable:: x(:, :), w(:) integer, allocatable:: pair_i(:), pair_j(:) do n = 1, 4 ! define problem size ntotal = nx(n)**3 ! number of points maxinter = 150*ntotal ! maximum number of iterations dx = 1._real64/dble(nx(n)) ! average spacing between points hsml = 1.5_real64*dx ! smoothing length (an SPH term - defines cutoff) ! allocate and initalize data allocate(x(dims, ntotal), pair_i(maxinter), pair_j(maxinter), w(maxinter)) niac = 0 call random_number(x) ! begin direct search tstart = system_clock_timer() call dsearch(x, ntotal, hsml, pair_i, pair_j, w, niac) tstop = system_clock_timer() ! write times to terminal write(output_unit, \u0026#34;(A, I0)\u0026#34;) \u0026#34;Size: \u0026#34;, ntotal; write(output_unit, \u0026#34;(A, F10.6, A)\u0026#34;) \u0026#34;Time: \u0026#34;, tstop-tstart, \u0026#34;s\u0026#34;; write(output_unit, \u0026#34;(A, I0)\u0026#34;) \u0026#34;No. of pairs: \u0026#34;, niac; ! deallocate arrays to be resized deallocate(x, pair_i, pair_j, w) end do end program main Note that the definition and initialisation of the 2D array, x was contained in 3 (non-consecutive) lines in main:\nreal(real64), allocatable:: x(:, :) ... allocate(x(dims, ntotal), ...) call random_number(x) Note that the only \u0026ldquo;import\u0026rdquo; is the iso_fortran_env, which is used only for real64 (real number precision), and output_unit (writing to stdout).\nWhen compiling and executing this code, the following times are:\nno. of points no flags -O0 -O3 1000 0.00351s 0.00358s 0.00143s 8000 0.15779s 0.15726s 0.05317s 27000 1.71186s 1.75904s 0.52538s 64000 9.71352s 9.83662s 2.91456s C++ version 1: a vector of structures # A challenge that arises in C++ and scientific computing is storing multi-dimensional data. This is relevant here as we need to store the position of the points in a 2D array - one dimension for the coordinates, and one for the points. My first version of the C++ code uses a vector of structures, where the structure is just an array of length dim (in this case dim = 3). Vectors are often the recommended way to store array data. But for multi-dimensional data, there are many ways to store that in a vector e.g., a vector of vectors or a vector of tructs. Here I opt to use a vector of structs as it is a fairly common way to organise point data when the point is assocated with many values.\n#include \u0026lt;vector\u0026gt; #include \u0026lt;algorithm\u0026gt; #include \u0026lt;cmath\u0026gt; #include \u0026lt;random\u0026gt; #include \u0026lt;iostream\u0026gt; #include \u0026lt;chrono\u0026gt; using namespace std; constexpr int dim = 3; constexpr double scale_k = 2.; constexpr double pi = 3.141592653589793; // declaring position struct which a vector will be made from struct position { double x[dim]; }; double kernel(const double r, const double hsml) { const double q = r/hsml; const double max02q = max(0., scale_k-q); const double factor = 21./(256.*pi*hsml*hsml*hsml); return factor*max02q*max02q*max02q*max02q*(2.*q + 1.); } void dsearch(const vector\u0026lt;position\u0026gt;\u0026amp; pos, const int ntotal, const double hsml, vector\u0026lt;int\u0026gt;\u0026amp; pair_i, vector\u0026lt;int\u0026gt;\u0026amp; pair_j, vector\u0026lt;double\u0026gt;\u0026amp; w, int\u0026amp; niac) { niac = 0; double r, dx; for (int i = 0; i \u0026lt; ntotal-1; ++i) { for (int j = i+1; j \u0026lt; ntotal; ++j) { r = 0.; for (int d = 0; d \u0026lt; dim; ++d) { dx = pos[i].x[d]-pos[j].x[d]; r += dx*dx; } if (r \u0026lt; scale_k*scale_k*hsml*hsml) { pair_i[niac] = i; pair_j[niac] = j; r = sqrt(r); w[niac] = kernel(r, hsml); niac++; } } } } int main() { const double from = 0.; const double to = 1.; random_device dev; mt19937 generator(dev()); uniform_real_distribution\u0026lt;double\u0026gt; distr(from, to); constexpr int nx[4] {10, 20, 30, 40}; int ntotal, maxinter; double dx, hsml; for (int n = 0; n \u0026lt; 4; ++n) { ntotal = nx[n]*nx[n]*nx[n]; maxinter = ntotal*150; dx = 1./static_cast\u0026lt;double\u0026gt;(nx[n]); hsml = 1.5*dx; // declaring all vectors vector\u0026lt;position\u0026gt; pos(ntotal); vector\u0026lt;int\u0026gt; pair_i(maxinter), pair_j(maxinter); vector\u0026lt;double\u0026gt; w(maxinter); int niac = 0; for (int i = 0; i \u0026lt; ntotal; ++i) { for (int d = 0; d \u0026lt; dim; ++d) { pos[i].x[d] = distr(generator); } } auto start = chrono::high_resolution_clock::now(); dsearch(pos, ntotal, hsml, pair_i, pair_j, w, niac); auto stop = chrono::high_resolution_clock::now(); double time = static_cast\u0026lt;double\u0026gt; (chrono::duration_cast\u0026lt;chrono::microseconds\u0026gt;(stop-start).count())/1000000.; cout \u0026lt;\u0026lt; \u0026#34;Size: \u0026#34; \u0026lt;\u0026lt; ntotal \u0026lt;\u0026lt; \u0026#34;\\n\u0026#34;; cout \u0026lt;\u0026lt; \u0026#34;Time: \u0026#34; \u0026lt;\u0026lt; time \u0026lt;\u0026lt; \u0026#34;s\\n\u0026#34;; cout \u0026lt;\u0026lt; \u0026#34;No. of pairs: \u0026#34; \u0026lt;\u0026lt; niac \u0026lt;\u0026lt; \u0026#34;\\n\u0026#34;; } return 0; } Some comments about this code:\nThe structure of the code is kept similar to the Fortran code for comparison purposes. There are many more \u0026ldquo;imports\u0026rdquo; needed than the Fortran code. algorithm is needed for max, cmath is needed for sqrt, random is needed for the random number generator, iostream is needed for IO, chrono is needed for timing, and vector is needed for the vector datatype. I would argue that this is a bit of a barrier to productivity for beginners as many functions and types are located in disaparate headers that one needs to lookup or remember. The dim function isn\u0026rsquo;t available in the algorithm header. The random number generator is more verbose to setup and use compared to the random_number subroutine in Fortran. The absence of a sum function that works on arrays requires the manual writing of the calculation of r (inter-point) distance. the pow function can be slow under default/no optimisations (see this discussion on stackeroverflow), so the quartic power was manually repeated. The chrono high precision timers are available without the need to program your own (unlike in the Fortran code). When compiling and executing this code, the following times are:\nno. of points no flags -O0 -O3 -fstrict-aliasing 1000 0.009426s 0.008331s 0.001344s 0.009052s 8000 0.493711s 0.51553s 0.06409s 0.528323s 27000 5.74999s 6.02984s 0.646653 6.14586s 64000 32.4458s 33.8446s 3.45058s 34.3546s We can see that all but the aggressively optimised version of this code are disgustingly - over 3x the time taken for the default optimisations Fortran version! However, the aggressive compiler optimisations bring doown the run time significantly - but still slower than the Fortran version.\nC++ version 2: manual loop unrolling # Let\u0026rsquo;s look at the loop that calculates the square of the inter-point distance:\nr = 0.; for (int d = 0; d \u0026lt; dim; ++d) { dx = pos[i].x[d]-pos[j].x[d]; r += dx*dx; } We can manually unroll this loop, since we know that dim = 3:\nr = (pos[i].x[0]-pos[j].x[0])*(pos[i].x[0]-pos[j].x[0]) + (pos[i].x[1]-pos[j].x[1])*(pos[i].x[1]-pos[j].x[1]) + (pos[i].x[2]-pos[j].x[2])*(pos[i].x[2]-pos[j].x[2]); Compiling this and running it we get:\nno. of points no flags -O0 -O3 -fstrict-aliasing 1000 0.017024s 0.015104s 0.001302s 0.013916s 8000 0.7424s 0.793698s 0.065268s 0.805783s 27000 8.92672s 9.0975s 0.590568s 9.22796s 64000 50.4143s 51.6696s 3.20469s 52.8981s And the aggressive optimisation version reduces in run time by about 10%, at the expense of all the other versions almost doubling! The code has also lost some flexibility in that the calculation of r no longer updates as we change dim.\nC++ version 3a: 2D (partially) dynamic arrays # Arrays are also an option in C++, and in some cases, like when dealing with matrices, 2D arrays might be more intuitive.\nI make use of one of the ways described in Learn C++\u0026rsquo;s tutorials.\nTo make this change, we change:\nstruct position { double x[dim]; }; ... vector\u0026lt;position\u0026gt; pos(ntotal); vector\u0026lt;int\u0026gt; pair_i(maxinter), pair_j(maxinter); vector\u0026lt;double\u0026gt; w(maxinter); to\nauto x = new double [ntotal][dim]; int* pair_i = new int [maxinter]; int* pair_j = new int [maxinter]; We also have to update the notation form pos[i].x[d] to x[i][d] as appropriate. The arrays now have to be manually deleted, so I add the the following code to just before the loop through the problem sizes ends:\ndelete[] x; delete pair_i; delete pair_j; delete w; which are now mandatory to avoid memory leaks. Finally, we have to change the dsearch function declaration from\nvoid dsearch(const vector\u0026lt;position\u0026gt;\u0026amp; pos, const int ntotal, const double hsml, vector\u0026lt;int\u0026gt;\u0026amp; pair_i, vector\u0026lt;int\u0026gt;\u0026amp; pair_j, vector\u0026lt;double\u0026gt;\u0026amp; w, int\u0026amp; niac) { to\nvoid dsearch(const double x[][dim], const int ntotal, const double hsml, int* pair_i, int* pair_j, double* w, int\u0026amp; niac) { I should note that the Learn C++ tutorial doesn\u0026rsquo;t explain how to pass the 2D array to a function, and I eventually had to Google this to figure out that the 2D array would be passed as double x[][dim].\nI should also note that this declaration only works because dim is a constexpr. One of the less simple approaches in the Learn C++ tutorial would have to be used if dim wasn\u0026rsquo;t constant. Fortran wouldn\u0026rsquo;t experience this problem.\nCompiling this and running, we get:\nno. of points no flags -O0 -O3 -fstrict-aliasing 1000 0.005682s 0.006304s 0.001652s 0.005846s 8000 0.304066s 0.314797s 0.046719s 0.322617s 27000 3.31205s 3.53902s 0.491014s 3.55314s 64000 18.8977s 19.5868s 2.81262s 19.5882s Which shows the aggressively optimised program is now faster than the Fortran version! However, the other versions are still much slower. I think -O3 optimisation is safe for most applications, but this result is still meaningful as the default optimisations are much safer and easier to debug.\nC++ version 3b: 2D (fully) dynamic arrays # In the current case, the dim variable is a constexpr, so x can be allocated with auto x = new double [ntotal][dim];. In the case where dim is a run-time variable, then the declaration changes to:\ndouble** x = new double*[ntotal]; for (int i = 0; i \u0026lt; ntotal; ++i) { x[i] = new double[dim]; } and the corresponding deallocation becomes:\nfor (int i = 0; i \u0026lt; ntotal; ++i) { delete[] x[i]; } delete[] x; Both the allocation and deallocation statements are pretty gross and unintuitive for beginners. Furthermore, are mandatory to avoid memory leaks. The definition of x in the dsearch declaration does become simpler however:\nvoid dsearch(double** x, const int ntotal, const double hsml, int* pair_i, int* pair_j, double* w, int\u0026amp; niac) { Compiling and running:\nno. of points no flags -O0 -O3 -fstrict-aliasing 1000 0.006434s 0.005638s 0.001538s 0.005885s 8000 0.295594s 0.32687s 0.057452s 0.299215s 27000 3.29056s 3.3863s 0.58139s 3.31922s 64000 18.6892s 18.9621s 3.03032s 19.3092s And the speed of the aggresively optimised version has now dropped, although the less-optimised versions appear to have gotten faster than version 3a.\nC++ version 4: contiguous 2D (partially) dynamic arrays # If you do some digging around, you may eventually find out that allocating 2D arrays like in version 3, does not guarantee that data will be contiguous when it comes to C++. Consequently, you can change the declaration of the x 2D array to\ndouble** x = new double*[ntotal]; double* xdata = new double[ntotal*dim]; for (int i = 0; i \u0026lt; ntotal; ++i, xdata += dim) x[i] = xdata; which is honestly pretty gross, but is necessary to ensure that the data is contiguous in memory. The corresponding freeing of this array is done by\ndelete x[0]; delete x; The definition of x in the dsearch declaration remains the same as version 3b.\nI clear disadvantage beeing the special definitions and deletions that would require some wrapping. It\u0026rsquo;s also not very beginner friendly from a conceptual standpoint.\nNow compiling and running:\nno. of points no flags -O0 -O3 -fstrict-aliasing 1000 0.005687s 0.006379s 0.001573s 0.006154s 8000 0.294367s 0.305595s 0.052221s 0.316611s 27000 3.17132s 3.38158s 0.537241s 3.44193s 64000 18.5441s 19.2792s 2.84747s 19.533s The aggresively optimised version is now about as fast as the Fortran version, which is faster than version 3b, but slower than 3a.\nAggresive compiler optimisations with strict aliasing # So far, the -fstrict-aliasing flag doesn\u0026rsquo;t seem to make a difference when combined with the default optimisations. How about combining it with -O3?\nno. of points -O3 -fstrict-aliasing 1000 0.00136s 8000 0.052659s 27000 0.532555s 64000 2.8715s Combining the -fstrict-aliasing with -O3 doesn\u0026rsquo;t seem to make a discernable difference, although it seems to slow down version 4 by a trivial amount - consistent with the other versions.\nDiscussion # It\u0026rsquo;s clear that C++ can be just as fast as Fortran. However, I would argue that the speed wasn\u0026rsquo;t as easy to arrive to as what it was with Fortran. The Fortran code is written in a way not very different from how a beginner would write it and using only intrinsics. Furthermore the only import was iso_fortran_env which wasn\u0026rsquo;t mandatory (real(real64) declerations could be replaced with double precision, _real64 suffixes could be replaced with d0, and output_unit could be replaced with *); whereas all the headers for the C++ versions were mandatory. The rather subtle optimisation of the calculation of r reduced the C++ code\u0026rsquo;s flexibility, but was necessary to bring down the speed to the level that GNU\u0026rsquo;s Fortran compiler\u0026rsquo;s sum function is at. The declaration of either partial or fully dynamic multi-dimensional arrays is unintuitive, lengthy (wait until you see even higher dimensions!), and requires caution; whereas Fortran\u0026rsquo;s allocation is easy to read and easy to write for higher dimensions. Finally, the important improvement of the code from using non-contiguous dynamic arrays to contiguous is not easily found or quick to understand by a beginner (I have the advantage of years of experience coding).\nHowever, I should mention that version 3a of the C++ code compiled with the aggressive optimisations was an unexepected and interesting result. I hypothesise that when x is declared in such a way, that the compiler forces the data to be contiguous. I think further optimisations are made by leveraging the fact that the inner dimension (or the column) is constant in size. It would be interesting to see if I can also leverage this in Fortran.\nBut before closing I should mention that this experiment handled a rather uncommon case where nearly all the code being written cannot leverage the plethora of C/C++ libraries available. I think the difference in how the dsearch function was timed is a good illustration of a key difference between Fortran and C++. And that is that C++ has a lot of packages (in the form of libraries and header files) that have a lot of high-level functionality that you can utilise, whereas this is not the case for Fortran. With Fortran, you will often need to write your own subrouties to achieve certain objectives, like how I needed to write my own high-precision timer function when the C++ code could leveragethe chrono STL headers. But I should also add that it\u0026rsquo;s not difficult to call C++ code from Fortran, so there is potential to mix-and-match (see this relatively simple example).\nConclusion # Fortran was designed for maths (mainly calculus and linear algebra), and so its syntax favours those applications - particularly in the use of arrays. It also has a pretty good collection of mathematical intrinsic functions that you don\u0026rsquo;t have to look to hard for. Consequently, if your problem can be expressed in multi-dimensional arrays (or tensors), then it will much easier to write fast code.\nThis simple expeirment demonstrates that C++ can definitly achieve the same performance for applications that Fortran was designed for. But, it\u0026rsquo;s a general purpose language, so it takes a fair bit of work to apply it to the subset of problems that Fortran targets. For example, strict aliasing and contiguous data in memory are required in Fortran, whereas there is a fair bit of effort needed to set that up in C++ when it comes ot multi-dimensional arrays.\nSo, I think I\u0026rsquo;ll continue using Fortran, because at the end of the day, I enjoy working with numerical methods and simulations, which ultimately was what Fortran was designed for.\nOne day I\u0026rsquo;ll have a look at how the new kid on the block, Julia (and maybe Rust?), compares!\n"},{"id":1,"href":"/worknotes/docs/cuda/cuda-wsl/","title":"CUDA + NVHPC on WSL","section":"CUDA Programming","content":" Installing CUDA + NVHPC on WSL # This page describes the steps to setup CUDA and NVHPC within the WSL2 container (Windows 10) - avoiding the need for dual-boot or a separate Linux PC. Note that WSL2 must not have been installed when beginning these steps.\nInstall the latest Windows CUDA graphics driver Install WSL2 open PowerShell as administrator Make sure to update WSL kernel to latest version wsl \u0026ndash;update if accidentally rolled back, follow the instructions here then wsl \u0026ndash;update again check which Linux flavours are available with wsl \u0026ndash;list \u0026ndash;online install the desired flavour by wsl \u0026ndash;install -d start WSL with wsl, or opening the WSL application from the Windows search bar sudo apt update \u0026amp;\u0026amp; sudo apt upgrade -y Close and restart WSL sudo apt update \u0026amp;\u0026amp; sudo apt upgrade -y \u0026amp;\u0026amp; sudo apt autoremove -y Install CUDA for WSL check which CUDA version is compatible with the desired version NVHPC kit here select the correct CUDA version here Select the right setup: Linux -\u0026gt; x86_64 -\u0026gt; WSL-Ubuntu -\u0026gt; 2.0 -\u0026gt; dev (local) before beginning install, delete old GPG key sudo apt-key del 7fa2af80 Perform the install with code below: wget https://developer.download.nvidia.com/compute/cuda/repos/wsl-ubuntu/x86_64/cuda-wsl-ubuntu.pin sudo mv cuda-wsl-ubuntu.pin /etc/apt/preferences.d/cuda-repository-pin-600 wget https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda-repo-wsl-ubuntu-11-7-local_11.7.1-1_amd64.deb sudo dpkg -i cuda-repo-wsl-ubuntu-11-7-local_11.7.1-1_amd64.deb sudo cp /var/cuda-repo-wsl-ubuntu-11-7-local/cuda-*-keyring.gpg /usr/share/keyrings/ sudo apt-get update sudo apt-get -y install cuda Install NVHPC Download the desired NVHPC kit tar xzvf nvhpc*.tar.gz sudo nvhpc*/install Uninstalling NVHPC and CUDA # sudo rm /opt/nvidia/hpc_sdk sudo apt-get purge -y cuda \u0026amp;\u0026amp; sudo apt-get autoremove -y sudo rm /usr/share/keyrings/cuda-*-keyring.gpg sudo dpdkg -P cuda-repo-wsl-ubuntu-*-*-local_*amd64.deb You can then perform the installation steps again for the desired NVHPC-CUDA combination\n"},{"id":2,"href":"/worknotes/docs/cuda/reference-codes/deviceQuery/","title":"deviceQuery","section":"nvfortran and nvc++ reference codes","content":" deviceQuery # Description # Function to query the properties of the NVIDIA GPUs detected on the system.\nCode (C++) # #include \u0026lt;stdio.h\u0026gt; int main() { int nDevices; cudaGetDeviceCount(\u0026amp;nDevices); if (nDevices == 0) { printf(\u0026#34;No CUDA devices found\\n\u0026#34;); } else if (nDevices == 1) { printf(\u0026#34;One CUDA device found\\n\u0026#34;); } else { printf(\u0026#34;%d CUDA devices found\\n\u0026#34;, nDevices); } // Loop over devices and print properties cudaDeviceProp prop; for (int i = 0; i \u0026lt; nDevices; ++i) { printf(\u0026#34;Device Number: %d\\n\u0026#34;, i); cudaGetDeviceProperties(\u0026amp;prop, i); // General device info printf(\u0026#34; Device Name: %s\\n\u0026#34;); printf(\u0026#34; Compute Capability: %d.%d\\n\u0026#34;, prop.major, prop.minor); printf(\u0026#34; Number of Multiprocessors: %d\\n\u0026#34;, prop.multiProcessorCount); printf(\u0026#34; Max Threads per Multiprocessor: %d\\n\u0026#34;, prop.maxThreadsPerMultiProcessor); printf(\u0026#34; Global Memory (GB): %f\\n\\n\u0026#34;, prop.totalGlobalMem/(1024.0*1024.0*1024.0)); // Execution Configuration printf(\u0026#34; Execution Configuration Limits\\n\u0026#34;); printf(\u0026#34; Max Grid Dims: %d x %d x %d\\n\u0026#34;, prop.maxGridSize[0], prop.maxGridSize[1], prop.maxGridSize[2]); printf(\u0026#34; Max Block Dims: %d x %d x %d\\n\u0026#34;, prop.maxThreadsDim[0], prop.maxThreadsDim[1], prop.maxThreadsDim[2]); printf(\u0026#34; Max Threads per Block: %d\\n\u0026#34;, prop.maxThreadsPerBlock); } return 0; } Code (Fortran) # program deviceQuery use cudafor implicit none type(cudaDeviceProp):: prop integer:: nDevices=0, i, ierr ! Number of CUDA-capable devices ierr = cudaGetDeviceCount(nDevices) if (nDevices == 0) then write(*,\u0026#34;(/, \u0026#39;No CUDA devices found\u0026#39;,/)\u0026#34;) stop else if (nDevices == 1) then write(*,\u0026#34;(/,\u0026#39;One CUDA device found\u0026#39;,/)\u0026#34;) else write(*,\u0026#34;(/,i0,\u0026#39; CUDA devices found\u0026#39;,/)\u0026#34;) nDevices end if ! Loop over devices and print properties do i = 0, nDevices-1 write(*,\u0026#34;(\u0026#39;Device Number: \u0026#39;,i0)\u0026#34;) i ierr = cudaGetDeviceProperties(prop, i) ! General device info write(*,\u0026#34;(\u0026#39; Device Name: \u0026#39;,a)\u0026#34;) trim(prop%name) write(*,\u0026#34;(\u0026#39; Compute Capability: \u0026#39;,i0,\u0026#39;.\u0026#39;,i0)\u0026#34;) prop%major, \u0026amp; prop%minor write(*,\u0026#34;(\u0026#39; Number of Multiprocessors: \u0026#39;,i0)\u0026#34;) prop%multiProcessorCount write(*,\u0026#34;(\u0026#39; Max Threads per Multiprocessor: \u0026#39;,i0)\u0026#34;) \u0026amp; prop%maxThreadsPerMultiprocessor write(*,\u0026#34;(\u0026#39; Global Memory (GB): \u0026#39;,f9.3,/)\u0026#34;) \u0026amp; prop%totalGlobalMem/1024.**3 ! Execution Configuration write(*,\u0026#34;(\u0026#39; Execution Configuration Limits\u0026#39;)\u0026#34;) write(*,\u0026#34;(\u0026#39; Max Grid Dims: \u0026#39;,2(i0,\u0026#39; x \u0026#39;),i0)\u0026#34;) prop%maxGridSize write(*,\u0026#34;(\u0026#39; Max Block Dims: \u0026#39;,2(i0,\u0026#39; x \u0026#39;),i0)\u0026#34;) prop%maxThreadsDim write(*,\u0026#34;(\u0026#39; Max Threads per Block: \u0026#39;,i0,/)\u0026#34;) prop%maxThreadsPerBlock end do end program deviceQuery "},{"id":3,"href":"/worknotes/docs/useful/fixed-cutoff-direct-pair-search/","title":"Direct pair search","section":"Useful code snippets","content":" Point-pairs search with fixed cutoff distance (direct) # Description # I work with point pair searches through particle-based simulations (mostly SPH and DEM). The algorithm here is the most basic way to perform a pair search. It is O(N2) time, so is not useful for any practical applications.\nI use it frequently to server as a reference when investigating other ways to search for pairs. It\u0026rsquo;s simple to code, so is harder to introduce conceptual and coding errors.\nIt does also have some real-world relevance, as the cell-list pair-search algorithm uses many smaller direct searches.\nThe Fortran code below contains the module dsearch_m, and a main program. To compile only the module, pass the -DNOMAIN option to the compiler, and the preprocessor will omit it.\nThe module has the dsearch and dsearch_compact interfaces. dsearch returns two lists, one with the \u0026ldquo;left-sided\u0026rdquo; points of the pairs, and pair_j, which returns the \u0026ldquo;right-sided\u0026rdquo; points in the pair list. In contrast, dsearch_compact returns a shorter list, endpos, where endpos(i) corresponds to the last pair with point i as the \u0026ldquo;left-sided\u0026rdquo; point. See the main program for an example of how to iterate over the lists.\nThe benefit of dsearch_compact is the output format has a smaller memory footprint, and iterating over the list requires fewer memory accesses and is consequently usually a bit faster. However, the drawback is that it is less intuitive and more awkward to iterate over.\nCode (Fortran) # ! Module to perform direct search for pairs of points within a fixed cutoff. ! dsearch subroutine ! dim: An integer defining dimension of the coordinates. ! npoints: An integer with number of points. ! x: A real/double precision array of dim x points dimension containing the list of points to find pairs of. ! cutoff: The distance (same type as x) which defines a pair ! maxnpair: An integer of the maximum number of pairs expected. ! npairs: An integer with the number of pairs found. ! pair_i: An integer array of maxnpairs length with the \u0026#34;left-sided\u0026#34; point in a pair. ! pair_j: An integer array of maxnpairs length with the \u0026#34;right-sided\u0026#34; point in a pair. ! dsearch_compact subroutine ! same as above, except except pair_i is replaced with endpos: a list of endlocations of the left-sided point in a ! pair. The main program shows an example of how to iterate through along the list. module dsearch_m public #ifdef SINGLEPRECISION parameter, integer:: f = kind(1.) #else parameter, integer:: f = kind(1.d0) #endif contains !--------------------------------------------------------------------------- subroutine dsearch_dp(dim, npoints, x, cutoff, maxnpair, npairs, pair_i, pair_j) ! subroutine to find pairs of points using direct search. Output is ! two lists describing which points are in the pair implicit none integer, intent(in):: dim, npoints, maxnpair real(f), intent(in):: x(dim, npoints), cutoff integer, intent(out):: npairs, pair_i(maxnpair), pair_j(maxnpair) integer:: i, j real(f):: xi(dim), r2 npairs = 0 do i = 1, npoints - 1 xi(:) = x(:, i) do j = i + 1, npoints r2 = sum((xi(:) - x(:, j))**2) if (r2 \u0026lt;= cutoff**2) then npairs = npairs + 1 pair_i(npairs) = i pair_j(npairs) = j end if end do end do end subroutine dsearch_dp !--------------------------------------------------------------------------- subroutine dsearch_compact_dp(dim, npoints, x, cutoff, maxnpair, npairs, endpos, pair_j) ! subroutine to find pairs using direct search. Output is two lists. One ! list describes the last pair with point `i` as the \u0026#34;left-sided\u0026#34; point. implicit none integer, intent(in):: dim, npoints, maxnpair real(f), intent(in):: x(dim, npoints), cutoff integer, intent(out):: endpos(npoints), npairs, pair_j(maxnpair) integer:: i, j real(f):: xi(dim), r2 npairs = 0 do i = 1, npoints xi(:) = x(:, i) do j = i + 1, npoints r2 = sum((xi(:) - x(:, j))**2) if (r2 \u0026lt;= cutoff**2) then npairs = npairs + 1 pair_j(npairs) = j end if end do ! update end position when done with point i endpos(i) = npairs end do end subroutine dsearch_compact_dp end module dsearch_m #ifndef NOMAIN program main use dsearch_m, only: f, dsearch, dsearch_compact implicit none integer, parameter:: n = 100, dim = 3, maxnpair = 60*n ! estimated using ! 2x the coaxial spacing if the points were arranged in a square real(f), parameter:: cutoff = 2*n**(-1.d0/dim) real(f):: x(dim, n) integer:: pair_i(maxnpair), pair_j(maxnpair), npairs, startpos, i, j, k, endpos(n) ! initialize positions with pseudo-random numbers call random_number(x) ! finding pairs call dsearch(dim, n, x, cutoff, maxnpair, npairs, pair_i, pair_j) write (*, \u0026#39;(A)\u0026#39;) \u0026#39;Executing \u0026#34;normal\u0026#34; dsearch\u0026#39; write (*, \u0026#39;(2x, A,I4,A)\u0026#39;) \u0026#39;Found \u0026#39;, n, \u0026#39; pairs\u0026#39; write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;First and last 5 pairs of points found:\u0026#39; write (*, \u0026#39;(2x, 4(A4, 1x))\u0026#39;) \u0026#39;Pair\u0026#39;, \u0026#39;i\u0026#39;, \u0026#39;j\u0026#39; do k = 1, npairs if (k \u0026lt;= 5 .or. k \u0026gt; npairs - 4) write (*, \u0026#39;(2x, 3(I4, 1x))\u0026#39;) k, pair_i(k), pair_j(k) if (k == 6) write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;...\u0026#39; end do write (*, *) write (*, \u0026#39;(A)\u0026#39;) \u0026#39;Executing \u0026#34;compact\u0026#34; dsearch\u0026#39; write (*, \u0026#39;(2x, A,I4,A)\u0026#39;) \u0026#39;Found \u0026#39;, n, \u0026#39; pairs\u0026#39; write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;First and last 5 pairs of points found:\u0026#39; write (*, \u0026#39;(2x, 4(A4, 1x))\u0026#39;) \u0026#39;Pair\u0026#39;, \u0026#39;i\u0026#39;, \u0026#39;j\u0026#39; call dsearch_compact(dim, n, x, cutoff, maxnpair, npairs, endpos, pair_j) do i = 1, n if (i == 1) startpos = 1 if (i \u0026gt; 1) startpos = endpos(i - 1) + 1 do k = startpos, endpos(i) if (k \u0026lt;= 5 .or. k \u0026gt; npairs - 4) write (*, \u0026#39;(2x, 3(I4,1x))\u0026#39;) k, i, pair_j(k) if (k == 6) write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;...\u0026#39; end do end do end program main #endif "},{"id":4,"href":"/worknotes/docs/manual-vectorization/motivation/","title":"Motivation","section":"Investigating Manual Vectorization for SPH","content":"SPH is a continuum particle method that is often used for simulations. Typically, the most time consuming part of codes that aim to perform SPH simulations, is finding the pairs of SPH particles that are within a fixed-cutoff of each other (the pair-search step from herein), and calculating the contribution to particles\u0026rsquo; motion, due to its corresponding pair (the force calculation sweep step from herein). These steps can be combined together when organising the code, but it\u0026rsquo;s useful to seperate them when needing to re-use the pair list.\nThe pattern that the force calculation sweep looks like, can be illustrated with code (in C++) to calculate the rate-of-change of density due to the continuity equation (drho/dt = div(v)):\nfor (int k = 0; k \u0026lt; num_pairs; ++k) { i = pair_i[k]; j = pair_j[k]; dvx = vx[i] - vx[j]; dvy = vy[i] - vy[j]; dvz = vz[i] - vz[j]; vcc = mass*(dvx*dwdx[k] + dvy*dwdy[k] + dvz*dwdz[k]); drhodt[i] += vcc; drhodt[j] += vcc; } where v_ stores the velocity of each particle in x, y, and z directions; drho_dt stores the rate of change of density; and dwd_x stores the spatial gradient of the kernel function in the x, y, and z directions.\nThis pattern is not automatically vectorized by compilers, due to the non-sequential memory access pattern in the algorithm. Consequently, I wanted to invetigate manually vectorizing this loop, to see if I could reduce run times.\nx86 CPUs have a long list of assembly instructions that can be accessed in C/C++ via Intel\u0026rsquo;s SIMD Intrinsics.\nPlatform # Everything on this page is run on the following platform:\nCPU: Xeon Gold 6342 CPU (Icelake) OS: CentOS 7 Compiler: g++ v10.3.0 Code is benchmarked with the Google microbenchmark framework\n"},{"id":5,"href":"/worknotes/docs/cfdem/cavitycfdem/","title":"OpenFOAM cavity case to CFDEM","section":"CFDEM","content":" Converting the OpenFOAM cavity example for CFDEM # The cavity case is a good one to start with as it forms the beginning of the official OpenFOAM tutorials. These steps assume we\u0026rsquo;re using the PUBLIC version of CFDEM which couples LIGGGHTS-PUBLIC 3.8.0 and OpenFOAM-5.x. It also assumes that your environment variables have already been setup as per the CFDEM insallation instructions.\nGetting the case files # The lid-driven cavity flow example case comes with the OpenFOAM source code. If you don\u0026rsquo;t have it already from install OpenFOAM, get it by:\ngit clone https://github.com/OpenFOAM/OpenFOAM-5.x.git and the cavity tutorial files are located in OpenFOAM-5.x/tutorials/incompressible/icoFoam/cavity/cavity.\n1. Set up the directory structure # In the directory of your choice, setup the directory structure. You will need a \u0026ldquo;DEM\u0026rdquo; folder and a \u0026ldquo;CFD\u0026rdquo; folder. Initialize the CFD folder with the files from the cavity OpenFOAM example.\n# create a cavity-cfdem directory to store all the case files, and the DEM # subdirectory mkdir -p cavity-cfdem/DEM # copy the cavity case into the CFD folder cp -r OpenFOAM-5.x/tutorials/incompressible/icoFoam/cavity/cavity cavity-cfdem/CFD # move into the case folder cd cavity-cfdem 2. Copy a CFDEM executor wrapper script and modify # From the CFDEM tutorial files, get a parCFDDEMrun.sh script.\ncp \u0026#34;$CFDEM_PROJECT_DIR\u0026#34;/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/{Allrun.sh,parCFDDEMrun.sh} . The Allrun.sh script is fine as-is, but we need to modify the parCFDDEMrun.sh script. From the existing file, change the corresponding variables to match what\u0026rsquo;s below:\nheaderTest=cfdemSolverIB_cavity_CFDEM runOctave=\u0026#34;false\u0026#34; You may also want to set the nrProcs variable to suit your computer. The case we will be building is also quite small, so I will choose nrProcs=2. The remaining variables can remain the same. Note that we will still be using the \u0026ldquo;Immersed Boundary\u0026rdquo; CFDEM solver.\n3. Update OpenFOAM mesh # 3a. Make the mesh finer # The cavity example case uses a 2D grid of cells which posesses 20 cells in both directions. This is a little course, so we will double the number of cells in each direction. Do this by changing the blocks dictionary to\nblocks ( hex (0 1 2 3 4 5 6 7) (40 40 1) simpleGrading (1 1 1) ); Everything else can remain the same.\n3b. Add the mesh decomposition dictionary # Create the CFD/system/decomposeParDict file with the contents below:\n/*--------------------------------*- C++ -*----------------------------------*\\ | o | | | o o | HELYX-OS | | o O o | Version: v2.4.0 | | o o | Web: http://www.engys.com | | o | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; location system; object decomposeParDict; } numberOfSubdomains 2; method simple; simpleCoeffs { n ( 2 1 1); delta 0.001; } This ensures the decomposition is done on two processors. Modify this accordingly for the number of processors you would like to run this case on.\n4. Update initial conditions # In the original cavity example, only initial boundary velocity and pressure needs to be defined. To adapt this for CFDEM, initial boundary conditions for the variables phiIB, Us, and voidfraction need to be defined.\nphiIB # /*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 5 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class volScalarField; object phiB; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // dimensions [0 2 -1 0 0 0 0]; internalField uniform 0; boundaryField { movingWall { type zeroGradient; } fixedWalls { type zeroGradient; } frontAndBack { type empty; } } // ************************************************************************* // Us # /*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 5 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class volVectorField; object Us; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // dimensions [0 1 -1 0 0 0 0]; internalField uniform (0 0 0); boundaryField { movingWall { type zeroGradient; } fixedWalls { type zeroGradient; } frontAndBack { type empty; } } // ************************************************************************* // voidfraction # /*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 5 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class volScalarField; object voidfraction; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // dimensions [0 0 0 0 0 0 0]; internalField uniform 1; boundaryField { movingWall { type zeroGradient; } fixedWalls { type zeroGradient; } frontAndBack { type empty; } } // ************************************************************************* // 5. Update solver schemes # These are controlled by the CFD/system/fvSchemes dictionary. This will exist already, but need schemes related to the additional parameters we\u0026rsquo;ve added. Do so by modifying the file to match below:\n/*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.6 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; location \u0026#34;system\u0026#34;; object fvSchemes; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // ddtSchemes { default Euler; } gradSchemes { default Gauss linear; grad(p) Gauss linear; grad(U) Gauss linear; } divSchemes { default Gauss linear; div(phi,U) Gauss limitedLinearV 1; div(phi,k) Gauss limitedLinear 1; div(phi,epsilon) Gauss limitedLinear 1; div(phi,R) Gauss limitedLinear 1; div(R) Gauss linear; div(phi,nuTilda) Gauss limitedLinear 1; div((nuEff*dev(grad(U).T()))) Gauss linear; div(U) Gauss linear; } laplacianSchemes { default Gauss linear corrected; laplacian(nuEff,U) Gauss linear corrected; laplacian((1|A(U)),p) Gauss linear corrected; laplacian((voidfraction2|A(U)),p) Gauss linear corrected; laplacian(DkEff,k) Gauss linear corrected; laplacian(DepsilonEff,epsilon) Gauss linear corrected; laplacian(DREff,R) Gauss linear corrected; laplacian(DnuTildaEff,nuTilda) Gauss linear corrected; laplacian(phiIB) Gauss linear corrected; laplacian(U) Gauss linear corrected; } interpolationSchemes { default linear; interpolate(U) linear; } snGradSchemes { default corrected; } fluxRequired { default no; p ; } // ************************************************************************* // 6. Update fvSolution # This needs to be modified to ensure solvers are added for the turbulence and CFDEM parameters being used with the cfdemSolverIB solver.\n/*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.6 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; location \u0026#34;system\u0026#34;; object fvSolution; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // solvers { p { solver PCG; preconditioner DIC; tolerance 1e-06; relTol 0.1; } pFinal { solver PCG; preconditioner DIC; tolerance 1e-06; relTol 0; } U { solver PBiCG; preconditioner DILU; tolerance 1e-05; relTol 0; } k { solver PBiCG; preconditioner DILU; tolerance 1e-05; relTol 0; } epsilon { solver PBiCG; preconditioner DILU; tolerance 1e-05; relTol 0; } R { solver PBiCG; preconditioner DILU; tolerance 1e-05; relTol 0; } nuTilda { solver PBiCG; preconditioner DILU; tolerance 1e-05; relTol 0; } phiIB { solver PCG; preconditioner DIC; tolerance 1e-06; relTol 0; } } PISO { nCorrectors 4; nNonOrthogonalCorrectors 0; pRefCell 0; pRefValue 0; } // ************************************************************************* // 7. Add constants # transportProperties # This will already exist from the original case. Modify it to match below:\n/*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.6 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; location \u0026#34;constant\u0026#34;; object transportProperties; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // transportModel Newtonian; nu nu [ 0 2 -1 0 0 0 0 ] 0.01;//0.111426;//1.875e-03;//7.5e-03;//0.265883; CrossPowerLawCoeffs { nu0 nu0 [ 0 2 -1 0 0 0 0 ] 1e-06; nuInf nuInf [ 0 2 -1 0 0 0 0 ] 1e-06; m m [ 0 0 1 0 0 0 0 ] 1; n n [ 0 0 0 0 0 0 0 ] 1; } BirdCarreauCoeffs { nu0 nu0 [ 0 2 -1 0 0 0 0 ] 1e-06; nuInf nuInf [ 0 2 -1 0 0 0 0 ] 1e-06; k k [ 0 0 1 0 0 0 0 ] 0; n n [ 0 0 0 0 0 0 0 ] 1; } // ************************************************************************* // g # the cfdemSolverIB solver needs gravity to be described. We won\u0026rsquo;t apply a meaningful gravity here, so we are assigning a value of 0 with the file below.\n/*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.6 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class uniformDimensionedVectorField; location \u0026#34;constant\u0026#34;; object g; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // dimensions [0 1 -2 0 0 0 0]; value (0 0 0); //value (0 0 0); // ************************************************************************* // RASProperties # /*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.6 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; location \u0026#34;constant\u0026#34;; object RASProperties; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // RASModel laminar;//kEpsilon; turbulence on; printCoeffs on; // ************************************************************************* // turbulenceProperties # /*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.6 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; location \u0026#34;constant\u0026#34;; object turbulenceProperties; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // //simulationType RASModel;//OFversion24x simulationType laminar;//OFversion30x // ************************************************************************* // couplingProperties # /*---------------------------------------------------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.4 | | \\\\ / A nd | Web: http://www.openfoam.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; root \u0026#34;\u0026#34;; case \u0026#34;\u0026#34;; instance \u0026#34;\u0026#34;; local \u0026#34;\u0026#34;; class dictionary; object couplingProperties; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // //===========================================================================// // sub-models \u0026amp; settings modelType none; couplingInterval 10; depth 0; voidFractionModel IB;//bigParticle;//centre; // locateModel engineIB;//standard;// meshMotionModel noMeshMotion; dataExchangeModel twoWayMPI;//twoWayFiles; IOModel basicIO; probeModel off; averagingModel dilute; clockModel off; smoothingModel off; forceModels ( ShirgaonkarIB ArchimedesIB ); momCoupleModels ( ); //turbulenceModelType RASProperties;//LESProperties; //OFversion24x turbulenceModelType turbulenceProperties; //OFversion30x //===========================================================================// // sub-model properties ShirgaonkarIBProps { velFieldName \u0026#34;U\u0026#34;; pressureFieldName \u0026#34;p\u0026#34;; } ArchimedesIBProps { gravityFieldName \u0026#34;g\u0026#34;; voidfractionFieldName \u0026#34;voidfractionNext\u0026#34;; } twoWayFilesProps { maxNumberOfParticles 2; DEMts 0.0002; } twoWayMPIProps { maxNumberOfParticles 2; liggghtsPath \u0026#34;../DEM/in.liggghts_run\u0026#34;; } IBProps { maxCellsPerParticle 1000; alphaMin 0.30; scaleUpVol 1.0; } bigParticleProps { maxCellsPerParticle 1000; alphaMin 0.30; scaleUpVol 1.0; } centreProps { alphaMin 0.30; } dividedProps { alphaMin 0.05; scaleUpVol 1.2; } engineIBProps { treeSearch false; zSplit 8; xySplit 16; } // ************************************************************************* // dynamicMeshDict # /*--------------------------------*- C++ -*----------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.6 | | \\\\ / A nd | Web: www.OpenFOAM.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; location \u0026#34;constant\u0026#34;; object dynamicMeshDict; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // dynamicFvMesh dynamicRefineFvMesh;//staticFvMesh;// dynamicRefineFvMeshCoeffs { refineInterval 1;//refine every refineInterval timesteps field interFace; lowerRefineLevel .0001; upperRefineLevel 0.99; unrefineLevel 10; nBufferLayers 1; maxRefinement 1;//maximum refinement level (starts from 0) maxCells 1000000; correctFluxes ( (phi U) (phi_0 U) ); dumpLevel false; } // ************************************************************************* // liggghtsCommands # /*---------------------------------------------------------------------------*\\ | ========= | | | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox | | \\\\ / O peration | Version: 1.4 | | \\\\ / A nd | Web: http://www.openfoam.org | | \\\\/ M anipulation | | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; root \u0026#34;\u0026#34;; case \u0026#34;\u0026#34;; instance \u0026#34;\u0026#34;; local \u0026#34;\u0026#34;; class dictionary; object liggghtsCommands; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // liggghtsCommandModels ( runLiggghts ); // ************************************************************************* // 8. Write the LIGGGHTS input file # atom_style granular atom_modify map array communicate single vel yes boundary f f f newton off units si processors * * 1 region reg block -0.01 0.11 -.01 0.11 -.01 0.03 units box create_box 1 reg neigh_modify delay 0 binsize 0.0 # Material properties required for new pair styles fix m1 all property/global youngsModulus peratomtype 5.e7 fix m2 all property/global poissonsRatio peratomtype 0.45 fix m3 all property/global coefficientRestitution peratomtypepair 1 0.9 fix m4 all property/global coefficientFriction peratomtypepair 1 0.5 # pair style pair_style gran model hertz tangential history #Hertzian without cohesion pair_coeff * * # timestep, gravity timestep 0.0002 fix gravi all gravity 0 vector 0.0 0.0 -1.0 # walls fix xwalls1 all wall/gran model hertz tangential history primitive type 1 xplane 0. fix xwalls2 all wall/gran model hertz tangential history primitive type 1 xplane 0.1 fix ywalls1 all wall/gran model hertz tangential history primitive type 1 yplane 0. fix ywalls2 all wall/gran model hertz tangential history primitive type 1 yplane 0.1 shear x 1 fix zwalls1 all wall/gran model hertz tangential history primitive type 1 zplane 0. fix zwalls2 all wall/gran model hertz tangential history primitive type 1 zplane 0.1 # cfd coupling fix cfd all couple/cfd couple_every 10 mpi fix cfd2 all couple/cfd/force # create single partciles create_atoms 1 single .05 .025 0.01 create_atoms 1 single .05 .075 0.01 set atom 1 diameter 0.005 density 2600 vx 0 vy 0 vz 0 set atom 2 diameter 0.005 density 2600 vx 0 vy 0 vz 0 # apply nve integration to all particles that are inserted as single particles fix integr all nve/sphere #wenn das ausgeblendet, dann kein vel update # screen output compute rke all erotate/sphere thermo_style custom step atoms ke c_rke vol thermo 1000 thermo_modify lost ignore norm no compute_modify thermo_temp dynamic yes # insert the first particles so that dump is not empty dump\tdmp2 all custom/vtk 50 post/dump*.vtk id type type x y z vx vy vz radius run 1 "},{"id":6,"href":"/worknotes/docs/pi/trim-jetsonnano/","title":"Reducing Jetson Nano OS for Server","section":"Raspberry Pis","content":" Reducing Jetson Nano OS for Server # The Nvidia Jetson Nano is a Single Board Computer (SBC) with a scaled-down Nvidia GPU (Tegra X1). I have the 2GB version, the smallest available. The OS that Nvidia forces you to use comes with a full-blown desktop environment, which chews through the 2GB of RAM pretty easily - not leaving as much room as I\u0026rsquo;d like for other things.\nConsequently, this page is to document the steps to trim down the OS to save disk space and RAM - adding to steps documented elsewhere.\nAfter installing the OS, hooking up periferals, inserting the flashed SD card, and setting up the Jetson, you can run the following:\nsudo systemctl stop gdm3 sudo systemctl disable gdm3 sudo sytemctl set-default multi-user.target sudo reboot sudo apt remove --purge -y \\ ubuntu-desktop \\ # meta-package for Ubuntu desktop gnome* \\ # packages to create desktop GUI libreoffice* \\ # Microsoft office equivalent nautilus \\ # GUI file exporer thunderbird \\ # GUI email app chromium-browser # crap browser sudo apt autoremove -y # remove packages no longer needed sudo apt autoclean # removes cached package files sudo apt update sudo apt install \\ ubuntu-server \\ # meta-package for Ubuntu server firefox # a better browser sudo apt upgrade -y "},{"id":7,"href":"/worknotes/docs/pi/webserver/","title":"Setting Up Public Webserver on Raspberry Pi","section":"Raspberry Pis","content":" Setting Up Public Webserver on Raspberry Pi # The instructions here assumes you\u0026rsquo;re using Raspberry Pi Lite as the OS on the Raspberry Pi. Other OS\u0026rsquo; are largely similar though. The main difference will be the packages and package managers, and the firewall tool.\nSetup the pi to host the server\nFlash disk with raspberry pi lite. Insert disk into pi and power them on. (follow the instructions here up to step 4) Setup router to forward http/https/ssh requests to the Raspberry Pi\nObtain MAC address of Raspberry Pi e.g.,\n$ ip addr 1: lo: \u0026lt;LOOPBACK,UP,LOWER_UP\u0026gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: \u0026lt;NO-CARRIER,BROADCAST,MULTICAST,UP\u0026gt; mtu 1500 qdisc pfifo_fast state DOWN group default qlen 1000 link/ether b8:27:eb:24:cb:98 brd ff:ff:ff:ff:ff:ff inet 10.0.0.1/8 brd 10.255.255.255 scope global noprefixroute eth0 valid_lft forever preferred_lft forever inet6 fe80::1209:6b01:5af1:d12b/64 scope link tentative valid_lft forever preferred_lft forever 3: wlan0: \u0026lt;BROADCAST,MULTICAST,UP,LOWER_UP\u0026gt; mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether b8:27:eb:71:9e:cd brd ff:ff:ff:ff:ff:ff inet 192.168.0.104/24 brd 192.168.0.255 scope global dynamic noprefixroute wlan0 valid_lft 4773sec preferred_lft 3873sec inet6 fe80::9a0:b632:2b19:bb89/64 scope link valid_lft forever preferred_lft forever and the MAC address is the first address following link/ether for the interface you\u0026rsquo;re using. If the Raspberry Pi is connected to the router via WiFi, then you use the wlan address. If connected via ethernet cable, then you use the eth0 address. In the above example, I used the address b8:27:eb:71:9e:cd, because my Pi is connected to the router via Wifi. We use this MAC address to link the physical device to an IP address in the next step.\nOn the router:\nAssign a static IP address to the Pi. Instructions are readily available online for most manufacturers. To do this on my routher (TP-Link AC1200), I browsed to http://192.168.0.1, signed in, and then went to \u0026ldquo;IP \u0026amp; MAC Binding\u0026rdquo; → \u0026ldquo;Binding Settings\u0026rdquo; → \u0026ldquo;Add New\u0026rdquo; and enter the MAC address collected from above and the desired IP to bind it to. Setup port forwarding to forward packages to port 22, 80, 443, and 8443 to the IP address you\u0026rsquo;ve assigned the pi in the previous step. On my router, I navigated to \u0026ldquo;Forwarding\u0026rdquo; → \u0026ldquo;Virtual Server\u0026rdquo; → \u0026ldquo;Add New\u0026rdquo;. This step forward incoming requests matching those ports, to the Raspberry Pi. Test the setup. While on a machine connected to the router, you can test ssh and http connection from the command line with ssh pi@\u0026lt;ip-address\u0026gt; and curl http://\u0026lt;ip-address\u0026gt;. You can also check http by entering the URL into your web browser: http://\u0026lt;ip-address\u0026gt;. Note that \u0026lt;ip-address\u0026gt; is the static IP address you assigned to the Pi on the router. Sign an SSL certificate to enable HTTPS connections. This requires signing an SSL certificate. There are many paid services out there, but the free Let\u0026rsquo;s Encrypt SSL certificate signing service is the best option (because it\u0026rsquo;s free). Using the automatic cert-bot option should be fine ( instructions for Ubuntu). You can test that the signed certificate works, by navigating to the Pi web server in a browser on another device i.e., https://\u0026lt;ip-address\u0026gt; (note the \u0026ldquo;s\u0026rdquo;).\nGet your ISP to open ports 22, 80, 443, and 8443 to your house. I had to send an online request to them and they followed up with me on the phone. Some ISPs may not be blocking any ports either.\nPurchase a domain and setup resource records so that the domain points your IP address. I setup:\nHost Name\tType\tTTL\tData ed-yang.com\tA\t10 minutes\t\u0026lt;my IP\u0026gt; ed-yang.com\tCAA\t10 minutes\t0 issues \u0026quot;letsencrypt.org\u0026quot; www.ed-yang.com\tCNAME\t10 minutes\ted-yang.com Setting up a development webserver on WSL2 # If you\u0026rsquo;re on a Windows machine like I am, you can use WSL2 to test the html code. To setup WSL2, follow the instructions here.\nYou then need to setup systemd so the Apache web server package will work properly. To set it up, follow the instructions below (reproduced from these instructions in case the page disappears):\nInside the WSL2 instance, edit/create the /etc/wsl.conf file and add the following lines of code:\n[boot] systemd=true Restart WSL by exiting WSL, opening a powershell terminal as administrator, and running the command wsl --shutdown. After doing so, systemd should be running!\nYou can now go ahead and install the Apache web server packages:\nsudo apt install apache2 and then copy your html code into /var/www/html.\n"},{"id":8,"href":"/worknotes/docs/pi/slurm-cluster/","title":"Setting Up Raspberry Pi Slurm Cluster","section":"Raspberry Pis","content":" Setting Up Raspberry Pi Slurm Cluster # Flash disk(s) with raspberry pi lite. Insert disk(s) into pi(s) and power them on.\nRun sudo raspi-config\nupdate raspi-config change hostname setup ssh change password for pi user set wlan locale set timezone setup login-less ssh between nodes.\ncreate key on one of the nodes\nsudo ssh-keygen # save it somewhere central like in /etc/ssh configure ssh to use the newly created key by editing /etc/ssh/ssh_config and adding\nIdentityFile /etc/ssh/\u0026lt;key name\u0026gt; make the key readable\nchmod +r /etc/ssh/\u0026lt;key name\u0026gt; for each other node, copy ssh key to the nodes (e.g. with scp) and repeat above steps.\nupdate /etc/hosts with all hosts names. e.g.:\n10.0.0.1 pimananager 10.0.0.2 picompute1 10.0.0.3 picompute2 Setup dhcp server on manager/login node (detailed instructions)\nInstall packages required for this step:\nsudo apt install iptables dnsmasq In /etc/dhcpcd.conf, add the following:\ninterface eth0 # for the ethernet network static ip_address=10.0.0.1/8 # provide static ip of 10.0.0.1 static domain_name_servers=8.8.8.8,8.8.4.4 #??? nolink # sets up the interface without being attached to ethernet move default config file and create new one with text below: sudo mv /etc/dnsmasq.conf /etc/dnsmasq.conf.old sudo nano /etc/dnsmasq.conf interface=eth0 listen-address=10.0.0.1 dhcp-range=10.0.0.32,10.0.0.128,12h dhcp-host=\u0026lt;mac-address of nodes\u0026gt;,10.0.0.2 dhcp-host=\u0026lt;mac-address of nodes\u0026gt;,10.0.0.3 ... server=8.8.8.8 server=8.8.4.4 bind-interfaces domain-needed bogus-priv expand-hosts add sleep 10 to very start of /etc/init.d/dnsmasq. needs to be improved on\nreboot manager/login node\nfrom /etc/sysctl.conf, uncomment net.ipv4.ip_forward=1\nAdd iptables rules:\nsudo iptables -t nat -A POSTROUTING -o wlan0 -j MASQUERADE sudo iptables -A FORWARD -i wlan0 -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT sudo iptables -A FORWARD -i eth0 -o wlan0 -j ACCEPT ensure iptables rules are presistent between boots:\nsudo apt install iptables-persistent reboot switch and check DHCP leases were granted:\ncat /var/lib/misc/dnsmasq.leases Internet should be being forwarded to new nodes\nupdate existing packages on all nodes:\nsudo apt update; sudo apt upgrade -y; sudo apt update packages to install:\nsudo apt install cmake tcl tcl-dev ntpdate tmux git slurm-wlm openmpi-bin openmpi-common libopenmpi3 \\ libopenmpi-dev git tree -y turn off swapfiles (kills the sd card):\nsudo dphys-swapfile swapoff sudo dphys-swapfile uninstall sudo update-rc.d dphys-swapfile remove Setup NFS for shared filesystem:\non the node with the drive connected:\ninstall nfs server package:\nsudo apt install nfs-kernel-server -y identify partition with lsblk and corresponding UUID with blkid\nformat disk:\nsudo mkfs -t ext4 /dev/\u0026lt;partition\u0026gt; make directory drive is to be mounted on:\nsudo mkdir \u0026lt;directory\u0026gt; mount drive by sudo nano /etc/fstab add the following:\nUUID=\u0026lt;UUID\u0026gt; \u0026lt;directory to mount to\u0026gt; ext4 defaults 0 \u0026lt;next integer in the list\u0026gt; export the nfs by editing sudo nano /etc/exports/ and add the following\n\u0026lt;directory to export\u0026gt; \u0026lt;ip style\u0026gt;/\u0026lt;search format\u0026gt;(rw,sync,no_root_squash,no_subtree_check) examples:\n/clusterfs 192.168.1.0/255.255.255.0(rw,sync,no_root_squash,no_subtree_check) /clusterfs 10.0.0.0/8.8.8.0(rw,sync,no_root_squash,no_subtree_check) mount the drive and then export:\nsudo mount -a sudo exportfs -a on the nodes not connect to the drive:\ninstall nfs common package:\nsudo apt install nfs-common -y make directory drive where nfs folder is to be located. For purposes of MPI programs, make the folder name the same on all nodes\nmount drive by sudo nano /etc/fstab and add the following lines:\n\u0026lt;ip of node with drive\u0026gt;:\u0026lt;directory\u0026gt; \u0026lt;directory\u0026gt; nfs defaults 0 \u0026lt;next index\u0026gt; mount drive:\nsudo mount -a install spack on nfs and setup spack env to load for root\nClone the Spack repo\ncd /clusterfs git clone -c feature.manyFiles=true https://github.com/spack/spack.git add spack setup script to root .bashrc by sudo nano /root/.bashrc and add:\n. /share/spack/setup-env.sh (would prefer this to be setup system wide)\n"},{"id":9,"href":"/worknotes/docs/cfdem/snappyhexmesh/","title":"Snappy Hex Mesh Basics","section":"CFDEM","content":" Snappy Hex Mesh Basics # This is a summary of meshing in OpenFOAM using the snappyHexMesh tool. I\u0026rsquo;m writing this in detail because I couldn\u0026rsquo;t find any comprehensive tutorial that is beginner friendly. The ones I could find were like as if they were picking up from where someone else left off. Consequently, this tool is a beginner guide and aims only to recommend easy-to-pickup tools, rather than the most fully featured tools. This only covers absolute basics and creating simple geometries. It only covers snappyhexMesh tool as the user guide for the basic tool blockMesh is pretty ok.\nBackground # The high-level of how snappyHexMesh works can be found here.\nThe basic steps are to:\ndefine the background mesh The background mesh is where the fluid flow will be simulated. create the stl file This step is to create the object you wish to insert onto the background mesh. run surfaceFeatureExtract This requires the system/surfaceFeatureExtractDict to be created, which defines how edges and surfaces of the stl mesh are translated into a mesh. run snappyHexMesh This will use the inputs created in steps 1-3. Note that setting up for running snappyHexMesh in parallel is different from running in serial (i.e., one CPU core). Both setups will be described here. Step 1: Defining the Background Mesh # This steps assumes that you know how to generate a simple mesh using the blockMesh utility. If you don\u0026rsquo;t follow that guide first, and work through the excercises.\nThe background mesh has to satisfy a few requirements:\ncells ought to be approximately cubic within the vicinity of the stl mesh, otherwise snappyHexMesh will fail the background mesh must be positioned and sized such that the stl mesh is within it or located on the boundary Otherwise defining a mesh in a manner described on the BlockMesh guide is sufficient.\nStep 2: Creating an stl file # As noted above, the stl mesh must be created in a way such that it is located inside or on the boundary of the background mesh. In addition to that, the mesh must be structured nicely (the exact requirements I\u0026rsquo;m not sure of). When I first started out, I couldn\u0026rsquo;t find a tool that created nice meshes and was also easy to use. But, instead I found tools that can create crappy meshes, and another tool that can turn these meshes into nice meshes.\nStep 2a: Creating the basic shape # This step requires a Computer Aided Design (CAD) program. I found the following CAD programs to be basic, but free and perfectly fine for basic geometries:\nSketchUp free ( tutorial). stl files can be exported using the \u0026ldquo;download\u0026rdquo; utility. FreeCad ( tutorial series). stl files can be exported using the \u0026ldquo;export\u0026rdquo; utility. Step 2b: Converting the mesh into something nice # The meshes the above tools create are not guaranteed to be nice e.g., meshes produced by the above utilities will have cells that are far from being equilateral.\nThe tool that I found pretty easy to use and pickup is MeshLab, but I found it to be pretty capable. To make a mesh nicer for use with snappyHexMesh, the \u0026ldquo;filters\u0026rdquo; dropdown list has a few tools that are pretty handy.\nOriginal mesh produced by SketchUp of a baffle array (square columns). snappyHexMesh doesn\u0026rsquo;t like the small angles between lines at the corners\nRemeshing with MeshLab using Filters → Polygonal and Quad Mesh → Turn into Quad-Dominant mesh. Since the mesh faces are rectangular, a cell can occupy a whole face i.e., cell edges coincide with face edges.\nRemeshing with Meshlab using Filters → Remeshing, Simplification and Reconstruction → Remeshing: Isotropic Explicit Remeshing. While looking more complex, this mesh may be more preferred than the second figure above, as the triangles are closer to being equilateral.\nTo save the remeshed mesh, you go to File → Export Mesh As\u0026hellip; and choose to save it as an stl file. Save this to the project folder in constant/triSurface/ with any name you like.\nStep 3: surfaceFeatureExtract # Create the file system/surfaceFeatureExtractDict with the contents (modifying the mesh file name):\n/*--------------------------------*- C++ -*----------------------------------*\\ ========= | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\\\ / O peration | Website: https://openfoam.org \\\\ / A nd | Version: 6 \\\\/ M anipulation | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; object surfaceFeatureExtractDict; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // Untitled2.stl //change this to match your file name { // How to obtain raw features (extractFromFile || extractFromSurface) extractionMethod extractFromSurface; extractFromSurfaceCoeffs { // Mark edges whose adjacent surface normals are at an angle less // than includedAngle as features // - 0 : selects no edges // - 180: selects all edges includedAngle 120; } /* subsetFeatures { // Keep nonManifold edges (edges with \u0026gt;2 connected faces) nonManifoldEdges yes; // Keep open edges (edges with 1 connected face) openEdges yes; } */ // Write options // Write features to obj format for postprocessing writeObj yes; } // ************************************************************************* // After this is saved, run the surfaceFeatureExtract tool in the project root directory, which creates an .emesh file in constant/triSurface.\nStep 4: snappyHexMesh # Create the file system/snappyHexMeshDict with the contents of the file below, modifying it to point to the correct files. See the appendix to see the full file.\nIf it\u0026rsquo;s been setup correctly, you can then run snappyHexMesh in the project root directory to create the new mesh. This will create new time-step folders with the mesh created in steps. If you wish for the mesh to be saved into constant/polyMesh, pass the -overwrite option. The alternative is to move it there yourself.\nRunning snappyHexMesh in Parallel # The system/decomposeParDict file needs to be created. This is the same file to be used for parallel simulation. Example contents of the file (with 4 processors - change this to your desired number of parallel processes):\n/*--------------------------------*- C++ -*----------------------------------*\\ ========= | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\\\ / O peration | Website: https://openfoam.org \\\\ / A nd | Version: 6 \\\\/ M anipulation | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; object blockMeshDict; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // numberOfSubdomains 4; //modify this to suit the number of cores method scotch; // ************************************************************************* // and in the project root directory, run decomposePar which creates processor* folders. snappyHexMesh is then invoked in parallel by\nmpiexec -n 4 snappyHexMesh -parallel -overwrite The mesh can then be reconstructed with the command reconstructParMesh in the project root directory.\nAppendix # /*--------------------------------*- C++ -*----------------------------------*\\ ========= | \\\\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\\\ / O peration | Website: https://openfoam.org \\\\ / A nd | Version: 6 \\\\/ M anipulation | \\*---------------------------------------------------------------------------*/ FoamFile { version 2.0; format ascii; class dictionary; object snappyHexMeshDict; } // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // Which of the steps to run castellatedMesh true; snap true; addLayers false; // Geometry. Definition of all surfaces. All surfaces are of class // searchableSurface. // Surfaces are used // - to specify refinement for any mesh cell intersecting it // - to specify refinement for any mesh cell inside/outside/near // - to \u0026#39;snap\u0026#39; the mesh boundary to the surface geometry { Untitled2.stl // modify this { type triSurfaceMesh; name baffles; // modify this and use in below PatchInfo { type wall; } } }; // Settings for the castellatedMesh generation. castellatedMeshControls { // Refinement parameters // ~~~~~~~~~~~~~~~~~~~~~ // If local number of cells is \u0026amp;gr; maxLocalCells on any processor // switches from from refinement followed by balancing // (current method) to (weighted) balancing before refinement. maxLocalCells 100000; // Overall cell limit (approximately). Refinement will stop immediately // upon reaching this number so a refinement level might not complete. // Note that this is the number of cells before removing the part which // is not \u0026#39;visible\u0026#39; from the keepPoint. The final number of cells might // actually be a lot less. maxGlobalCells 7000000; // The surface refinement loop might spend lots of iterations refining just a // few cells. This setting will cause refinement to stop if \u0026lt;= minimumRefine // are selected for refinement. Note: it will at least do one iteration // (unless the number of cells to refine is 0) minRefinementCells 0; // Allow a certain level of imbalance during refining // (since balancing is quite expensive) // Expressed as fraction of perfect balance (= overall number of cells / // nProcs). 0=balance always. maxLoadUnbalance 0.10; // Number of buffer layers between different levels. // 1 means normal 2:1 refinement restriction, larger means slower // refinement. nCellsBetweenLevels 1; // Explicit feature edge refinement // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Specifies a level for any cell intersected by its edges. // This is a featureEdgeMesh, read from constant/triSurface for now. features ( { file \u0026#34;Untitled2.eMesh\u0026#34;; // modify this level 1; } ); // Surface based refinement // ~~~~~~~~~~~~~~~~~~~~~~~~ // Specifies two levels for every surface. The first is the minimum level, // every cell intersecting a surface gets refined up to the minimum level. // The second level is the maximum level. Cells that \u0026#39;see\u0026#39; multiple // intersections where the intersections make an // angle \u0026gt; resolveFeatureAngle get refined up to the maximum level. refinementSurfaces { baffles // modify this to match name defined earlier { // Surface-wise min and max refinement level level (2 2); } } // Resolve sharp angles resolveFeatureAngle 30; // Region-wise refinement // ~~~~~~~~~~~~~~~~~~~~~~ // Specifies refinement level for cells in relation to a surface. One of // three modes // - distance. \u0026#39;levels\u0026#39; specifies per distance to the surface the // wanted refinement level. The distances need to be specified in // descending order. // - inside. \u0026#39;levels\u0026#39; is only one entry and only the level is used. All // cells inside the surface get refined up to the level. The surface // needs to be closed for this to be possible. // - outside. Same but cells outside. refinementRegions { } // Mesh selection // ~~~~~~~~~~~~~~ // After refinement patches get added for all refinementSurfaces and // all cells intersecting the surfaces get put into these patches. The // section reachable from the locationInMesh is kept. // NOTE: This point should never be on a face, always inside a cell, even // after refinement. locationInMesh (0 0 0.1); // Whether any faceZones (as specified in the refinementSurfaces) // are only on the boundary of corresponding cellZones or also allow // free-standing zone faces. Not used if there are no faceZones. allowFreeStandingZoneFaces true; } // Settings for the snapping. snapControls { //- Number of patch smoothing iterations before finding correspondence // to surface nSmoothPatch 3; //- Relative distance for points to be attracted by surface feature point // or edge. True distance is this factor times local // maximum edge length. tolerance 4.0; //- Number of mesh displacement relaxation iterations. nSolveIter 100; //- Maximum number of snapping relaxation iterations. Should stop // before upon reaching a correct mesh. nRelaxIter 5; nFeatureSnapIter 10; } // Settings for the layer addition. addLayersControls { // Are the thickness parameters below relative to the undistorted // size of the refined cell outside layer (true) or absolute sizes (false). relativeSizes true; // Per final patch (so not geometry!) the layer information layers { baffles // modify this to match name defined earlier { nSurfaceLayers 5; } } // Expansion factor for layer mesh expansionRatio 1.1; // Wanted thickness of final added cell layer. If multiple layers // is the thickness of the layer furthest away from the wall. // Relative to undistorted size of cell outside layer. // See relativeSizes parameter. finalLayerThickness 0.8; // Minimum thickness of cell layer. If for any reason layer // cannot be above minThickness do not add layer. // Relative to undistorted size of cell outside layer. minThickness 0.1; // If points get not extruded do nGrow layers of connected faces that are // also not grown. This helps convergence of the layer addition process // close to features. nGrow 0; // Advanced settings // When not to extrude surface. 0 is flat surface, 90 is when two faces // are perpendicular featureAngle 60; // Maximum number of snapping relaxation iterations. Should stop // before upon reaching a correct mesh. nRelaxIter 3; // Number of smoothing iterations of surface normals nSmoothSurfaceNormals 1; // Number of smoothing iterations of interior mesh movement direction nSmoothNormals 3; // Smooth layer thickness over surface patches nSmoothThickness 2; // Stop layer growth on highly warped cells maxFaceThicknessRatio 0.5; // Reduce layer growth where ratio thickness to medial // distance is large maxThicknessToMedialRatio 0.3; // Angle used to pick up medial axis points minMedianAxisAngle 90; // Create buffer region for new layer terminations nBufferCellsNoExtrude 0; // Overall max number of layer addition iterations nLayerIter 50; } // Generic mesh quality settings. At any undoable phase these determine // where to undo. meshQualityControls { //- Maximum non-orthogonality allowed. Set to 180 to disable. maxNonOrtho 65; //- Max skewness allowed. Set to \u0026lt;0 to disable. maxBoundarySkewness 20; maxInternalSkewness 4; //- Max concaveness allowed. Is angle (in degrees) below which concavity // is allowed. 0 is straight face, \u0026lt;0 would be convex face. // Set to 180 to disable. maxConcave 80; //- Minimum projected area v.s. actual area. Set to -1 to disable. minFlatness 0.5; //- Minimum pyramid volume. Is absolute volume of cell pyramid. // Set to a sensible fraction of the smallest cell volume expected. // Set to very negative number (e.g. -1E30) to disable. minVol 1e-13; minTetQuality 1e-30; //- Minimum face area. Set to \u0026lt;0 to disable. minArea -1; //- Minimum face twist. Set to \u0026lt;-1 to disable. dot product of face normal // and face centre triangles normal minTwist 0.02; //- Minimum normalised cell determinant // 1 = hex, \u0026lt;= 0 = folded or flattened illegal cell minDeterminant 0.001; //- minFaceWeight (0 -\u0026gt; 0.5) minFaceWeight 0.02; //- minVolRatio (0 -\u0026gt; 1) minVolRatio 0.01; // must be \u0026gt;0 for Fluent compatibility minTriangleTwist -1; // Advanced //- Number of error distribution iterations nSmoothScale 4; //- Amount to scale back displacement at error points errorReduction 0.75; } // Advanced // Merge tolerance. Is fraction of overall bounding box of initial mesh. // Note: the write tolerance needs to be higher than this. mergeTolerance 1E-6; // ************************************************************************* // "},{"id":10,"href":"/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-search/","title":"Cell list pair search","section":"Useful code snippets","content":" Point-pairs search with fixed cutoff distance (using cell-lists) # Description # This is an improvement to the direct search algorithm for searching for pairs of points within a cutoff distance. It uses a grid of cells whose side-length is equal to the specified cutoff distance.\nThis is beneficial to performance since for any given point, all its neighbours are guaranteed to be within its own cell, or in adjacent cells. This means that instead of performing a comparison with all other points, comparisons only need to be made to points in the same or adjacent cells.\nThis results in O(N) time, assuming the grid cells or overall grid size, is resized to suit the number of points.\nThe below Fortran code is a relatively naive way to implement it. It looks like how someone trying it for the first time might attempt the implementation. Other pages will demonstrate how the algorithm can be sped up.\nThe main steps are:\nDetermine the min/max extents of the grid (based on point positions). Map points to grid. Sweep through the cells to find pairs. The interface to the code looks similar to the direct search version, with the addition of a maxpcell variable that needs to be added to size the grid array.\nCode (Fortran) # ! Module to perform direct search for pairs of points within a fixed cutoff. ! cellList subroutine ! dim: An integer defining dimension of the coordinates. ! npoints: An integer with number of points. ! maxpercell: An integer with max number of points in a cell. ! x: An real/real(f) array of dim x points dimension containing the list of points to find pairs of. ! cutoff: The distance (same type as x) which defines a pair ! maxnpair: An integer of the maximum number of pairs expected. ! npairs: An integer with the number of pairs found. ! pair_i: An integer array of maxnpairs length with the \u0026#34;left-sided\u0026#34; point in a pair. ! pair_j: An integer array of maxnpairs length with the \u0026#34;right-sided\u0026#34; point in a pair. module cellList_m use iso_fortran_env, only: error_unit public private:: coordsToCell #ifdef SINGLEPRECISION parameter, integer:: f = kind(1.) #else parameter, integer:: f = kind(1.d0) #endif contains !--------------------------------------------------------------------------- function coordsToCell(dim, xi, minx, cutoff) result(icell) ! function to convert coordinates to grid cells implicit none real(f), intent(in):: xi(dim), minx(3), cutoff integer, intent(in):: dim integer:: icell(3) icell(1:dim) = int((xi(:) - minx(1:dim))/cutoff) + 1 if (dim == 2) icell(3) = 1 end function coordsToCell !--------------------------------------------------------------------------- subroutine cellList(dim, npoints, maxpercell, x, cutoff, maxnpair, npairs, pair_i, pair_j) implicit none integer, intent(in):: dim, npoints, maxnpair, maxpercell real(f), intent(in):: x(dim, npoints), cutoff integer, intent(out):: npairs, pair_i(maxnpair), pair_j(maxnpair) integer:: i, j, k, ngridx(3), icell(3), xi, yi, zi, jcell(3) real(f):: r2, minx(3), maxx(3) integer, allocatable:: pincell(:, :, :), cells(:, :, :, :) ! determine grid min-max coordinates, with concessions made for dim=2 case minx(1:dim) = minval(x, dim=2) if (dim == 2) minx(3) = 0.d0 maxx(1:dim) = maxval(x, dim=2) if (dim == 2) maxx(3) = 0.d0 ! creating buffer layers minx(:) = minx(:) - 2.d0*cutoff maxx(:) = maxx(:) + 2.d0*cutoff ! initializing grid arrays allocate (pincell(ngridx(1), ngridx(2), ngridx(3)), \u0026amp; cells(maxpercell, ngridx(1), ngridx(2), ngridx(3))) pincell(:, :, :) = 0 ! populating grid with point information do i = 1, npoints icell = coordsToCell(dim, x(:, i), minx, cutoff) pincell(icell(1), icell(2), icell(3)) = \u0026amp; pincell(icell(1), icell(2), icell(3)) + 1 #ifdef DEBUG if (pincell(icell(1), icell(2), icell(3)) \u0026gt; maxpercell) then write (error_unit, \u0026#39;(A, 3(I2, 1x), A)\u0026#39;) \u0026#39;Number of particles in cell \u0026#39;, icell(1), icell(2), icell(3), \u0026amp; \u0026#39; exceeds the input maxpercell value.\u0026#39; write (error_unit, \u0026#39;(A)\u0026#39;) \u0026#39;Terminating...\u0026#39; error stop 1 end if #endif cells(pincell(icell(1), icell(2), icell(3)), icell(1), icell(2), icell(3)) = i end do ! perform pair search npairs = 0 do i = 1, npoints icell(:) = coordsToCell(dim, x(:, i), minx, cutoff) do xi = -1, 1 jcell(1) = icell(1) + xi do yi = -1, 1 jcell(2) = icell(2) + yi do zi = -1, 1 jcell(3) = icell(3) + zi do k = 1, pincell(jcell(1), jcell(2), jcell(3)) j = cells(k, jcell(1), jcell(2), jcell(3)) if (j \u0026gt; i) then r2 = sum((x(:, i) - x(:, j))**2) if (r2 \u0026lt;= cutoff**2) then npairs = npairs + 1 pair_i(npairs) = i pair_j(npairs) = j end if end if end do end do end do end do end do end subroutine cellList end module cellList_m #ifndef NOMAIN program main use cellList_m, only: f, cellList implicit none integer, parameter:: n = 100, dim = 3, maxnpair = 60*n ! estimated using ! 2x the coaxial spacing if the points were arranged in a square real(f), parameter:: cutoff = 2*n**(-1.d0/dim) integer:: npairs, k real(f), allocatable:: x(:, :) integer, allocatable:: pair_i(:), pair_j(:) allocate(x(dim, n), pair_i(maxnpair), pair_j(maxnpair)) ! initialize positions with pseudo-random numbers call random_number(x) ! finding pairs call cellList(dim, n, 27, x, cutoff, maxnpair, npairs, pair_i, pair_j) write (*, \u0026#39;(A)\u0026#39;) \u0026#39;Executing cellList\u0026#39; write (*, \u0026#39;(2x, A,I4,A)\u0026#39;) \u0026#39;Found \u0026#39;, npairs, \u0026#39; pairs\u0026#39; write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;First and last 5 pairs of points found:\u0026#39; write (*, \u0026#39;(2x, 4(A4, 1x))\u0026#39;) \u0026#39;Pair\u0026#39;, \u0026#39;i\u0026#39;, \u0026#39;j\u0026#39; do k = 1, npairs if (k \u0026lt;= 5 .or. k \u0026gt; npairs - 4) write (*, \u0026#39;(2x, 3(I4, 1x))\u0026#39;) k, \u0026amp; pair_i(k), pair_j(k) if (k == 6) write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;...\u0026#39; end do end program main #endif "},{"id":11,"href":"/worknotes/docs/caf/","title":"Coarray Fortran Things","section":"Docs","content":" Coarray Fortran Things # ignoring the basics. Best tutorial to start with is this one. Basic, but couldn\u0026rsquo;t find very beginner friendly ones.\nInstall # OpenCoarrays is a library usable with gfortran and uses MPI 1-sided comms as the to perform the communications. install via linuxbrew or spack.\nIntel compilers don\u0026rsquo;t rely on external libraries and should be ready to use coarrays with intel-MPI. It only requires compilation with -coarray option.\nCompiling and Running # GNU + OpenCoarrays # Compile: caf -fcoarray=lib\ncaf is just a wrapper around the mpif90 command, which is in turn a wrapper around the underlying compiler (in this case gfortran). But you can always use the command caf --show to see the include flags and libraries, in case you don\u0026rsquo;t want to use the caf wrapper.\nrun: cafrun [MPI options] yourprogram\nTo compile for a single image (i.e., serial): gfortran -fcoarray=single ... and execute like any other executable (./myprogram).\nIntel # Compile: ifort -coarray. Run like any other program: ./myprogram. Control the number of images with the FOR_COARRAY_NUM_IMAGES environment variable.\nTo compile for a single image (serial): ifort -coarray=single. However, unlike GNU+OpenCoarrays, you can still execute this executable with mpiexec [MPI options] myprogram.\nnote the intel compiler caf shared library requires intel mpi (even if compiling with -coarray=single).\nDefining static coarrays # For variables with copies on all images, just add [*] to end of variable declaration e.g.,\ninteger:: a[*] ! a has a copy on each image real:: b(2, 2)[*] ! each image will have b, a 2x2 array logical, codimension[*]:: c(3, 3, 3) ! the codimension property can be used instead. user defined types are defined in the normal way as well.\ntype(my_type):: d[*] type(my_type2), codimension[*]:: e(10) Defining and allocating allocatable coarrays # Like static variables, allocatable variables are defined with the [:] suffix or with the codimension property. The colon being used instead of the * is needed and means the number of images is being deferred (maybe you can allocate variables only on some images then? Needs testing).\nreal, allocatable:: f(:)[:] integer, allocatable, codimension[:]:: g(:,:) Remember that when you allocate the array, you must allocate them using the same size. The program will continue without issue, but references to other images\u0026rsquo; arrays will be misplaced. This is because each image uses the allocated size of its local copy as a template for other images\u0026rsquo; copies of the variable (from what I can tell - tested with gfortran and opencoarrays).\nCoarrays in subroutines/functions # the codimension property can be used to declare input variables of a subroutine.\nCoarrays in subroutines/functions must either be: passed as an argument, defined as allocatable, or with the save attribute to be declared. The save attribute is for when you know the number of images to be used in the subroutine. allocatable must be used otherwise.\nSome notes on performance # MPI/osh_put are generally faster than gets, so keep that in mind when writing algorithms. ( source)\nsingle node performance can be subpar compared to MPI ( Source)\nMore to come\u0026hellip;\n"},{"id":12,"href":"/worknotes/docs/cuda/reference-codes/","title":"nvfortran and nvc++ reference codes","section":"CUDA Programming","content":"A collection of downloadable sample codes from the CUDA Fortran for Scientists and Engineers book and a rough translation into C++.\nMight put in the real examples at a later date. E.g., estimating pi via Monte-Carlo methods, option pricing, FDM, p2p and MPI GPU matrix transpose, FFTs, and Poisson solver.\n"},{"id":13,"href":"/worknotes/docs/cuda/reference-codes/precision_m/","title":"precision_m","section":"nvfortran and nvc++ reference codes","content":" precision_m # Description # Module containing static types for single and double precision. Note that nvfortran has the options that can be used to demote/promote the precision of real variable/parameter declarations e.g. -r4 asks the compiler to interpret real declarations as real(4), -r8 interprets real as real(8), and -M[no]r8 will promote real declarations to double precision. nvc, nvcc, and nvc++ don\u0026rsquo;t have an equivalent as far as I know.\nCode (C++) # #ifndef PRECISION #define PRECISION // Compile with -DDOUBLE to compile with double precision #ifdef DOUBLE typedef double userfp_t; #else typedef float userfp_t; #endif #endif Code (Fortran) # module precision_m integer, parameter:: sf = kind(0.) integer, parameter:: df = kind(0.d0) ! compile with -DDOUBLE to compile with double precision #ifdef DOUBLE integer, parameter:: f = df #else integer, parameter:: f = sf #endif end module precision_m "},{"id":14,"href":"/worknotes/docs/cfdem/liggghts-intel-comp/","title":"Speeding Up LIGGGHTS with Intel Compilers and Compiler Options","section":"CFDEM","content":" Speeding Up LIGGGHTS with Intel Compilers and Compiler Options # This page looks at using basic optimization options and Intel OneAPI Compilers (for x86 CPU architectures) to reduce the run-time of LIGGGHTS.\nThe page will first show the difference in performance of the Intel compilers compared to the GNU compilers and also looks at different compiler options. After hopefully convincing you of why you should use the intel compilers, the page then goes on to explain how to build LIGGGHTS with the Intel compilers.\nFor those unaware, GNU compilers are shipped with most Linux systems, including Ubuntu. You can invoke the GNU C compiler with gcc.\nEverything on this page is run on the following platform:\nAn Inspiron 7501 with i7-10750H CPU 24GB (16GB + 8GB) @ 2933MHz RAM Windows 10 WSL2 Ubuntu 20.04 The tests use the chute-wear example files in LIGGGHTS-PUBLIC/examples/LIGGGHTS/Tutorials_public/chute_wear.\nFor this investigation, I\u0026rsquo;m using the VTK and OpenMPI packages from APT, which give me VTK v6.3.0 and OpenMPI v4.0.3.\nCompilers\u0026rsquo; and Compiler Options\u0026rsquo; Performance # The compiler options to investigate # The -O2 optimisation option that the Makefile.mpi file specifies is a default level of optimisation that GNU and Intel compilers use. On the other hand, the -funroll-loops is an extra optimisation that unrolls loops which may improve speed of the compiled program.\nHere, we\u0026rsquo;re going to look at the following options:\n-O3, which is increasing the optimisation level from default. This isn\u0026rsquo;t guaranteed to improve run-times. -march=native/-xhost, which, respectively, ask the GNU and Intel compilers to compile code that can better take advantage of the CPU architecture on the computer. -flto/-ipo, which, respectively, ask the GNU and Intel compilers to perform interprocedural optimization, which may improve speed of the compiled program. I don\u0026rsquo;t consider using options like -Ofast or -ffast-math because they can reduce the accuracy of numerical operations, which is not ideal, as time-integration simulations\u0026rsquo; results can be sensitive to numerical accuracy.\nThe Tests # Test No. Compiler Options 1 CCFLAGS = -O2 -funroll-loops -fPIC, LINKFLAGS = -O2 -fPIC 2 CCFLAGS = -O2 -fPIC, LINKFLAGS = -O2 -fPIC 3 CCFLAGS = -O3 -fPIC, LINKFLAGS = -O3 -fPIC 4 CCFLAGS = -O3 -fPIC -march=native (-xhost), LINKFLAGS = -O3 -fPIC -march=native (-xhost) 5 CCFLAGS = -O3 -fPIC -flto (-ipo), LINKFLAGS -O3 -fPIC -flto (-ipo) Where the options in brackets are the equivalent option for the Intel Compilers. Each test is run 5 times and the average is reported in the results below.\nResults # The average (of 5 runs) run-time of LIGGGHTS chute-wear example, when LIGGGHTS is built with either GNU and Intel compilers and with different compiler options. The runs where the LIGGGHTS executable was built with the -ipo Intel Compiler option failed, so no results reported.\nDiscussion # The Intel compiler performed better in all tests, where the fastest average run-time of the Intel Compiler tests ran 2s (~11%) faster than the fastest average run-time of the GNU Compiler tests.\nThe Intel compilers meaningfully benefitted from the -xhost compiler option, whereas the GNU compiler didn\u0026rsquo;t benefit from its equivalent (-march=native). In fact, it was slower! I should probably look into it further.\nI failed to make use of the Intel -ipo option, which I would like to work as this option has been quite useful in my Fortran simulation code. I suspect it\u0026rsquo;s because the VTK libraries are dynamic, not static.\nThe -O3 option didn\u0026rsquo;t really make a difference, and the loop unrolling seemed to slow down the LIGGGHTS code. However, I need to see if both of those effects are observed with larger simulation (the chute-wear example uses only ~800 concurrent DEM particles).\nGetting the Intel compilers # The Intel OneAPI compilers are free for anybody to use. For LIGGGHTS, only the base kit is necessary.\nOnce you have installed the compilers by following Intel\u0026rsquo;s instructions, you will need to make sure you have the VTK and OpenMPI APT packages.\nModifying the make File to use the Intel compiler # The basic LIGGGHTS installation documentation will suggest users to build LIGGGHTS using the make auto command. But, here, we\u0026rsquo;ll be using the make mpi command, which uses the Makefile.mpi Make file. LIGGGHTS assumes we\u0026rsquo;re using the default APT packages, so we have to modify the Makefile. Edit LIGGGHTS-PUBLIC/src/MAKE/Makefile.mpi so that the following lines have been modified:\nCC = icpx # this is the Intel OneAPI C++ compiler LINK = icpx # These tell Make where your OpenMPI libraries and include files are. You may need to modify them to suit you. MPI_INC = -I/usr/lib/x86_64-linux-gnu/openmpi/include MPI_PATH = -L/usr/lib/x86_64-linux-gnu/openmpi/lib MPI_LIB = -lmpi # These tell Make where your VTK libraries and include files. You may need to modify them to suit you. I had to change the default 6.2 to 6.3 to reflect the version of the APT package. VTK_INC = -I/usr/include/vtk-6.3 VTK_PATH = -L/usr/lib/x86_64-linux-gnu VTK_LIB = -lvtkCommonCore-6.3 -lvtkIOCore-6.3 -lvtkIOXML-6.3 -lvtkIOLegacy-6.3 -lvtkCommonDataModel-6.3 -lvtkIOParallel-6.3 -lvtkParallelCore-6.3 -lvtkParallelMPI-6.3 -lvtkIOImage-6.3 -lvtkCommonExecutionModel-6.3 -lvtkFiltersCore-6.3 -lvtkIOParallelXML-6.3 If you\u0026rsquo;ve modified the Make file correctly for your system, the make mpi command run inside the src directory should build the lmp_mpi executable. Changing\nChanging the LIGGGHTS default compiler options # The default make auto command uses some default compiler options. While you can add to the default compiler options by editing the Makefile.user file, it\u0026rsquo;s less straightforward to change the compiler options. To change them, you will want to edit the LIGGGHTS_PUBLIC/src/MAKE/Makefile.mpi file and use the make mpi command instead. At the time of writing, the default compiler and linker options are:\nCCFLAGS = -O2 -funroll-loops -fstrict-aliasing -Wall -Wno-unused-result -fPIC LINKFLAGS = -O2 -fPIC Note that -O2 and -funroll-loops are the optimisation options, and the rest are compiler warnings, although -fPIC is neither, and is telling the compiler to produce Position Independent Code.\nTo change the compiler options, change the code following the CCFLAGS = or LINKFLAGS =. E.g.,\nCCFLAGS = -O3 -xhost -fPIC LINKFLAGS = -O3 -xhost -fPIC would be the best of the options looked at on this page. The meaning of the above changes are explained above.\n"},{"id":15,"href":"/worknotes/docs/manual-vectorization/addition/","title":"Vectorizing Array Addition","section":"Investigating Manual Vectorization for SPH","content":" Vectorizing Array Addition # A common place to start, is to manually vectorize the addition of two arrays, and storing the result in a third array (from herein, ABC addition):\nvoid abc_base_sgl(const int nelem, float* a, float* b, float*) { for (int i = 0; i \u0026lt; nelem; ++i) { c[i] = a[i] + b[i]; } } In the benchmarking code, the arrays are created with new.\nDoing the Vectorization # Vectorizing the ABC function with 128-bit vectors (SSE instructions) # This function, manually vectorized, looks like:\n// 128bit width (SSE) void abc_128_sgl(const int nelem, float* a, float* b, float* c) { for (int i = 0; i \u0026lt; nelem; i += 4) { __m128 v1 = _mm_loadu_ps(\u0026amp;a[i]); __m128 v2 = _mm_loadu_ps(\u0026amp;b[i]); __m128 res = _mm_add_ps(v1, v2); _mm_storeu_ps(\u0026amp;c[i], res); } } Note that this requires #include \u0026lt;immintrin.h\u0026gt; to make use of the types and functions.\nThe _mm_loadu_ps function name can be broken down as:\nmm: MultiMedia (Intel instructions intended for media). mm alone, usually indicates 128-bit width vectors. loadu: Load data into SIMD vectors. This is specifically for \u0026ldquo;unaligned\u0026rdquo; data. More on this later. ps: Packed Single precision data. Hence, this function loads single precision floats from a float pointer/array, to a packed SIMD single precision vector. This load operation takes the supplied pointer, and loads the next 4 elements following the address. It knows to load 4 elements, as single precision data is 32 bits. Loading 4 elements at a time, is the reason why the loop counter is incremented by 4 every iteration.\n_mm_add_ps is part of the same MultiMedia extensions, and works with single precision floats, but performs an add operation on two 128-bit vectors. The add operation will operate on each element on each vector.\n_mm_storeu_ps takes the res vector, and places back at the position specified by the supplied pointer (in this case, \u0026amp;c[i]).\n__m128 is the 128-bit wide vector for single precision floating point numbers.\nThe double precision version looks similar:\n// 128bit width (SSE) void abc_128_dbl(const int nelem, double* a, double* b, double* c) { for (int i = 0; i \u0026lt; nelem; i += 2) { __m128d v1 = _mm_loadu_pd(\u0026amp;a[i]); __m128d v2 = _mm_loadu_pd(\u0026amp;b[i]); __m128d res = _mm_add_pd(v1, v2); _mm_storeu_pd(\u0026amp;c[i], res); } } There are 3 main differences in this version. Two of them are that ps is replaced by pd everywhere, and d is added to the end of the __m128 types. These ensure that the equivalent functions and types for double precision floating point numbers are used. The 3rd change is that the loop counter is incremented by two instead of 4 every iteration. This is because double precision floating point data is stored using 64 bits, so only two of them can fit into an 128-bit SIMD vector.\nVectorizing the ABC function with 256-bit vectors (AVX instructions) # The AVX instruction set adds new instructions to the collection of SIMD instructions, as well as allowing for 256-bit vectors. For this example, the code looks pretty similar. The single precision code looks like:\n// 256bit width (AVX) void abc_256_sgl(const int nelem, float* a, float* b, float* c) { for (int i = 0; i \u0026lt; nelem; i += 8) { __m256 v1 = _mm256_loadu_ps(\u0026amp;a[i]); __m256 v2 = _mm256_loadu_ps(\u0026amp;b[i]); __m256 res = _mm256_add_ps(v1, v2); _mm256_storeu_ps(\u0026amp;c[i], res); } } Where mm has been changed to mm256, to indicate these instructions are part of the 256-bit wide vector instructions, and the incrememnt has been doubled from 4 to 8, due to the doubling in width of the vectors. The same changes are true for the double precision version:\n// 256bit width (AVX) void abc_256_dbl(const int nelem, double* a, double* b, double* c) { for (int i = 0; i \u0026lt; nelem; i += 4) { __m256d v1 = _mm256_loadu_pd(\u0026amp;a[i]); __m256d v2 = _mm256_loadu_pd(\u0026amp;b[i]); __m256d res = _mm256_add_pd(v1, v2); _mm256_storeu_pd(\u0026amp;c[i], res); } } Performance of the SINGLE PRECISION manually vectorized ABC function with 128-bit vectors (SSE instructions) # Here, I make use of Google\u0026rsquo;s benchmark tool. Each array, a, b, and c, are between 4,096 and 16,777,216 elements long. The program compilation command:\ng++ -o abc.x abc.cpp -mavx2 -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread First, the single precision data and using the benchmark comparison tool:\nComparing abc_base_sgl (from ./abc.x) to abc_128_sgl (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------- [abc_base_sgl vs. abc_128_sgl]/4096 -0.5372 -0.5372 12619 5841 12590 5827 [abc_base_sgl vs. abc_128_sgl]/32768 -0.5164 -0.5164 100548 48628 100314 48514 [abc_base_sgl vs. abc_128_sgl]/262144 -0.5091 -0.5093 802916 394148 801042 393099 [abc_base_sgl vs. abc_128_sgl]/2097152 -0.5027 -0.5027 6484863 3225174 6469666 3217643 [abc_base_sgl vs. abc_128_sgl]/16777216 -0.5172 -0.5172 54029226 26084896 53902545 26023999 OVERALL_GEOMEAN -0.5166 -0.5167 0 0 0 0 In this results, the \u0026ldquo;Old\u0026rdquo; is the non-manually vectorized loop, and the \u0026ldquo;New\u0026rdquo; is the vectorized loop. The comparison shows at least (1-0.5166)^-1 = 2x speedup by manually vectorization for all array sizes tested here! To compare the manually vectorized code to compiler\u0026rsquo;s automatic vectorization, we can compile the code again with:\ng++ -o abc.x abc.cpp -msse -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread which\nreplaces -mavx2 with -msse, enabling only 128-bit wide vectors and SSE instructions, add -O2, which enables level 2 compiler optimisations, and add -ftree-vectorize, which enables automatic vectorization of loops. These options will make the code faster through compiler optimisations, which should benefit both the original base code and the manually vectorized code, but they will also enable automatic vectorization on the original loop only. Comparing the results performance of the now automatically vectorized function (abc_base_sgl), with the manually vectorized function (abc_128_sgl), I get:\nComparing abc_base_sgl (from ./abc.x) to abc_128_sgl (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------- [abc_base_sgl vs. abc_128_sgl]/4096 -0.0001 +0.0002 630 630 629 629 [abc_base_sgl vs. abc_128_sgl]/32768 -0.0021 -0.0021 6345 6332 6330 6317 [abc_base_sgl vs. abc_128_sgl]/262144 +0.0162 +0.0163 104701 106392 104437 106144 [abc_base_sgl vs. abc_128_sgl]/2097152 +0.0023 +0.0027 908528 910590 906019 908466 [abc_base_sgl vs. abc_128_sgl]/16777216 +0.0225 +0.0230 14303786 14625654 14263831 14591516 OVERALL_GEOMEAN +0.0077 +0.0080 0 0 0 0 which shows that at worst, the manually vectorized code is approx. 2% slower than the automatically vectorized loop and none of the differences are statistically significant.\nPerformance of the SINGLE PRECISION manually vectorized ABC function with 256-bit vectors (AVX instructions) # Using the same test method as the 128-bit version, using no compiler optimisations, I get:\nComparing abc_base_sgl (from ./abc.x) to abc_256_sgl (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------ ------------------------------------------------------------------ [abc_base_sgl vs. abc_256_sgl]/4096 -0.7394 -0.7395 12390 3229 12361 3221 [abc_base_sgl vs. abc_256_sgl]/32768 -0.7276 -0.7277 99812 27192 99579 27118 [abc_base_sgl vs. abc_256_sgl]/262144 -0.7242 -0.7243 802667 221387 800794 220789 [abc_base_sgl vs. abc_256_sgl]/2097152 -0.7151 -0.7152 6437792 1834148 6422769 1829137 [abc_base_sgl vs. abc_256_sgl]/16777216 -0.6711 -0.6713 53525278 17602131 53399838 17553344 OVERALL_GEOMEAN -0.7164 -0.7165 0 0 0 0 Which shows a further speedup over the 128-bit version. Over the base version, the speedup is approximately (1-0.7164)^-1 = 3.5x. You may notice that the speedup obtained decreases slightly with the size of test, which I suspect is due to the size of cache being used on the CPU. The 4,096 element test (4096 elements * 4 bytes * 2 arrays = ~33kB), is enough to sit within L1 cache. The next test moves to L2 cache as 32,768 elements * 4 bytes * 2 arrays = ~262kB moves to L2 unified cache. The next test (2,097,152 elements * 4 bytes * 2 arrays = ~16.8MB) must move to L3 unified cache, and the final test (16, 777, 216 * 4 bytes * 2 arrays = ~134MB) must go to RAM. This behaviour isn\u0026rsquo;t observed with 128-bit width registers, as the smaller vectors means less data needing to be fetched per transaction.\nThe minimum speedup is ~3x, and the best is ~3.8x - neither of which is 2x that observed with the 128-bit vectors.\nMemory type Capacity Capacity (single precision elements) L1 cache 48KiB 12,288 L2 cache (unified) 1280KiB 327,680 L3 cache (unified) 36864KiB 9,437,184 RAM 500GiB lots Confirming similarity of manual vectorization to automatic vectorization:\ng++ -o abc.x abc.cpp -mavx -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread -O2 -ftree-vectorize Comparing abc_base_sgl (from ./abc.x) to abc_256_sgl (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------- [abc_base_sgl vs. abc_256_sgl]/4096 +0.0010 +0.0012 633 633 631 632 [abc_base_sgl vs. abc_256_sgl]/32768 +0.0012 +0.0016 6097 6104 6080 6090 [abc_base_sgl vs. abc_256_sgl]/262144 +0.0009 +0.0013 105703 105800 105417 105551 [abc_base_sgl vs. abc_256_sgl]/2097152 -0.0020 -0.0016 910727 908903 908205 906771 [abc_base_sgl vs. abc_256_sgl]/16777216 +0.0281 +0.0286 14356772 14760474 14316874 14725833 OVERALL_GEOMEAN +0.0058 +0.0062 0 0 0 0 I see a similarly small variation in performance between the the automatically vectorized base function and the manually vectorized version.\nThe code shown here achieves far less than ideal speedup e.g., ~2x achieved for 128-bit vectors, when 4x is ideal; and ~3.5x achieved for 256-bit vectors, when 8x is ideal. I have yet to understand the details of the mechanism behind this, but I think there are many factors that go into this, like the ordering of instructions or whether the data is prefetched.\nPerformance of the DOUBLE PRECISION manually vectorized ABC function with 128-bit vectors (SSE instructions) # Using the compilation command with no compiler options:\ng++ -o abc.x abc.cpp -msse -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread We get the results between the non-vectorized base function and the manually vectorized function:\nComparing abc_base_dbl (from ./abc.x) to abc_128_dbl (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------- -------- [abc_base_dbl vs. abc_128_dbl]/4096 -0.0514 -0.0516 12544 11899 12515 11869 [abc_base_dbl vs. abc_128_dbl]/32768 -0.0230 -0.0230 101078 98749 100842 98518 [abc_base_dbl vs. abc_128_dbl]/262144 -0.0429 -0.0431 812827 777929 810929 775969 [abc_base_dbl vs. abc_128_dbl]/2097152 -0.0112 -0.0116 6743898 6668566 6728128 6650022 [abc_base_dbl vs. abc_128_dbl]/16777216 -0.0746 -0.0748 56644964 52418793 56512720 52284069 OVERALL_GEOMEAN -0.0409 -0.0411 0 0 0 0 Which shows very modest improvement of approx 4%. Furthermore, only the largest test is determined by the comparison tool to be statistically significant. This is not surprising, given the results from the single precision manually vectorized code. The single precision code obtained a speedup of approximately 2x, despite being able to combine 4 add operations into one. However, the double precision code can only combine 2 add operations, which is apparently insufficient to offset the memory latency/bandwidth.\nChecking the manually vectorized code against automatic vectorization:\ng++ -o abc.x abc.cpp -msse -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread -O2 -ftree-vectorize Comparing abc_base_dbl (from ./abc.x) to abc_128_dbl (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------- [abc_base_dbl vs. abc_128_dbl]/4096 +0.0233 +0.0237 1532 1568 1528 1565 [abc_base_dbl vs. abc_128_dbl]/32768 -0.0313 -0.0309 13050 12641 13014 12612 [abc_base_dbl vs. abc_128_dbl]/262144 +0.0126 +0.0128 212077 214751 211534 214245 [abc_base_dbl vs. abc_128_dbl]/2097152 -0.0357 -0.0353 2723765 2626471 2716242 2620327 [abc_base_dbl vs. abc_128_dbl]/16777216 -0.0097 -0.0095 30242092 29949972 30165349 29880070 OVERALL_GEOMEAN -0.0084 -0.0081 0 0 0 0 Which again, shows minimal difference between the manually and automatically vectorized code.\nPerformance of the DOUBLE PRECISION manually vectorized ABC function with 256-bit vectors (AVX instructions) # Comparison results:\nComparing abc_base_dbl (from ./abc.x) to abc_256_dbl (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------- [abc_base_dbl vs. abc_256_dbl]/4096 -0.4700 -0.4699 12537 6644 12506 6629 [abc_base_dbl vs. abc_256_dbl]/32768 -0.4609 -0.4606 101533 54741 101257 54613 [abc_base_dbl vs. abc_256_dbl]/262144 -0.4548 -0.4546 814199 443934 811995 442898 [abc_base_dbl vs. abc_256_dbl]/2097152 -0.3759 -0.3758 6699499 4180956 6682306 4171199 [abc_base_dbl vs. abc_256_dbl]/16777216 -0.3386 -0.3385 56663574 37476558 56520373 37388836 OVERALL_GEOMEAN -0.4224 -0.4222 0 0 0 0 Like the single precision 256-bit results, we can see a decreasing speedup as the tests increase in size. The best speedup achieved is ~1.9x and slowest is ~1.5x.\nImproving SIMD Performance By Aligning Memory # When explaining the _mm_storeu_px and _mm_loadu_px functions, I mentioned that these, respectively, store and load data to/from \u0026ldquo;aligned\u0026rdquo; memory. When using the \u0026ldquo;classic\u0026rdquo; malloc or new functions to allocate memory for arrays, these arrays may be \u0026ldquo;unaligned\u0026rdquo;. This means its integer address is the computer\u0026rsquo;s memory, is not divisible by its size (in bytes). I\u0026rsquo;m not sure of the details of why, but this means that accessing this memory can be more expensive. It may also negatively affect caching behaviour.\nBecause of this unalignment, we must use the storeu and loadu functions so that they can properly store and load data to/from locations that are unaligned. However, there are store and load functions that can be used on aligned data, which can be more performant.\nMigrating the existing code to use aligned data # The existing code allocates memory for our arrays using:\nfloat *a = new float[nelem], *b = new float[nelem], *c = new float[nelem]; for single precision data (replace float with double for the double precision version). To ensure the arrays are aligned, you must use the std::aligned_alloc function, which comes with the cstdlib header. Our allocation becomes:\nconst size_t size = sizeof(float) * nelem, alignment = 64; float *a = static_cast\u0026lt;float*\u0026gt;(std::aligned_alloc(alignment, size)); float *b = static_cast\u0026lt;float*\u0026gt;(std::aligned_alloc(alignment, size)); float *c = static_cast\u0026lt;float*\u0026gt;(std::aligned_alloc(alignment, size)); Replace float with double and you have the double precision version. These arrays also have to be deallocated with free instead of delete.\nAligning data ensures that your allocated memory begins at a memory address that is a multiple of a power of 2. This could matter for performance because of how caching works. When the CPU pulls data from the main memory (RAM) into cache, it does so in chunk sizes, which depend on the CPU. This chunk size is known as a \u0026ldquo;cache line\u0026rdquo;, and is commonly 64 bytes, but can also be 32 or 128 bytes as well. Cache lines that are pulled into cache, are aligned with cache line chunks. So, if your array begins and ends at a cache-line boundary, it mitigates the possibility of the CPU pulling in unneeded data to the cache.\nAnother reason, more relevant to vectorization, is that when the program is loading or storing data from/to data that is aligned to vector register widths, the store and load functions can be used. store and load are faster than their unaligned equivalents, storeu and loadu, as they do not need to do extra work to check for alignment.\nAlligned memory may not be useful when allocating lots of small bits of information e.g., allocating 4 2-element arrays that are aligned on 64-byte boundaries will have them look like:\n|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--| |a1|a2| | | | | | |b1|b2| | | | | | |c1|c2| | | | | | |d1|d2| | | | | | | |--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--| where each block is an 8-byte element. Aligning these 4 two-element double arrays on 64-byte boundaries causes them to be spread out in memory. This could be undesirable as your program might use more memory, and they would be pulled into the cache in seperate cache lines - meaning your program may run slower between accesses of the arrays.\nPerformance of aligned vs unaligned data for SIMD instructions # The results below compare run times between the unaligned and aligned versions of each code for each of the datatypes and vector sizes. These tests use the std::aligned_alloc to align the arrays to 64-byte boundaries, and all loadu and storeu functions are swapped with load and store\nAligned memory: single precision 128-bit # Comparing abc_128_sgl (from ./abc.x) to abc_128_sgl_aligned (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------- [abc_128_sgl vs. abc_128_sgl_aligned]/4096 -0.0639 -0.0635 5807 5436 5792 5424 [abc_128_sgl vs. abc_128_sgl_aligned]/32768 -0.0965 -0.0963 50260 45408 50129 45302 [abc_128_sgl vs. abc_128_sgl_aligned]/262144 -0.0604 -0.0600 390059 366517 389004 365662 [abc_128_sgl vs. abc_128_sgl_aligned]/2097152 -0.0609 -0.0605 3161569 2968982 3152943 2962053 [abc_128_sgl vs. abc_128_sgl_aligned]/16777216 +0.0281 +0.0285 25573480 26291967 25502874 26230381 OVERALL_GEOMEAN -0.0247 -0.0244 0 0 0 0 Aligned memory: single precision 256-bit # Comparing abc_256_sgl (from ./abc.x) to abc_256_sgl_aligned (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------- [abc_256_sgl vs. abc_256_sgl_aligned]/4096 -0.1635 -0.1634 3312 2770 3303 2764 [abc_256_sgl vs. abc_256_sgl_aligned]/32768 -0.1261 -0.1260 27567 24090 27497 24034 [abc_256_sgl vs. abc_256_sgl_aligned]/262144 -0.1344 -0.1342 225293 195017 224719 194562 [abc_256_sgl vs. abc_256_sgl_aligned]/2097152 -0.1191 -0.1191 1833080 1614842 1828802 1611060 [abc_256_sgl vs. abc_256_sgl_aligned]/16777216 -0.0395 -0.0392 18248268 17528127 18201311 17487204 OVERALL_GEOMEAN -0.0699 -0.0697 0 0 0 0 Aligned memory: double precision 128-bit # Comparing abc_128_dbl (from ./abc.x) to abc_128_dbl_aligned (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------- [abc_128_dbl vs. abc_128_dbl_aligned]/4096 -0.0647 -0.0647 12000 11224 11972 11198 [abc_128_dbl vs. abc_128_dbl_aligned]/32768 -0.0703 -0.0705 97467 90611 97240 90383 [abc_128_dbl vs. abc_128_dbl_aligned]/262144 -0.0627 -0.0631 780496 731551 778675 729556 [abc_128_dbl vs. abc_128_dbl_aligned]/2097152 -0.0579 -0.0583 6742195 6351631 6726466 6334012 [abc_128_dbl vs. abc_128_dbl_aligned]/16777216 +0.0270 +0.0270 52430133 53845667 52307789 53719950 OVERALL_GEOMEAN -0.0234 -0.0236 0 0 0 0 Aligned memory: double precision 256-bit # Comparing abc_256_dbl (from ./abc.x) to abc_256_dbl_aligned (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------- [abc_256_dbl vs. abc_256_dbl_aligned]/4096 -0.0969 -0.0968 6658 6012 6641 5998 [abc_256_dbl vs. abc_256_dbl_aligned]/32768 -0.0931 -0.0927 54958 49842 54809 49726 [abc_256_dbl vs. abc_256_dbl_aligned]/262144 -0.1182 -0.1179 448770 395737 447559 394813 [abc_256_dbl vs. abc_256_dbl_aligned]/2097152 -0.0735 -0.0735 4322564 4004882 4312476 3995495 [abc_256_dbl vs. abc_256_dbl_aligned]/16777216 -0.0479 -0.0475 38081022 36256189 37975969 36171577 OVERALL_GEOMEAN -0.0436 -0.0434 0 0 0 0 Effect of std::aligned_malloc # Almost all test sizes perform better than the unaligned counterparts. The only exceptions are the largest tests for the 128-bit vector code - these seem to perform almost 3% worse. This probably is related to the fact that the a and b arrays are entirely on RAM. This is also consistent with the 256-bit vector tests where the largest tests perform the worst when compared to their unaligned conterparts.\nThe 256-bit vector codes have the most increased performance - espcially for the tests that can fit in L1 and L2 cache. For example, the biggest improvement is seen in the single precision 128-bit vector code, for the problem size of 4,096 elements. The 16% reduction in run time makes it ~2.3 faster than its corresponding base code (up from ~2x).\nMost of the other tests gain at least 4% from its unaligned counterpart.\nAdding compiler optimisations # Compiling the code with -O2 and rerunning the tests with aligned memory:\nAligned memory + lvl 2 optimisations: single precision 128-bit # Comparing abc_128_sgl (from ./abc.x) to abc_128_sgl_aligned (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------- [abc_128_sgl vs. abc_128_sgl_aligned]/4096 +0.0099 +0.0103 630 636 628 635 [abc_128_sgl vs. abc_128_sgl_aligned]/32768 +0.0474 +0.0476 6316 6616 6300 6600 [abc_128_sgl vs. abc_128_sgl_aligned]/262144 +0.0001 +0.0001 105151 105166 104905 104919 [abc_128_sgl vs. abc_128_sgl_aligned]/2097152 -0.0035 -0.0033 907374 904171 905084 902062 [abc_128_sgl vs. abc_128_sgl_aligned]/16777216 +0.0273 +0.0275 14132072 14517526 14096031 14483670 OVERALL_GEOMEAN +0.0190 +0.0192 0 0 0 0 Aligned memory + lvl 2 optimisations: single precision 256-bit # Comparing abc_256_sgl (from ./abc.x) to abc_256_sgl_aligned (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------- [abc_256_sgl vs. abc_256_sgl_aligned]/4096 -0.4919 -0.4921 632 321 631 320 [abc_256_sgl vs. abc_256_sgl_aligned]/32768 -0.2773 -0.2774 6209 4488 6195 4476 [abc_256_sgl vs. abc_256_sgl_aligned]/262144 +0.0066 +0.0064 104733 105426 104488 105162 [abc_256_sgl vs. abc_256_sgl_aligned]/2097152 +0.0063 +0.0059 908430 914187 906310 911683 [abc_256_sgl vs. abc_256_sgl_aligned]/16777216 +0.0078 +0.0073 14697929 14812229 14663625 14771162 OVERALL_GEOMEAN -0.0882 -0.0885 0 0 0 0 Aligned memory + lvl 2 optimisations: double precision 128-bit # Comparing abc_128_dbl (from ./abc.x) to abc_128_dbl_aligned (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------- [abc_128_dbl vs. abc_128_dbl_aligned]/4096 +0.0612 +0.0617 1567 1663 1563 1660 [abc_128_dbl vs. abc_128_dbl_aligned]/32768 +0.0766 +0.0770 12643 13611 12608 13579 [abc_128_dbl vs. abc_128_dbl_aligned]/262144 -0.0037 -0.0033 211424 210649 210844 210148 [abc_128_dbl vs. abc_128_dbl_aligned]/2097152 +0.0810 +0.0815 2597400 2807846 2590158 2801291 [abc_128_dbl vs. abc_128_dbl_aligned]/16777216 +0.0345 +0.0299 30125298 31163848 30049063 30948143 OVERALL_GEOMEAN +0.0411 +0.0405 0 0 0 0 Aligned memory + lvl 2 optimisations: double precision 256-bit # Comparing abc_256_dbl (from ./abc.x) to abc_256_dbl_aligned (from ./abc.x) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------------- [abc_256_dbl vs. abc_256_dbl_aligned]/4096 -0.2838 -0.2839 1558 1116 1555 1113 [abc_256_dbl vs. abc_256_dbl_aligned]/32768 -0.2637 -0.2640 12501 9205 12472 9180 [abc_256_dbl vs. abc_256_dbl_aligned]/262144 -0.0077 -0.0081 213428 211785 212930 211203 [abc_256_dbl vs. abc_256_dbl_aligned]/2097152 +0.0229 +0.0224 2699745 2761498 2693422 2753700 [abc_256_dbl vs. abc_256_dbl_aligned]/16777216 +0.0322 +0.0320 30455778 31437745 30384702 31356069 OVERALL_GEOMEAN -0.0423 -0.0427 0 0 0 0 Effects of adding compiler optimisation # Quite interestingly, the -O2 compiler optimisations amplify the performance of SIMD instructions for the tests that fit within L1 and L2 cache. The 128-bit tests are brought much closer to ideal performance for those small problem sizes. The 128-bit tests do not seem to perform very differently with or without aligned memory. Aligning the data allocations even seem to cause the 128-bit tests to perform worse by a little bit! In contrast, the 256-bit vector instructions apparently require aligned memory to benefit form -O2 optimisations in terms of speedup.\nTest ID Best speedup over base abc_128_sgl 4.18x abc_128_sgl_aligned 3.97x abc_256_sgl 3.97x abc_256_sgl_aligned 5.84x abc_128_dbl 1.78x abc_128_dbl_aligned 1.74x abc_256_dbl 1.92x abc_256_dbl_aligned 2.6x Note that the non-vectorized code also benefits from using aligned data, so the best speedups shown above, comparing like-for-like i.e., aligned vectorized, with aligned non-vectorized; and non-aligned vectorized, with non-aligned non-vectorized.\nConclusion # Hopefully it\u0026rsquo;s clear here how manual vectorization works. The main takeaways are:\n_mm_storeu_xx and _mm_loadu_xx are used to load contiguous 128-bit chunks into vector registers (xx is pd or ps for floating point data) 256-bit and 512-bit equivalents use the _mm256_ and _mm512_ prefix instead of _mm_. If the data is aligned i.e., allocated with std::aligned_alloc, or with the alignas keyword, storeu and loadu can be replaced with store and load respectively. I demonstrate a meaningful performance gain when using aligned memory with manual vectorization In later pages, I\u0026rsquo;ll show an example where aligned array allocation is unsuitable Once loaded, vectors are added with _mm_add_xx function. While not shown here, there are equivalent functions for subtraction (sub), multiplication mul, division (div), min/max (min/max) and others. For the array addition example used here, it\u0026rsquo;s not too complicated to achieve the same performance as the compilers auto-vectorization (assuming other compiler optimisations as used in tandem). "},{"id":16,"href":"/worknotes/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search/","title":"Cell list pair search - reducing search space","section":"Useful code snippets","content":" Cell-lists point-pairs search with fixed cutoff - reducing search space # Description # The naive cell-list pair search, is a good start. But a drawback of that implementation is that it searches through all adjacent cells, and uses an if statement to avoid duplicate checks. However, looping over all adjacent cells still takes time, and the if statement can be expensive by introducing branching.\nHowever, to eliminate duplicate checks, we only need to compare particles in the current cell with particles in half of the adjacent cells. For instance, if you\u0026rsquo;ve checked particle A in one cell against particle B in a neighboring cell, you don\u0026rsquo;t need to check B against A again when you move to that neighboring cell. Hence, you can omit some cells from the search for each particle to avoid this redundancy. See below sketches that illustrate how coverage is not reduced with this strategy.\nSketch showing a 2D grid that would be used to perform the pair search. The grid is setup to include at least one buffer layer of cells\nWhen looping over each particle to find pairs, the particles\u0026rsquo; cell will be searched, as well as 4 adjacent cells. This reduces the search space from 9 cells, to 5 cells in 2D; and 27 cells, to 14 cells in 3D.\nThe coverage still contains the cells with particles, and the only cells missed are empty buffer cells.\nThe code below copies the setup code from the naive implementation, but makes modifications to reduce the search space. Note the introduction of the idxs parameter array used to selectively loop through adjacent cells.\nAdditional notes # I observed about 1.5x speedup on finding pairs in 100,000 elements, with default optimisation, but this speedup vanishes if compiling with -O1 and above.\nSubsequent pages will demonstrate how this algorithmic optimisation can still be useful.\nCode (Fortran) # Save the code as cll2.F90, and compile it with:\ngfortran -o cll2.x cll2.F90 ! Module to perform direct search for pairs of points within a fixed cutoff. ! cellList subroutine ! dim: An integer defining dimension of the coordinates. ! npoints: An integer with number of points. ! maxpercell: An integer with max number of points in a cell. ! x: An real/real(f) array of dim x points dimension containing the list of points to find pairs of. ! cutoff: The distance (same type as x) which defines a pair ! maxnpair: An integer of the maximum number of pairs expected. ! npairs: An integer with the number of pairs found. ! pair_i: An integer array of maxnpairs length with the \u0026#34;left-sided\u0026#34; point in a pair. ! pair_j: An integer array of maxnpairs length with the \u0026#34;right-sided\u0026#34; point in a pair. module cellList_m use iso_fortran_env, only: error_unit public private:: coordsToCell #ifdef SINGLEPRECISION parameter, integer:: f = kind(1.) #else parameter, integer:: f = kind(1.d0) #endif contains !--------------------------------------------------------------------------- function coordsToCell(dim, xi, minx, cutoff) result(icell) implicit none real(f), intent(in):: xi(dim), minx(3), cutoff integer, intent(in):: dim integer:: icell(3) icell(1:dim) = int((xi(:) - minx(1:dim))/cutoff) + 1 if (dim == 2) icell(3) = 1 end function coordsToCell !--------------------------------------------------------------------------- subroutine cellList(dim, npoints, maxpercell, x, cutoff, maxnpair, npairs, pair_i, pair_j) implicit none integer, intent(in):: dim, npoints, maxnpair, maxpercell real(f), intent(in):: x(dim, npoints), cutoff integer, intent(out):: npairs, pair_i(maxnpair), pair_j(maxnpair) integer:: i, j, k, kk, ngridx(3), icell(3), xi, yi, zi, jcell(3) real(f):: r2, minx(3), maxx(3) integer, allocatable:: pincell(:, :, :), cells(:, :, :, :) ! Below array is used to store which adjacent cells to search through integer, parameter:: sweep_idxs(3, 13) = reshape([-1, -1, -1, \u0026amp; -1, -1, 0, \u0026amp; -1, -1, 1, \u0026amp; -1, 0, -1, \u0026amp; -1, 0, 0, \u0026amp; -1, 0, 1, \u0026amp; -1, 1, -1, \u0026amp; -1, 1, 0, \u0026amp; -1, 1, 1, \u0026amp; 0, -1, -1, \u0026amp; 0, -1, 0, \u0026amp; 0, -1, 1, \u0026amp; 0, 0, -1], [3, 13]) ! determine grid min-max coordinates, with concessions made for dim=2 case minx(1:dim) = minval(x, dim=2) if (dim == 2) minx(3) = 0.d0 maxx(1:dim) = maxval(x, dim=2) if (dim == 2) maxx(3) = 0.d0 ! creating buffer layers minx(:) = minx(:) - 2.d0*cutoff maxx(:) = maxx(:) + 2.d0*cutoff ! initializing grid arrays allocate (pincell(ngridx(1), ngridx(2), ngridx(3)), \u0026amp; cells(maxpercell, ngridx(1), ngridx(2), ngridx(3))) pincell(:, :, :) = 0 ! populating grid with point information do i = 1, npoints icell = coordsToCell(dim, x(:, i), minx, cutoff) pincell(icell(1), icell(2), icell(3)) = \u0026amp; pincell(icell(1), icell(2), icell(3)) + 1 #ifdef DEBUG if (pincell(icell(1), icell(2), icell(3)) \u0026gt; maxpercell) then write (error_unit, \u0026#39;(A, 3(I2, 1x), A)\u0026#39;) \u0026#39;Number of particles in cell \u0026#39;, icell(1), icell(2), icell(3), \u0026amp; \u0026#39; exceeds the input maxpercell value.\u0026#39; write (error_unit, \u0026#39;(A)\u0026#39;) \u0026#39;Terminating...\u0026#39; error stop 1 end if #endif cells(pincell(icell(1), icell(2), icell(3)), icell(1), icell(2), icell(3)) = i end do npairs = 0 do i = 1, npoints icell(:) = coordsToCell(dim, x(:, i), minx, cutoff) ! first loop to search through particle i\u0026#39;s cell do k = 1, pincell(icell(1), icell(2), icell(3)) j = cells(k, icell(1), icell(2), icell(3)) if (j \u0026gt; i) then r2 = sum((x(:, i) - x(:, j))**2) if (r2 \u0026lt;= cutoff**2) then npairs = npairs + 1 pair_i(npairs) = i pair_j(npairs) = j end if end if end do ! second loop to search through adjacent 13 cells do kk = 1, 13 jcell(:) = icell(:) + sweep_idxs(:, kk) do k = 1, pincell(jcell(1), jcell(2), jcell(3)) j = cells(k, jcell(1), jcell(2), jcell(3)) r2 = sum((x(:, i) - x(:, j))**2) if (r2 \u0026lt;= cutoff**2) then npairs = npairs + 1 pair_i(npairs) = i pair_j(npairs) = j end if end do end do end do end subroutine cellList end module cellList_m #ifndef NOMAIN program main use cellList_m, only: f, cellList implicit none integer, parameter:: n = 100, dim = 3, maxnpair = 60*n ! estimated using ! 2x the coaxial spacing if the points were arranged in a square real(f), parameter:: cutoff = 2*n**(-1.d0/dim) integer:: npairs, k real(f), allocatable:: x(:, :) integer, allocatable:: pair_i(:), pair_j(:) allocate(x(dim, n), pair_i(maxnpair), pair_j(maxnpair)) ! initialize positions with pseudo-random numbers call random_number(x) ! finding pairs call cellList(dim, n, 27, x, cutoff, maxnpair, npairs, pair_i, pair_j) write (*, \u0026#39;(A)\u0026#39;) \u0026#39;Executing cellList\u0026#39; write (*, \u0026#39;(2x, A,I4,A)\u0026#39;) \u0026#39;Found \u0026#39;, npairs, \u0026#39; pairs\u0026#39; write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;First and last 5 pairs of points found:\u0026#39; write (*, \u0026#39;(2x, 4(A4, 1x))\u0026#39;) \u0026#39;Pair\u0026#39;, \u0026#39;i\u0026#39;, \u0026#39;j\u0026#39; do k = 1, npairs if (k \u0026lt;= 5 .or. k \u0026gt; npairs - 4) write (*, \u0026#39;(2x, 3(I4, 1x))\u0026#39;) k, \u0026amp; pair_i(k), pair_j(k) if (k == 6) write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;...\u0026#39; end do end program main #endif "},{"id":17,"href":"/worknotes/docs/cfdem/","title":"CFDEM","section":"Docs","content":"This covers any work/study involving the CFDEM software, and its component tools: OpenFOAM and LIGGGHTS. Respectively, these software packages perform Computational Fluid Dynamics (CFD) and granular materials (e.g., sand and pills) simulations. LIGGGHTS simulates granular materials using the Discrete Element Method (DEM). CFDEM is the combination of these two tools, and aims to perform coupled CFD-DEM simulations i.e., simulations of granular materials immersed in fluid. Notes below cover all three software packages. Note that these pages only cover the open source versions of CFDEM and LIGGGHTS (OpenFOAM does not have a paid version).\nThis stuff I learnt after I graduated from my masters, and was done under the supervision of Nhu Nguyen in my free time. I only work on this intermittently.\n"},{"id":18,"href":"/worknotes/docs/cuda/reference-codes/error/","title":"Error Handling","section":"nvfortran and nvc++ reference codes","content":" Error Handling # Description # Basic functionality to check errors in CUDA functions and kernel subroutines.\nThe C++ code is very similar to the Fortran code, so I\u0026rsquo;m not including it.\nCode (Fortran) # ! the cuda GetErrorString function can be used to obtain error messages from error codes ierr = cudaGetDeviceCount(nDevices) if (ierr/= cudaSuccess) write (*,*) cudaGetErrorString(ierr) ! kernel errors are checked using cudaGetLastError call increment \u0026lt;\u0026lt;\u0026lt;1,n\u0026gt;\u0026gt;\u0026gt;(a_d , b) ierrSync = cudaGetLastError() ierrAsync = cudaDeviceSynchronize() if (ierrSync /= cudaSuccess) write(*,*) \u0026#39;Sync kernel error\u0026#39;, cudaGetErrorString(ierrSync) if (ierrAsync /= cudaSuccess) write(*,*) \u0026#39;Async kernel error:\u0026#39;, cudaGetErrorString(cudaGetLastError()) "},{"id":19,"href":"/worknotes/docs/useful/grid-rows-spatial-hashing/","title":"grid dimension hashing","section":"Useful code snippets","content":" Hashing grid cell indices based on grid dimensions # Description # This strategy is based on this NVIDIA article. The idea being that instead of storing indices of particles in a grid data structure, you can convert these 3-valued indices to single hashes. These hashes can then be used to sort the particle data so that the particle data is ordered based on their grid cell hash index. This is beneficial for GPUs, which is why it\u0026rsquo;s mentioned in the above article, but is also useful for CPUs as it iterating over the particle pairs more cache-friendly.\nRead the above article for details. The code makes use of the C++ sort function available in the algorithm header.\nAdditional notes # I managed to obtain another 1.4x speedup for 1E5 points over The half-search cell-list pair search when using no optimisations. A speedup of 4x over the previous version when compiling all the codes with -O3 (using total program time, measured roughly with time).\nThe significant speedup obtained is for a few reasons:\nWhen searching adjacent cells, points in 3 consecutive cells can be searched in a single loop. This makes the search much more cache-friendly. Branching is reduced due to the removed if statement when searching through the cell of interest, and the fewer number of loops. In the case of compiling with more aggressive optimisations, the new structure is better able to leverage the compiler\u0026rsquo;s automatic SIMD instructions. The main downside of the code is that now the points are ordered differently from the input. If the order of the points is not critical, then it is better to maintain this new order,as data locality would be much better. However, if necessary, reordering the points can be reversed easily, and with only a modest time penalty.\nAnother benefit is the lower memory usage by the algorithm. This is because it doesn\u0026rsquo;t need to allocate the cells array which is maxpercell*ngridx(1)*ngridx(2)*ngridx(3).\nCode (Fortran and C++) # Save the C++ file as sort.cpp, and Fortran code as cll3.F90. Compile the code with\ng++ -c sort.cpp gfortran -o cll3.x cll3.F90 sort.o -lstdc++ // sort.cpp // C++ source code containing STL sort - modified for sort by indices #include \u0026lt;algorithm\u0026gt; extern \u0026#34;C\u0026#34; { void sort_hashes(int n, int* hashes, int* idx) { // filling idx with numbers 0:n-1 for (int i = 0; i \u0026lt; n; ++i) { idx[i] = i; } // sorting idx based on hashes std::sort(idx, idx + n, [\u0026amp;](const int\u0026amp; a, const int\u0026amp; b) { return (hashes[a] \u0026lt; hashes[b]); } ); // adapting idx for 1-based indexing for (int i = 0; i \u0026lt; n; ++i) { idx[i]++; } } } ! cll3.F90 ! Module to perform direct search for pairs of points within a fixed cutoff. ! cellList subroutine ! dim: An integer defining dimension of the coordinates. ! npoints: An integer with number of points. ! maxpercell: An integer with max number of points in a cell. ! x: An real/real(f) array of dim x points dimension containing the list of points to find pairs of. ! cutoff: The distance (same type as x) which defines a pair ! maxnpair: An integer of the maximum number of pairs expected. ! npairs: An integer with the number of pairs found. ! pair_i: An integer array of maxnpairs length with the \u0026#34;left-sided\u0026#34; point in a pair. ! pair_j: An integer array of maxnpairs length with the \u0026#34;right-sided\u0026#34; point in a pair. module cellList_m use iso_fortran_env, only: error_unit, real32, real64 use iso_c_binding public #ifdef SINGLEPRECISION integer, parameter:: f = real32 #else integer, parameter:: f = real64 #endif interface sort_hashes subroutine sort_hashes(n, hashes, idx) bind(C) use iso_c_binding integer(c_int), value, intent(in):: n integer(c_int), intent(in):: hashes(*) integer(c_int), intent(out):: idx(*) end subroutine sort_hashes end interface sort_hashes contains !--------------------------------------------------------------------------- function coordsToHash(n, dim, x, minx, ngridx, cutoff) result(gridhash) ! converts raw 2/3D positions to gridhashes based on the grid dimensions ! described by minx (lowest grid coordinate), and ngridx (number of grid ! cells in xyz direction). implicit none integer, intent(in):: n, dim, ngridx(3) real(f), intent(in):: x(dim, n), minx(dim), cutoff integer:: i, icell(3) integer(c_int):: gridhash(n) if (dim == 2) icell(3) = 1 do i = 1, n icell(1:dim) = int((x(1:dim, i) - minx(1:dim))/cutoff) + 1 gridhash(i) = ngridx(1)*ngridx(2)*(icell(3) - 1) + ngridx(1)*(icell(2) - 1) + icell(1) end do end function coordsToHash !--------------------------------------------------------------------------- function hashIncrement(ngridx, dcellx, dcelly, dcellz) result(dhash) ! calculates the change in hash, given a change in xyz cell indices implicit none integer, intent(in):: ngridx(3), dcellx, dcelly, dcellz integer:: dhash dhash = ngridx(1)*ngridx(2)*dcellx + ngridx(1)*dcelly + dcellz end function hashIncrement !--------------------------------------------------------------------------- subroutine rearrange(n, dim, idx, gridhash, x) ! rearranges gridhash and x, using the indices obtained from a sort by ! key. implicit none integer, intent(in):: n, dim, idx(n) integer(c_int), intent(inout):: gridhash(n) real(f), intent(inout):: x(dim, n) integer(c_int), allocatable:: tmpgridhash(:) real(f), allocatable:: tmpx(:, :) integer:: i allocate (tmpgridhash(n), tmpx(dim, n)) do i = 1, n tmpgridhash(i) = gridhash(idx(i)) tmpx(:, i) = x(:, idx(i)) end do gridhash(:) = tmpgridhash(:) x(:, :) = tmpx(:, :) end subroutine rearrange !--------------------------------------------------------------------------- subroutine find_starts(n, ngridx, gridhash, starts) ! finds the starting index for each grid cell. See \u0026#34;Building the grid ! using Sorting\u0026#34; section in https://developer.download.nvidia.com/assets/cuda/files/particles.pdf implicit none integer, intent(in):: n, ngridx(3) integer(c_int), intent(in):: gridhash(n) integer, intent(out):: starts(:) integer:: i starts(1:gridhash(1)) = 1 do i = 2, n if (gridhash(i) /= gridhash(i - 1)) starts(gridhash(i - 1) + 1:gridhash(i)) = i end do starts(gridhash(n) + 1:ngridx(1)*ngridx(2)*ngridx(3)) = n + 1 end subroutine find_starts !--------------------------------------------------------------------------- subroutine update_pair_list(cutoff, jstart, jend, i, x, npairs, pair_i, pair_j) ! subroutine to loop over sequential grid cells and check if j point is ! within a given i point implicit none integer, intent(in):: jstart, jend, i real(f), intent(in):: cutoff, x(:, :) integer, intent(inout):: npairs, pair_i(:), pair_j(:) integer:: j real(f):: r2 do j = jstart, jend r2 = sum((x(:, i) - x(:, j))**2) if (r2 \u0026lt;= cutoff*cutoff) then npairs = npairs + 1 pair_i(npairs) = i pair_j(npairs) = j end if end do end subroutine update_pair_list !--------------------------------------------------------------------------- subroutine cellList(dim, npoints, x, cutoff, maxnpair, npairs, pair_i, pair_j) ! main subroutine to perform pair search implicit none integer, intent(in):: dim, npoints, maxnpair real(f), intent(in):: cutoff real(f), intent(inout):: x(dim, npoints) integer, intent(out):: npairs, pair_i(maxnpair), pair_j(maxnpair) integer:: i, hashi, hashj, ngridx(3) real(f):: minx(3), maxx(3) integer(c_int), allocatable:: gridhash(:), idx(:), starts(:) ! determine grid min-max coordinates, with concessions made for dim=2 case minx(1:dim) = minval(x, dim=2) if (dim == 2) minx(3) = 0.d0 maxx(1:dim) = maxval(x, dim=2) if (dim == 2) maxx(3) = 0.d0 ! creating buffer layers minx(:) = minx(:) - 2.d0*cutoff maxx(:) = maxx(:) + 2.d0*cutoff ! determining no. of grid cells and adjusting maximum extent ngridx(:) = int((maxx(:) - minx(:))/cutoff) + 1 maxx(:) = maxx(:) + ngridx(:)*cutoff ! convert coordinates to cell grid hashes allocate (gridhash(npoints)) gridhash = coordsToHash(npoints, dim, x, minx, ngridx, cutoff) ! get sorting indices to reorder points by ascending grid hash allocate (idx(npoints)) call sort_hashes(npoints, gridhash, idx) ! reorder points (x, gridhash) by ascending grid hash call rearrange(npoints, dim, idx, gridhash, x) ! find starting points of each grid cell allocate (starts(product(ngridx))) call find_starts(npoints, ngridx, gridhash, starts) ! perform pair search npairs = 0 do hashi = gridhash(1), gridhash(npoints) ! for all relevant grid cell points do i = starts(hashi), starts(hashi + 1) - 1 ! loop over all points in the cell ! loop over \u0026#34;other\u0026#34; points in cell + next cell (1st adjacent tell: top of hashi cell) call update_pair_list(cutoff, i + 1, starts(hashi + 2) - 1, i, x, npairs, pair_i, pair_j) ! loop over 2nd-4rd adjacent cells (bottom-south-west to top-south-west) hashj = hashi + hashIncrement(ngridx, -1, -1, -1) call update_pair_list(cutoff, starts(hashj), starts(hashj + 3) - 1, i, x, npairs, pair_i, pair_j) ! loop over 5th-7th adjacent cells (bottom-centre-west to top-centre-west) hashj = hashi + hashIncrement(ngridx, -1, 0, -1) call update_pair_list(cutoff, starts(hashj), starts(hashj + 3) - 1, i, x, npairs, pair_i, pair_j) ! loop over 8th-10th adjacent cells (bottom-north-west to top-north-west) hashj = hashi + hashIncrement(ngridx, -1, 1, -1) call update_pair_list(cutoff, starts(hashj), starts(hashj + 3) - 1, i, x, npairs, pair_i, pair_j) ! loop over 11th-13th adjacent cells (bottom-south-centre to top-south-centre) hashj = hashi + hashIncrement(ngridx, 0, -1, -1) call update_pair_list(cutoff, starts(hashj), starts(hashj + 3) - 1, i, x, npairs, pair_i, pair_j) end do end do end subroutine cellList end module cellList_m #ifndef NOMAIN program main use cellList_m, only: f, cellList implicit none integer, parameter:: n = 1e2, dim = 3, maxnpair = 60*n ! estimated using ! 2x the coaxial spacing if the points were arranged in a square real(f), parameter:: cutoff = 2*n**(-1.d0/dim) real(f):: x(dim, n) integer, allocatable:: pair_i(:), pair_j(:) integer:: npairs, k allocate (pair_i(maxnpair), pair_j(maxnpair)) ! initialize positions with pseudo-random numbers call random_number(x) write (*, \u0026#39;(A)\u0026#39;) \u0026#39;Executing cellList\u0026#39; ! finding pairs call cellList(dim, n, x, cutoff, maxnpair, npairs, pair_i, pair_j) write (*, \u0026#39;(2x, A,I4,A)\u0026#39;) \u0026#39;Found \u0026#39;, npairs, \u0026#39; pairs\u0026#39; write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;First and last 5 pairs of points found:\u0026#39; write (*, \u0026#39;(2x, 4(A4, 1x))\u0026#39;) \u0026#39;Pair\u0026#39;, \u0026#39;i\u0026#39;, \u0026#39;j\u0026#39; do k = 1, npairs if (k \u0026lt;= 5 .or. k \u0026gt; npairs - 4) write (*, \u0026#39;(2x, 3(I4, 1x))\u0026#39;) k, \u0026amp; pair_i(k), pair_j(k) if (k == 6) write (*, \u0026#39;(2x, A)\u0026#39;) \u0026#39;...\u0026#39; end do end program main #endif "},{"id":20,"href":"/worknotes/docs/manual-vectorization/sumreduce/","title":"Vector Sum Reduction","section":"Investigating Manual Vectorization for SPH","content":" Vector Sum Reduction # We now we can add two SIMD vectors together element-by-element. However, I\u0026rsquo;m sure you can imagine scenarios where you might want to add all the elements in a vector together i.e., a sum-reduction. This page will explain how to do this with 128-bit and 256-bit vectors the approach is different for each vector width. I would show how to do this with 512-bit vectors, but I don\u0026rsquo;t have a CPU with AVX512 instructions handy.\nReducing 4 floats # base case # We\u0026rsquo;ll assume a 1D array that has every 8 elements reduced into an element of another array:\nvoid reduce_base_sgl(const int nelem, float* a, float* b) { for (int i = 0; i \u0026lt; nelem; i += 8) { for (int j = 0; j \u0026lt; 8; ++j) { b[i/8] += a[i+j]; // assume b is already zeroed } } } The loop tries to mimick cases where many small sum-reductions are performed.\n128-bit vectorization (SSE) # In the first demonstration of vectorizing the sumreduction loop above, we\u0026rsquo;ll make use of a few new functions:\n__m128 v = _mm_set_ss(val); // creates a vector where the lowermost element is val, and the rest are 0 __m128 v = _mm_add_ss(a, b); // adds only the lowermost elements of a and b. The rest are same as a. __m128 v = _mm_hadd_ps(a, b); // performs a \u0026#34;horizontal\u0026#34; add (see explanation below) _mm_store_ss(\u0026amp;a, v); // stores the lower-most value in the vector The _mm_hadd_xx function will be the workhorse of the vectorized reduction. It takes two vectors and adds pairs from each half of both vectors. The first element will be the sum of the second half of the first vector, a2+a3. The second element will be the first half of the first vector, a0+a1. The third and fourth elements will be the same, but for the second input vector.\n// __m128 a = _mm_set_ps(a0, a1, a2, a3); // __m128 b = _mm_set_ps(b0, b1, b2, b3); _mm128 v = _mm_hadd_ps(a, b); // v = [a2+a3, a0+a1, b2+b3, b0+b1]; Because hadd adds two vectors in this manner, it can be executed twice in a row to sum all elements of an __m128 vector:\n// __m128 a = _mm_set_ps(a0, a1, a2, a3); // first horizontal add _mm128 v = _mm_hadd_ps(a, a); // v = [a2+a3, a0+a1, a2+a3, a0+a1] // second horizontal add v = _mm_hadd_ps(v, v); // v = [v2+v3, v0+v1, v2+v3, v0+v1] // = [a2+a3+a0+a1, a2+a3+a0+a1, a2+a3+a0+a1, a2+a3+a0+a1] The double hadd functions results in 3 adds for every 4 elements (the two horizontal adds, and an add to the reduction variable), compared to 8 adds in the serial sum of all the elements in the vector. Implementing these new functions to vectorize our loop:\nvoid reduce_128_sgl(const int nelem, float* a, float* b) { for (int i = 0; i \u0026lt; nelem; i += 8) { __m128 vb = _mm_set_ss(b[i/8]); // initializes a vector with the sum-reduction variable for (int j = 0; j \u0026lt; 8; j += 4) { // start the reduction loop __m128 va = _mm_loadu_ps(\u0026amp;a[i+j]); // load the data va = _mm_hadd_ps(va, va); // first horizontal add va = _mm_hadd_ps(va, va); // second horizontal add vb = _mm_add_ss(va, vb); // add result to the sum-reduction vector variable } _mm_store_ss(\u0026amp;b[i/8], vb); // store the left-most value in the sum-reduction variable } } The use of the __m128 vb vector helps speed up the loop slightly by mitigating the need to increment the float *b array directly in the inner loop. In this case, it reduces the number of stores by one.\nIn the double precision version, it would reduce the stores by 3, since the inner loop would have 4 iterations instead of only 2. Note that with the double precision version, reducing a 128-bit double vector to one value doesn\u0026rsquo;t reduce the number of additions. Consequently, there shouldn\u0026rsquo;t be much difference between codes.\nPerformance (128-bit) # Compilation command (using the Google benchmark framework):\ng++ -o reduce.x reduce.cpp -msse4 -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread -std=c++17 -O2 Single precision # Comparing reduce_base_sgl (from ./reduce_base_sgl.json) to reduce_128_sgl (from ./reduce_128_sgl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------- [reduce_base_sgl vs. reduce_128_sgl]/4096 -0.2329 -0.2329 1625 1247 1621 1243 [reduce_base_sgl vs. reduce_128_sgl]/32768 -0.2644 -0.2644 13565 9978 13531 9953 [reduce_base_sgl vs. reduce_128_sgl]/262144 -0.2627 -0.2625 108413 79935 108135 79748 [reduce_base_sgl vs. reduce_128_sgl]/2097152 -0.2669 -0.2669 872616 639709 870408 638091 [reduce_base_sgl vs. reduce_128_sgl]/16777216 -0.1476 -0.1478 8405297 7164522 8385642 7146262 [reduce_base_sgl vs. reduce_128_sgl]/134217728 -0.1198 -0.1197 68570641 60358462 68394712 60204987 OVERALL_GEOMEAN -0.2179 -0.2179 0 0 0 0 The tests that fit on L1-L3 cache benefit the most with about 1.3x speedup. The largest test shows the worst speedup, at ~1.1x speedup. A 1.3x speedup is about in line with rough estimates, based on the ratios of adds per 8 elements (8/6 = 1.33).\nDouble Precision # Comparing reduce_base_dbl (from ./reduce_base_dbl.json) to reduce_128_dbl (from ./reduce_128_dbl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------ [reduce_base_dbl vs. reduce_128_dbl]/4096 -0.1052 -0.1050 1627 1456 1623 1452 [reduce_base_dbl vs. reduce_128_dbl]/32768 -0.1106 -0.1106 13462 11973 13431 11945 [reduce_base_dbl vs. reduce_128_dbl]/262144 -0.0369 -0.0370 112035 107906 111772 107633 [reduce_base_dbl vs. reduce_128_dbl]/2097152 -0.0386 -0.0386 934021 898008 931648 895718 [reduce_base_dbl vs. reduce_128_dbl]/16777216 +0.0122 +0.0122 12864721 13022105 12831738 12988742 [reduce_base_dbl vs. reduce_128_dbl]/134217728 +0.0102 +0.0102 107276227 108369477 107002000 108088106 OVERALL_GEOMEAN -0.0461 -0.0461 0 0 0 0 As expected, the performance difference is minimal, although there appears to be some meaningful improvements to the test sizes that fit within L1 and L2 cache (i.e., the 4,096 and 32,768 element tests). I\u0026rsquo;m not sure on this, but I am guessing that the improvements are due to SIMD vectors being used to store the intermediate reduction data.\nAnother interesting observation is that the double precision version base version performs faster on the smallest case than the equivalent for the single precision version. This is probably CPU-specific, as I don\u0026rsquo;t observe this on my desktop CPU (Intel i7-10750H).\n256-bit vectorization (AVX) # 256-bit vectors double the vector width before. This means 8 elements in a single precision (__m128) vector, and 4 elements in a double precision vector. For single precision data, the _mm256_hadd_ps function works like:\n// __m128 a = _mm_set_ps(a0, a1, a2, a3, a4, a5, a6, a7); // __m128 b = _mm_set_ps(b0, b1, b2, b3, b4, b5, b6, b7); _mm256 v = _mm256_hadd_ps(a, b); // v = [a0+a1, a2+a3, b0+b1, b2+b3, a4+a5, a6+a7, b4+b5, b6+b7]; If we were to apply this three times, in an attempt to apply the hadd strategy to perform the reduction:\n// __m128 a = _mm_set_ps(a0, a1, a2, a3, a4, a5, a6, a7); // first horizontal add _mm256 v = _mm256_hadd_ps(a, a); // v = [a0+a1, a2+a3, a0+a1, a2+a3, a4+a5, a6+a7, a4+a5, a6+a7]; // second horizontal add _mm256 v = _mm256_hadd_ps(v, v); // v = [v0+v1, v2+v3, v0+v1, v2+v3, v4+v5, v6+v7, v4+v5, v6+v7]; // v = [a0+a1+a2+a3, a0+a1+a2+a3, a0+a1+a2+a3, a0+a1+a2+a3, a4+a5+a6+a7, a4+a5+a6+a7, a4+a5+a6+a7, a4+a5+a6+a7] // third horizontal add _mm256 v = _mm256_hadd_ps(v, v); // v = [v0+v1, v2+v3, v0+v1, v2+v3, v4+v5, v6+v7, v4+v5, v6+v7]; // v [a0+a1+a2+a3+a0+a1+a2+a3, ... , ... , ... , a4+a5+a6+a7+a4+a5+a6+a7, ... , ... , ...] Where the repeated entries are skipped in the last addition. Each element of the bottom 128 bits of the vector have 2*(a0+a1+a2+a3) and the top 128 bits have 2*(a4+a5+a6+a7). Hopefully it\u0026rsquo;s clear that simply performing three hadds in a row would not be possible to perform our reduction, at least for 8 elements of single precision data.\nA similar problem occurs when performing two consecutive hadd operations on double precision data. The _mm256_hadd_pd function works like:\n// __m128d a = _mm256_set_pd(a0, a1, a2, a3); // __m128d b = _mm256_set_pd(b0, b1, b2, b3); _mm256 v = _mm256_hadd_pd(a, b); // v = [a0+a1, b0+b1, a2+a3, b2+b3]; And so trying to apply this twice for the purposes of reduction:\n// __m128d a = _mm256_set_pd(a0, a1, a2, a3); // first horizontal add _mm256 v = _mm256_hadd_pd(a, a); // v = [a0+a1, a0+a1, b2+b3, b2+b3] // second horizontal add _mm256 v = _mm256_hadd_pd(v, v); //v = [v0+v1, v0+v1, v2+v3, v2+v3] // = [a0+a1+a0+a1, ..., a2+a3+a2+a3, ...] Which should illustrate that double _mm256_hadd_pd has the same limitation as _mm256_hadd_ps, as the two halves of the resultant 256-bit vector merely contains double of the sum of all elements in each of the corresponding 128-bit halves.\n__m256 vectors can be reduced by first splitting the vector into two __m128 vectors, adding the two halves together and then performing the 128-bit vector reduction shown previously. For example, for single precision:\nvoid reduce_256_sgl(const int nelem, float* a, float* b) { for (int i = 0; i \u0026lt; nelem; i += 8) { __m128 vb = _mm_set_ss(b[i/8]); // initializes a vector with the sum-reduction variable for (int j = 0; j \u0026lt; 8; j += 8) { // start the reduction loop __m256 va = _mm256_loadu_ps(\u0026amp;a[i+j]); // load the data __m128 va_low = _mm256_castps256_ps128(va); // \u0026#34;truncate\u0026#34; the 256-bit vector to get the lower 128-bit vector __m128 va_hi = _mm256_extractf128_ps(va, 1); // extract the upper 128-bit vector va_low = _mm_add_ps(va_low, va_hi); // add the two halves together va_low = _mm_hadd_ps(va_low, va_low); // first horizontal add va_low = _mm_hadd_ps(va_low, va_low); // second horizontal add vb = _mm_add_ss(va_low, vb); // add result to the sum-reduction vector variable } _mm_store_ss(\u0026amp;b[i/8], vb); // store the left-most value in the sum-reduction variable } } Note that in this verison, 8 elements are loaded at a time into the 256-bit vector, which removes the need for the inner loop. But, the inner loop is kept so that it is more comparable to the base and 128-bit versions.\nTo extract the lower 128-bit part of the 256-bit vector, the _mm256_castps256_ps128 function is used. This function casts an __m256 vector to a __m128 vector by truncating the upper 128-bits from the input vector. The upper half is extracted using _mm256_extractf128_ps. The second argument being 1, is asking for the second 128-bit part of the vector. Passing 0 would provide the first half, and could also be used to extract the lower 128-bit half, but _mm256_castps256_ps128 is apparently faster. The next step is to add these two halves using the _mm_add_ps function as used previously. The remainder of the reduction follows the same double hadd idiom used to reduce the 128-bit single precision vectors.\nThe same strategy can be employed with double precision 256-bit vectors, except that only one _mm_hadd_pd is used, like when performing the reduction with 128-bit double precision vectors.\nPerformance (256-bit) # Compilation command (using the Google benchmark framework):\ng++ -o reduce.x reduce.cpp -mavx -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread -std=c++17 -O2 Single precision # Comparing reduce_base_sgl (from ./reduce_base_sgl.json) to reduce_256_sgl (from ./reduce_256_sgl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New -------------------------------------------------------------------------------------------------------------- [reduce_base_sgl vs. reduce_256_sgl]/4096 -0.5072 -0.5072 1625 801 1621 799 [reduce_base_sgl vs. reduce_256_sgl]/32768 -0.5265 -0.5264 13565 6423 13531 6408 [reduce_base_sgl vs. reduce_256_sgl]/262144 -0.5155 -0.5155 108413 52526 108135 52393 [reduce_base_sgl vs. reduce_256_sgl]/2097152 -0.4869 -0.4868 872616 447734 870408 446687 [reduce_base_sgl vs. reduce_256_sgl]/16777216 -0.2345 -0.2345 8405297 6434629 8385642 6419522 [reduce_base_sgl vs. reduce_256_sgl]/134217728 -0.1776 -0.1776 68570641 56392320 68394712 56245917 OVERALL_GEOMEAN -0.4240 -0.4240 0 0 0 0 Best speedup obtained is now ~2.1x and worst is ~1.2x. Given that for every 8 elements added, there are 4 addition operations (2 horizontal, 1 normal, and 1 to update the reduction variable), you would expect that this implementation would achieve around (8)/4 ~= 2x. speedup over the non-vectorized base version. The better performance for smaller problems might be because normal adds are cheaper than horizontal adds. The minimum speedup of ~1.6x is actually worse than that achieved by the equivalent 128-bit vector version.\nDouble precision # Comparing reduce_base_dbl (from ./reduce_base_dbl.json) to reduce_256_dbl (from ./reduce_256_dbl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------ [reduce_base_dbl vs. reduce_256_dbl]/4096 -0.4083 -0.4083 1627 963 1623 960 [reduce_base_dbl vs. reduce_256_dbl]/32768 -0.3764 -0.3765 13462 8395 13431 8374 [reduce_base_dbl vs. reduce_256_dbl]/262144 -0.1858 -0.1857 112035 91225 111772 91011 [reduce_base_dbl vs. reduce_256_dbl]/2097152 -0.1667 -0.1665 934021 778325 931648 776501 [reduce_base_dbl vs. reduce_256_dbl]/16777216 -0.0118 -0.0116 12864721 12712606 12831738 12682778 [reduce_base_dbl vs. reduce_256_dbl]/134217728 -0.0017 -0.0017 107276227 107096199 107002000 106821947 OVERALL_GEOMEAN -0.2079 -0.2079 0 0 0 0 Here, the speedup is between ~1x and ~1.7x. The maximum speedup is significantly better than the equivalent for 128-bit vectors, but of course, gets worse as data gets more distant from the CPU due to the test sizes. The ~1.7x speedup achieved falls short of the 2x expected, probably because of the extractions of the 128-bit vectors from the 256-bit vector, as well as the relatively more expensive horizontal add operation.\nLike with the single precision version, the test sizes that are in RAM, do not show any meaningful speedup over the 128-bit equivalent. Although it seems that this one is at least not any worse.\nConclusion # The horizontal add intrinsics were introduced here, as well as other needed, to perform vectorized reductions of chunks of data in an array. With single precision data, the speedups achieved for all tests were meaningful. For tests that fit within cache, the achieved speedup was around expectations of using the horizontal add strategy. These results correspond with the upper end of the ranges.\nSingle Double 128-bit 1.1x-1.3x 1x-1.1x 256-bit 1.2x-2.4x 1x-1.7x In contrast, double precision vectorization attempts with 128-bit vectors only achieved 1.1x at best, for problems that fit in cache, and did not obtain any speedup for tests that could only fit in RAM. The 256-bit vectorization attempt improved the upper end to 1.7x, but still failed to improve the speed for tests in RAM.\nFaster sum-reduction examples are shown on this page.\n"},{"id":21,"href":"/worknotes/docs/manual-vectorization/faster-sumreduce/","title":"Faster Vector Sum Reduction","section":"Investigating Manual Vectorization for SPH","content":" Faster Vector Sum Reduce # In my page introducing vectorized reductions, I show how to use horizontal adds (hadd) functions to perform the reduction. However, this doesn\u0026rsquo;t produce optimal assembly, so isn\u0026rsquo;t the fastest way to do things.\nThis page will demonstrate a faster version to do each of the examples shown previously.\nSingle precision, SSE instructions # This section will show a faster sum-reduce example which makes use of 128-bit single precision vectors and SSE instructions.\nAfter loading the vector va, a new vector is created using _mm_movehdup_ps, using a as an input. This function duplicates the second and fourth elements, and duplicates them to one element lower.\n// __m128 va = _mm_set_ps(a0, a1, a2, a3); __m128 shuf = _mm_movehdup_ps(va); // shuf = [a1, a1, a3, a3] The resultant vector is then added back to va:\n__m128 sums = _mm_add_ps(va, shuf); // sums = [a0+a1, a1+a1, a2+a3, a3+a3] shuf and sums are combined together with _mm_movehl_ps. This function replaces the lower two elements of the first input vector, with the upper two elements of the second vector.\nshuf = _mm_movehl_ps(shuf, sums); // shuf = [sums2, sum3, shuf2, shuf3] // = [a2+a3, a3+a3, a3, a3] We finally add the lowermost element of shuf and sums to get the reduced sum in the lowermost element of the sums vector, and the remaing elements can be ignored.\nsums = _mm_add_ss(sums, shuf); // sums = [sums0+shuf0, sums1, sums2, sums3] // = [a0+a1+a2+a3, a1+a1, a2+a3, a3+a3] Implemented into our previous example:\nvoid reduce_128_sgl_SSE3(const int nelem, float* a, float* b) { for (int i = 0; i \u0026lt; nelem; i += 8) { __m128 vb = _mm_set_ss(b[i/8]); for (int j = 0; j \u0026lt; 8; j += 4) { __m128 va = _mm_loadu_ps(\u0026amp;a[i+j]); __m128 shuf = _mm_movehdup_ps(va); __m128 sums = _mm_add_ps(va, shuf); shuf = _mm_movehl_ps(shuf, sums); sums = _mm_add_ss(sums, shuf); vb = _mm_add_ss(vb, sums); } _mm_store_ss(\u0026amp;b[i/8], vb); } } Performance # Compilation command:\ng++ -o reduce.x reduce.cpp -msse4 -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread -std=c++17 -O2 Comparison with base case:\nComparing reduce_base_sgl (from reduce_base_sgl.json) to reduce_128_sgl_SSE3 (from reduce_128_sgl_SSE3.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------- [reduce_base_sgl vs. reduce_128_sgl_SSE3]/4096 -0.6108 -0.6107 1625 633 1621 631 [reduce_base_sgl vs. reduce_128_sgl_SSE3]/32768 -0.6270 -0.6271 13565 5059 13531 5046 [reduce_base_sgl vs. reduce_128_sgl_SSE3]/262144 -0.5984 -0.5983 108413 43541 108135 43439 [reduce_base_sgl vs. reduce_128_sgl_SSE3]/2097152 -0.5380 -0.5379 872616 403151 870408 402208 [reduce_base_sgl vs. reduce_128_sgl_SSE3]/16777216 -0.2613 -0.2613 8405297 6209082 8385642 6194461 [reduce_base_sgl vs. reduce_128_sgl_SSE3]/134217728 -0.2114 -0.2114 68570641 54072282 68394712 53933850 OVERALL_GEOMEAN -0.4997 -0.4996 0 0 0 0 Comparison with horizontal add strategy:\nComparing reduce_128_sgl (from reduce_128_sgl.json) to reduce_128_sgl_SSE3 (from reduce_128_sgl_SSE3.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------ [reduce_128_sgl vs. reduce_128_sgl_SSE3]/4096 -0.4926 -0.4925 1247 633 1243 631 [reduce_128_sgl vs. reduce_128_sgl_SSE3]/32768 -0.4930 -0.4930 9978 5059 9953 5046 [reduce_128_sgl vs. reduce_128_sgl_SSE3]/262144 -0.4553 -0.4553 79935 43541 79748 43439 [reduce_128_sgl vs. reduce_128_sgl_SSE3]/2097152 -0.3698 -0.3697 639709 403151 638091 402208 [reduce_128_sgl vs. reduce_128_sgl_SSE3]/16777216 -0.1334 -0.1332 7164522 6209082 7146262 6194461 [reduce_128_sgl vs. reduce_128_sgl_SSE3]/134217728 -0.1041 -0.1042 60358462 54072282 60204987 53933850 OVERALL_GEOMEAN -0.3602 -0.3602 0 0 0 0 The new vectorized reduction is between 1.1x to 2x faster than the previous attempt using horizontal adds. This brings up the speedup compared to the original to ~1.3x to ~2.6x times. Like every other example, performance gains are greater the closer the data is to the CPU core.\nSingle precision, AVX instructions # This implementation is not much different from the previous version, in that the reduction using a 256-bit vector is performed by splitting the 256-bit vector into 2 128-bit vectors, adding them, and performing the same reduction algorithm for 128-bit vectors:\nvoid reduce_256_sgl_AVX(const int nelem, float* a, float* b) { for (int i = 0; i \u0026lt; nelem; i += 8) { __m128 vb = _mm_set_ss(b[i/8]); for (int j = 0; j \u0026lt; 8; j += 8) { __m256 va = _mm256_loadu_ps(\u0026amp;a[i+j]); __m128 va_lo = _mm256_castps256_ps128(va); __m128 va_hi = _mm256_extractf128_ps(va, 1); va_lo = _mm_add_ps(va_lo, va_hi); __m128 shuf = _mm_movehdup_ps(va_lo); __m128 sums = _mm_add_ps(va_lo, shuf); shuf = _mm_movehl_ps(shuf, sums); sums = _mm_add_ss(sums, shuf); vb = _mm_add_ss(sums, vb); } _mm_store_ss(\u0026amp;b[i/8], vb); } } Performance # First comparing this new version to the first 256-bit vectorized version:\nComparing reduce_256_sgl (from reduce_256_sgl.json) to reduce_256_sgl_AVX (from reduce_256_sgl_AVX.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ----------------------------------------------------------------------------------------------------------------- [reduce_256_sgl vs. reduce_256_sgl_AVX]/4096 -0.1999 -0.2001 800 640 798 638 [reduce_256_sgl vs. reduce_256_sgl_AVX]/32768 -0.1937 -0.1935 6419 5175 6401 5162 [reduce_256_sgl vs. reduce_256_sgl_AVX]/262144 -0.1721 -0.1723 52211 43227 52079 43108 [reduce_256_sgl vs. reduce_256_sgl_AVX]/2097152 -0.0966 -0.0966 446301 403176 445074 402074 [reduce_256_sgl vs. reduce_256_sgl_AVX]/16777216 -0.0417 -0.0417 6538546 6266122 6520420 6248835 [reduce_256_sgl vs. reduce_256_sgl_AVX]/134217728 -0.0436 -0.0436 57544865 55037665 57385371 54886129 OVERALL_GEOMEAN -0.1271 -0.1272 0 0 0 0 The performance has definitely improved, albeit marginally for the test sizes in RAM. But compared to the 128-bit version, it actually performs very similarly (perhaps even a bit worse):\nComparing reduce_128_sgl_SSE3 (from reduce_128_sgl_SSE3.json) to reduce_256_sgl_AVX (from reduce_256_sgl_AVX.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------------------- [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/4096 +0.0101 +0.0102 634 640 632 638 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/32768 +0.0233 +0.0236 5057 5175 5043 5162 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/262144 +0.0210 +0.0208 42337 43227 42231 43108 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/2097152 +0.0043 +0.0041 401449 403176 400435 402074 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/16777216 +0.0047 +0.0045 6236505 6266122 6220706 6248835 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/134217728 -0.0041 -0.0043 55265558 55037665 55122733 54886129 OVERALL_GEOMEAN +0.0099 +0.0098 0 0 0 0 I\u0026rsquo;m not 100% sure why this happens. Firstly, there\u0026rsquo;s an additional _mm_add_ps function call, which eats into potential gains from using 256-bit vectors. And I believe the additional memory operations eat into the remaining gains. The _mm256_castps256_ps128 should be low cost or free as it doesn\u0026rsquo;t move any data around. However, _mm256_extractf128_ps has to copy the data from the top half of the original va vector into a different register. Which, I think, is more costly than an _mm_hadd_ps function call.\nUltimately, from this test, it\u0026rsquo;s probably not worth implementing vector reduction with 256-bit vectors compared to the fast 128-bit version.\nPerformance with LLVM compiler (clang++) # The LLVM compilers are a different family of compilers form the GNU compilers and can translate the C++ code to assembly differently. In the case of C++, we use the clang++ compiler. Let\u0026rsquo;s compare the performance of our test programs using clang++.\nThe compilation command for the manually vectorized code is:\nclang++ -o reduce.x reduce.cpp -mavx2 -Ibenchmark/include -Lbenchmark/build/src -lbenchmark -lpthread -O2 The base case needed additional options to ensure no SSE/AVX instructions were introdued:\n-mno-sse -mno-sse2 -mno-sse3 -mno-sse4 -mno-sse4.1 -mno-sse4.2 -mno-avx -mno-avx2 First comparing the faster version of the 256-bit vector code with the faster version of the 128-bit vector code.\nComparing reduce_128_sgl_SSE3 (from reduce_128_sgl_SSE3-clang.json) to reduce_256_sgl_AVX (from reduce_256_sgl_AVX-clang.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------------------- [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/4096 -0.4445 -0.4445 572 318 571 317 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/32768 -0.4693 -0.4693 4708 2499 4697 2493 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/262144 -0.4916 -0.4916 39230 19943 39138 19896 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/2097152 -0.5283 -0.5283 338017 159451 337229 159080 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/16777216 -0.8006 -0.8006 6395979 1275520 6381010 1272545 [reduce_128_sgl_SSE3 vs. reduce_256_sgl_AVX]/134217728 -0.8162 -0.8162 55513176 10205195 55383675 10181392 OVERALL_GEOMEAN -0.6294 -0.6294 0 0 0 0 The AVX code is significantly faster for all the test sizes. For the smallest problem sizes, the AVX code is 1.8x faster than the new SSE code. Furthermore, the larger the problem gets and further away the data is located from the CPU, the speedups also increase (up to 5.4x at the largest test size!). Comparing to the base case:\nComparing reduce_base_sgl (from reduce_base_sgl-clang.json) to reduce_256_sgl_AVX (from reduce_256_sgl_AVX-clang.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------ [reduce_base_sgl vs. reduce_256_sgl_AVX]/4096 -0.7450 -0.7450 1247 318 1244 317 [reduce_base_sgl vs. reduce_256_sgl_AVX]/32768 -0.7494 -0.7494 9970 2499 9947 2493 [reduce_base_sgl vs. reduce_256_sgl_AVX]/262144 -0.7505 -0.7505 79935 19943 79749 19896 [reduce_base_sgl vs. reduce_256_sgl_AVX]/2097152 -0.7506 -0.7506 639364 159451 637869 159080 [reduce_base_sgl vs. reduce_256_sgl_AVX]/16777216 -0.8318 -0.8318 7581972 1275520 7564185 1272545 [reduce_base_sgl vs. reduce_256_sgl_AVX]/134217728 -0.8375 -0.8375 62785954 10205195 62639140 10181392 OVERALL_GEOMEAN -0.7815 -0.7815 0 0 0 0 We now get a speedup of between 3.9x and 6.2x. The lesson here, is that it is important to benchmark your manually vectorized code with other compilers, as the translation to assembly and the heuristics used to automatically optimise the code can vary.\nDouble precision, SSE instructions # This algorithm tricks the compiler to produce optimal code. I don\u0026rsquo;t understand it, but will explain the specific functions used.\n// __m128d va = _mm_set_pd(a0, a1); __m128 undef = _mm_undefined_ps(); // undef = [?, ?, ?, ?] This function call simply creates a 128-bit single precision vector with undefined values. Explicitly saying the vector is undefined apparently helps the compiler optimise the code better.\n__m128 shuftmp = _mm_movehl_ps(undef, _mm_castpd_ps(va)); // _mm_castpd_ps(va) = [a0, a0.5, a1, a1.5] // shuftmp = [a1, a1.5, ?, ?] I found this pretty clever when I saw it. The motivation for this trick is that there is no _mm_movehl_ps equivalent for double precision data. What happens here is:\nva is cast to a single precision vector. This doesn\u0026rsquo;t change the data itself. It only changes how functions interepret it. i.e., the 128 bits in the vector are still the same. The upper 64 bits of va (i.e., the second element), is placed in the first element position of undef. The resultant vector is stored in shuftmp. This effectively moves the 2nd element of va into the first place of undef, despite not having an equivalent movehl function for double precision. This will be converted back into a double precision vector next.\n__m128d shuf = _mm_castps_pd(shuftmp) // shuf = [a1, ?] _mm_castps_pd converts shuftmp back into a double precision vector, which is saved as shuf.\nva = _mm_add_sd(va, shuf) // va = [a0+a1, ?] which has the reduced sum in the lower value of va. Implemented into the previous example:\nvoid reduce_128_sgl_SSE2(const int nelem, float* a, float* b) { for (int i = 0; i \u0026lt; nelem; i += 8) { __m128d vb = _mm_set_sd(b[i/8]); for (int j = 0; j \u0026lt; 8; j += 2) { __m128 va = _mm_loadu_ps(\u0026amp;a[i+j]); __m128 undef = _mm_undefined_ps(); __m128 shuftmp = _mm_movehl_ps(undef, _mm_castpd_ps(va)); __m128d shuf = _mm_castps_pd(shuftmp); va = _mm_add_sd(va, shuf); vb = _mm_add_sd(vb, va); } _mm_store_sd(\u0026amp;b[i/8], vb); } } Performance # Compilation command:\ng++ -o reduce.x reduce.cpp -msse4 -isystem benchmark/include -Lbenchmark/build/src -lbenchmark -lpthread -std=c++17 -O2 Comparison with base case:\nComparing reduce_base_dbl (from reduce_base_dbl.json) to reduce_128_dbl_SSE2 (from reduce_128_dbl_SSE2.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ----------------------------------------------------------------------------------------------------------------------- [reduce_base_dbl vs. reduce_128_dbl_SSE2]/4096 -0.1979 -0.1978 1627 1305 1623 1302 [reduce_base_dbl vs. reduce_128_dbl_SSE2]/32768 -0.1842 -0.1843 13462 10982 13431 10956 [reduce_base_dbl vs. reduce_128_dbl_SSE2]/262144 -0.1666 -0.1667 112035 93353 111772 93135 [reduce_base_dbl vs. reduce_128_dbl_SSE2]/2097152 -0.1767 -0.1765 934021 768984 931648 767179 [reduce_base_dbl vs. reduce_128_dbl_SSE2]/16777216 -0.0174 -0.0172 12864721 12640848 12831738 12611107 [reduce_base_dbl vs. reduce_128_dbl_SSE2]/134217728 -0.0080 -0.0078 107276227 106417505 107002000 106168603 OVERALL_GEOMEAN -0.0729 -0.0724 0 0 0 0 The gains are modest, with a speedup of 1.2x at most. Comparison with horizontal add strategy:\nComparing reduce_128_dbl (from reduce_128_dbl.json) to reduce_128_dbl_SSE2 (from reduce_128_dbl_SSE2.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------------------- [reduce_128_dbl vs. reduce_128_dbl_SSE2]/4096 -0.1037 -0.1033 1456 1305 1452 1302 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/32768 -0.0828 -0.0828 11973 10982 11945 10956 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/262144 -0.1349 -0.1347 107906 93353 107633 93135 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/2097152 -0.1437 -0.1435 898008 768984 895718 767179 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/16777216 -0.0293 -0.0291 13022105 12640848 12988742 12611107 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/134217728 -0.0180 -0.0178 108369477 106417505 108088106 106168603 OVERALL_GEOMEAN -0.0667 -0.0991 0 0 0 0 Which shows that this faster version is 1.1-1.4x faster than the horizontal add version.\nPerformance with LLVM compiler (clang++) # Like with the single precision AVX code, we can see whether using clang++ changes performance. The compiler options remain the same for the manually vectorized code:\nclang++ -o reduce.x reduce.cpp -mavx2 -Ibenchmark/include -Lbenchmark/build/src -lbenchmark -lpthread -O2 The base case needed additional options to ensure no SSE/AVX instructions were introdued:\n-mno-sse -mno-sse2 -mno-sse3 -mno-sse4 -mno-sse4.1 -mno-sse4.2 -mno-avx -mno-avx2 Comparing reduce_128_dbl (from reduce_128_dbl-clang.json) to reduce_128_dbl_SSE2 (from reduce_128_dbl_SSE2-clang.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------------------- [reduce_128_dbl vs. reduce_128_dbl_SSE2]/4096 -0.4131 -0.4131 1247 732 1244 730 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/32768 -0.0490 -0.0490 9970 9482 9946 9459 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/262144 +0.0132 +0.0132 84054 85161 83857 84962 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/2097152 +0.0415 +0.0416 692974 721728 691305 720034 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/16777216 -0.0231 -0.0230 12876249 12578212 12844822 12548782 [reduce_128_dbl vs. reduce_128_dbl_SSE2]/134217728 -0.0154 -0.0154 108116751 106446645 107853527 106196771 OVERALL_GEOMEAN -0.0904 -0.0903 0 0 0 0 And now it looks like the state of things have improved! This new version, compiled with clang++, shows an improvement, at least for the smallest test size. The others show a minor change, again demonstrating that it\u0026rsquo;s important to test your code with multiple compilers!\nDouble precision, AVX instructions # Like the 256-bit vectorized code so far, the pattern is to split the 256-bit vector into two 128-bit vectors, add the two halves together, and then perform a 128-bit sum reduction. Using the faster version of the single precision 128-bit reduction, the code looks like:\nvoid reduce_256_dbl_AVX(const int nelem, double* a, double* b) { for (int i = 0; i \u0026lt; nelem; i += 8) { __m128d vb = _mm_set_sd(b[i/8]); for (int j = 0; j \u0026lt; 8; j += 4) { __m256d va = _mm256_loadu_pd(\u0026amp;a[i+j]); __m128d va_low = _mm256_castpd256_pd128(va); __m128d va_hi = _mm256_extractf128_pd(va, 1); va_low = _mm_add_pd(va_low, va_hi); __m128 undef = _mm_undefined_ps(); __m128 shuftmp = _mm_movehl_ps(undef, _mm_castpd_ps(va_low)); __m128d shuf = _mm_castps_pd(shuftmp); va_low = _mm_add_sd(va_low, shuf); vb = _mm_add_sd(vb, va_low); } _mm_store_sd(\u0026amp;b[i/8], vb); } } Performance # Comparing reduce_256_dbl (from reduce_256_dbl.json) to reduce_256_dbl_AVX (from reduce_256_dbl_AVX.json) Benchmark Time CPU Time Old Time New CPU Old CPU New --------------------------------------------------------------------------------------------------------------------- [reduce_256_dbl vs. reduce_256_dbl_AVX]/4096 -0.1645 -0.1645 956 799 954 797 [reduce_256_dbl vs. reduce_256_dbl_AVX]/32768 -0.1435 -0.1435 8371 7170 8351 7153 [reduce_256_dbl vs. reduce_256_dbl_AVX]/262144 -0.0466 -0.0466 78961 75280 78776 75103 [reduce_256_dbl vs. reduce_256_dbl_AVX]/2097152 -0.0662 -0.0662 659969 616299 658426 614846 [reduce_256_dbl vs. reduce_256_dbl_AVX]/16777216 -0.0201 -0.0201 12603210 12350490 12573727 12321468 [reduce_256_dbl vs. reduce_256_dbl_AVX]/134217728 -0.0169 -0.0169 106022048 104234811 105773315 103990081 OVERALL_GEOMEAN -0.0781 -0.0781 0 0 0 0 The improvement here is meaningful - up to ~16% for smaller problems. When compiled with clang++, the performance increase is more modest.\nConclusion # The horizontal add strategy for reducing vectors is acceptable, but by using more subtle choices in vectorization, the sum reduction can be improved significantly. This was most evident in the 128-bit single precision sum reduction shown here. The improved algorithm shown on this page, was better than the horizontal add version by up to 2x!\nThe improved 256-bit version, obtained better performance over the horizontal add version, but only if the clang++ compiler is used. With clang++, the new 256-bit version is approximately between 2x and 5x faster than even the updated 128-bit sum reduction.\nIn contrast, the double precision 128-bit and 256-bit improved versions showed here showed modest improvements compared to the previous vectorized version. Both versions seemed to be a lot faster for the smallest test size when compiled with clang++ (around 2x faster).\n"},{"id":22,"href":"/worknotes/docs/cuda/reference-codes/limitingFactor/","title":"limitingFactor","section":"nvfortran and nvc++ reference codes","content":" limitingFactor # Description # Code to test whether computation or memory transfer is the bottleneck. Compiled program intended to be run with nvprof.\nThe book demonstrates the effect of compiling with -Mcuda=fastmath, which shows a significant speedup in the \u0026ldquo;base\u0026rdquo; and \u0026ldquo;math\u0026rdquo; kernels (note they use very old C2050 and K20 GPUs).\nCode (C++) # #include \u0026lt;stdio.h\u0026gt; __global__ void base(float *a, float *b) { int i = blockIdx.x * blockDim.x + threadIdx.x; a[i] = sin(b[i]); } __global__ void memory(float *a, float *b) { int i = blockIdx.x * blockDim.x + threadIdx.x; a[i] = b[i]; } __global__ void math(float *a, float b, int flag) { int i = blockIdx.x * blockDim.x + threadIdx.x; float v = sin(b); if (v*flag == 1.0) a[i] = v; } // this exists because cudaMemSet is weird __global__ void setval(float *a, float val) { int i = blockIdx.x * blockDim.x + threadIdx.x; a[i] = val; } int main() { const int n = 8*1024*1024, blockSize = 256; float *a, *a_d, *b_d; a = (float*)malloc(n*sizeof(float)); cudaMalloc(\u0026amp;a_d, n*sizeof(float)); cudaMalloc(\u0026amp;b_d, n*sizeof(float)); setval\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(b_d, 1.0); base\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, b_d); memory\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, b_d); math\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, 1.0, 0); cudaMemcpy(a_d, a, n*sizeof(float), cudaMemcpyDeviceToHost); printf(\u0026#34;%f\\n\u0026#34;, a[0]); } Code (Fortran) # module kernel_m contains attributes(global) subroutine base(a,b) real:: a(*), b(*) integer:: i i = (blockIdx%x-1) * blockDim%x + threadIdx%x a(i) = sin(b(i)) end subroutine base attributes(global) subroutine memory(a,b) real:: a(*), b(*) integer:: i i = (blockIdx%x-1) * blockDim%x + threadIdx%x a(i) = b(i) end subroutine memory attributes(global) subroutine math(a, b, flag) real:: a(*) real, value:: b integer, value:: flag real:: v integer:: i i = (blockIdx%x-1)*blockDim%x + threadIdx%x v = sin(b) if (v*flag == 1) a(i) = v end subroutine math end module kernel_m program limitingFactor use cudafor use kernel_m implicit none integer, parameter:: n=8*1024*1024, blockSize=256 real:: a(n) real, device:: a_d(n), b_d(n) b_d = 1. call base\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, b_d) call memory\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, b_d) call math\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, 1.0, 0) a = a_d write(*,*) a(1) end program limitingFactor "},{"id":23,"href":"/worknotes/docs/manual-vectorization/sweep/","title":"Vectorizing A Simple Pair Sweep","section":"Investigating Manual Vectorization for SPH","content":" Vectorizing A Simple Pair Sweep # One way to perform the force calculation sweep in SPH, is to use a double loop like:\nfor (int i = 0; i \u0026lt; nelem-1; ++i) { for (int j = i+1; j \u0026lt; nelem; ++j) { // compare particle i and j // calculate forces } } This is generally not a preferred way to do the force calculations, as the number of operations in this algorithm scales with the square of the number of particles. But it\u0026rsquo;s useful for playing around with manual vectorization.\nThe base case we will use is this:\nvoid sweep_base_sgl(const int nelem, float* a, float* b) { int k = 0 for (int i = 0; i \u0026lt; nelem-1; i += 8) { for (int j = i+1; j \u0026lt; nelem; ++j) { tmp = a[i]-a[j]; b[i] += tmp; b[j] -= tmp; } } } i is incremented by 8 to simplify vectorization. Swap out float with double, to get the double precision version.\nImportant to note here, is that this loop cannot automatically be vectorized by g++. For example, if we compile our code with -O3 -fopt-info-vec-missed, -O3 will try to vectorize the loop, and the other option will let you know when it cannot vectorize a given loop. At compilation, I get some important info in the output:\nsweep.cpp:51:27: missed: couldn\u0026#39;t vectorize loop sweep.cpp:51:27: missed: not vectorized: multiple nested loops. sweep.cpp:51:27: missed: couldn\u0026#39;t vectorize loop sweep.cpp:51:27: missed: not vectorized: inner-loop count not invariant. sweep.cpp:52:31: missed: couldn\u0026#39;t vectorize loop Which explains that the compiler missed the vectorization of these loops because:\nthe loops are nested, and the inner loop looping variable, j, is not constant for every loop. Hence, this is a meaningful candidate to vectorize. Why aligned store and load functions can\u0026rsquo;t be used # Recalling what is required for the store and load functions to work: that the memory address being accessed must be aligned on vector-width boundaries. In the nested loop considered here, i starts at element 0, but j starts at i + 1, which would be j = 1 when i = 0. So while the address of a[0] would be aligned to a vector width, the next element, a[1], would not be. Consequently, once the inner loop starts, the load/store functions would fail with a segmentation fault error. More generally, the outer loop would typically be incrementing by 1, which would mean accesses to a would mostly be unaligned.\nDoing the Vectorization # Vectorizing the direct pair sweep with 128-bit vectors (SSE instructions) # This attempt to vectorize the loop will turn the outer loop into a vector of all the same variable, and then the inner loop will load multiple sequential elements to operate the outer-loop vector. This looks like:\nvoid sweep_128_sgl(const int nelem, float* a, float* b) { float tmp[4]; for (int i = 0; i \u0026lt; nelem-1; i += 8) { __m128 vi = _mm_set1_ps(a[i]); for (int j = i+1; j \u0026lt; nelem; j += 4) { __m128 vj = _mm_loadu_ps(\u0026amp;a[j]); __m128 vdiff = _mm_sub_ps(vi, vj); __m128 vbj = _mm_loadu_ps(\u0026amp;b[j]); vbj = _mm_sub_ps(vbj, vdiff); _mm_storeu_ps(\u0026amp;b[j], vbj); _mm_storeu_ps(tmp, vdiff); b[i] += tmp[0]+tmp[1]+tmp[2]+tmp[3]; } } } Now this code looks more complicated than the ABC addition example used earlier.\n__m128 vi = _mm_set1_ps(a[i]); is creating a 128-bit vector to hold 4 single precision elements. These elements are all initialized with the value at a[i]. This vector is used for the entire duration of the inner loop. __m128 vj = _mm_loadu_ps(\u0026amp;a[j]); loads the 4 following single precision elements at the location of a[j]. __m128 vdiff = _mm_sub_ps(vi, vj); subtracts the vj vector from the vi vector and stores it in the vdiff vector. This is equivalent to the tmp = a[i]-a[j]; step in the base code. __m128 vbj = _mm_loadu_ps(\u0026amp;b[j]); loads the 4 following elements at the location of b[j]. This is so that the vdiff vector can be subtracted. vbj = _mm_sub_ps(vbj, vdiff); subtracts vdiff from vbj. This is equivalent to b[j] -= tmp; in the base code. _mm_storeu_ps(\u0026amp;b[j], vbj); takes the calculates vector and places it back in the location at b[j]. _mm_storeu_ps(tmp, vdiff); stores the calculated difference vector in a temporary array of 4 elements. This is so that the elements of the vector can be serially added to b[i]. b[i] += tmp[0]+tmp[1]+tmp[2]+tmp[3]; serially adds the 4 elements of the difference vector to b[i]. This is equivalent to b[i] += tmp; in the base code. The 256-bit vector version can be obtained by replacing __m128 types with __m256, and _mm function prefixes with _mm256. Double precision versions are obtained by adding d to the vector types, and replacing ps with pd in the function suffixes.\nPerformance of the manually vectorized sweep function: with -O2 optimizations # Here, compiler optimizations are used as it has been established that automatic vectorization doesn\u0026rsquo;t work for this code, and these optimizations are needed to get the best out of the manual vectorization. The tests used here are much smaller due to the long duration of this sweep algorithm. At worst, the loaded arrays are stored within L2 cache.\nComparing sweep_base_sgl (from results/sweep1d_base_sgl.json) to sweep_128_sgl (from results/sweep1d_128_sgl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------- [sweep_base_sgl vs. sweep_128_sgl]/4096 -0.7448 -0.7448 2837605 724git074 2830970 722381 [sweep_base_sgl vs. sweep_128_sgl]/8192 -0.7433 -0.7433 11412353 2929379 11385665 2922530 [sweep_base_sgl vs. sweep_128_sgl]/16384 -0.7411 -0.7411 45808257 11859336 45701186 11831605 [sweep_base_sgl vs. sweep_128_sgl]/32768 -0.7412 -0.7412 183443120 47483586 183014137 47372294 OVERALL_GEOMEAN The manual vectorization with the compiler optimizations are about 3.9x faster than the code without the manual vectorization for 128-bit vectors. Near-optimal speedup is gained for 256-bit vectors with a speedup of ~7.3x (see table below).\nComparing sweep_base_sgl (from results/sweep1d_base_sgl.json) to sweep_256_sgl (from results/sweep1d_256_sgl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------- [sweep_base_sgl vs. sweep_256_sgl]/4096 -0.8603 -0.8603 2837605 396449 2830970 395517 [sweep_base_sgl vs. sweep_256_sgl]/8192 -0.8623 -0.8623 11412353 1571707 11385665 1568032 [sweep_base_sgl vs. sweep_256_sgl]/16384 -0.8589 -0.8589 45808257 6462080 45701186 6446971 [sweep_base_sgl vs. sweep_256_sgl]/32768 -0.8454 -0.8454 183443120 28365528 183014137 28299199 OVERALL_GEOMEAN -0.8569 -0.8569 0 0 0 0 Although not shown here, without the compiler optimizations, the speedup of the manually vectorized code with 128-bit vectors is at best, ~1.5x faster; and ~2.7x faster for 256-bit vectors.\nThe double precision versions of the code also obtain close to optimal performance gain using 128-bit vectors. Like the single precision version,the double precision data managed to obtain very close to optimal (3.78x).\nComparing sweep_base_dbl (from results/sweep1d_base_dbl.json) to sweep_128_dbl (from results/sweep1d_128_dbl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------- [sweep_base_dbl vs. sweep_128_dbl]/4096 -0.4898 -0.4898 2837990 1448083 2831355 1444697 [sweep_base_dbl vs. sweep_128_dbl]/8192 -0.4868 -0.4868 11418819 5860700 11392117 5846937 [sweep_base_dbl vs. sweep_128_dbl]/16384 -0.4857 -0.4857 45805303 23556194 45698184 23501027 [sweep_base_dbl vs. sweep_128_dbl]/32768 -0.4882 -0.4882 184497440 94427345 184065995 94206550 OVERALL_GEOMEAN -0.4876 -0.4876 0 0 0 0 Comparing sweep_base_dbl (from results/sweep1d_base_dbl.json) to sweep_256_dbl (from results/sweep1d_256_dbl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------- [sweep_base_dbl vs. sweep_256_dbl]/4096 -0.7354 -0.7354 2837990 750796 2831355 749041 [sweep_base_dbl vs. sweep_256_dbl]/8192 -0.7337 -0.7337 11418819 3040832 11392117 3033721 [sweep_base_dbl vs. sweep_256_dbl]/16384 -0.7331 -0.7331 45805303 12227361 45698184 12198775 [sweep_base_dbl vs. sweep_256_dbl]/32768 -0.7253 -0.7253 184497440 50679024 184065995 50560536 OVERALL_GEOMEAN -0.7319 -0.7319 0 0 0 0 While not shown here, without the compiler optimisations, the 128-bit double precision manually vectorized code is up to 30% slower with 128-bit vectors, and at best, 1.62x faster with 256-bit vectors.\nYou may have noticed that the incrementing of b[i] looks like it\u0026rsquo;s serialized, but near-optimal performance is obtained in the tests. This is because g++ and clang++ will automatically convert b[i] += tmp[0] + ...; to assembly using SIMD instructions. I couldn\u0026rsquo;t figure out a way to prevent the compilers from doing this without also making the intel intrinsic functions unavailable as well. In principle, the serializations of the update of b[i] would prevent optimal performance, and the manually vectorized reductions discussed here and here would be needed to improve performance.\n2D version # A meaningful version of this sweep function would require at least 2 dimensions as few engineering problems occur in 1D. Updating the sweep function to work in 2D:\nvoid sweep_base_sgl(const int nelem, float* ax, float*, ay, float* bx, float* by) { int k = 0 for (int i = 0; i \u0026lt; nelem-1; i += 8) { for (int j = i+1; j \u0026lt; nelem; ++j) { tmp = ax[i]-ax[j]; bx[i] += tmp; bx[j] -= tmp; tmp = ay[i]-ay[j]; by[i] += tmp; by[j] -= tmp; } } } And the corresponding manually vectorized version (128-bit vectors, single precision data):\nvoid sweep_128_sgl(const int nelem, float* ax, float *ay, float* bx, float* by,) { float tmp[4]; for (int i = 0; i \u0026lt; nelem-1; i += 8) { __m128 vix = _mm_set1_ps(ax[i]); __m128 viy = _mm_set1_ps(ay[i]); for (int j = i+1; j \u0026lt; nelem; j += 4) { __m128 vj = _mm_loadu_ps(\u0026amp;ax[j]); __m128 vdiff = _mm_sub_ps(vix, vj); __m128 vbj = _mm_loadu_ps(\u0026amp;bx[j]); vbj = _mm_sub_ps(vbj, vdiff); _mm_storeu_ps(\u0026amp;bx[j], vbj); _mm_storeu_ps(tmp, vdiff); bx[i] += tmp[0]+tmp[1]+tmp[2]+tmp[3]; vj = _mm_loadu_ps(\u0026amp;ay[j]); vdiff = _mm_sub_ps(viy, vj); vbj = _mm_loadu_ps(\u0026amp;by[j]); vbj = _mm_sub_ps(vbj, vdiff); _mm_storeu_ps(\u0026amp;by[j], vbj); _mm_storeu_ps(tmp, vdiff); by[i] += tmp[0]+tmp[1]+tmp[2]+tmp[3]; } } } Which differs from the 1D version by basically duplicating the code to work on both ax/bx and ay/by.\nPerformance of the 2D version # Performance of the 128-bit, single precision version:\nComparing sweep_base_sgl (from results/sweep2d_base_sgl.json) to sweep_128_sgl (from results/sweep2d_128_sgl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------- [sweep_base_sgl vs. sweep_128_sgl]/4096 -0.7374 -0.7374 2872377 754223 2865660 752460 [sweep_base_sgl vs. sweep_128_sgl]/8192 -0.7342 -0.7342 11506013 3057812 11479107 3050661 [sweep_base_sgl vs. sweep_128_sgl]/16384 -0.7311 -0.7311 46017767 12372116 45909943 12343182 [sweep_base_sgl vs. sweep_128_sgl]/32768 -0.7296 -0.7296 184173481 49800358 183742778 49683899 OVERALL_GEOMEAN -0.7331 -0.7331 0 0 0 0 and the double precision version:\nComparing sweep_base_sgl (from results/sweep2d_base_sgl.json) to sweep_128_dbl (from results/sweep2d_128_dbl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------ [sweep_base_sgl vs. sweep_128_dbl]/4096 -0.4732 -0.4732 2872377 1513260 2865660 1509721 [sweep_base_sgl vs. sweep_128_dbl]/8192 -0.4439 -0.4439 11506013 6397967 11479107 6383008 [sweep_base_sgl vs. sweep_128_dbl]/16384 -0.4514 -0.4514 46017767 25243093 45909943 25184062 [sweep_base_sgl vs. sweep_128_dbl]/32768 -0.4497 -0.4497 184173481 101358663 183742778 101121153 OVERALL_GEOMEAN -0.4547 -0.4547 0 0 0 0 The single precision version gets a max speedup of 3.8x and the double precision version gets 1.9x which are both pretty close to optimal! Now what about the 256-bit version?\nsingle precision:\nComparing sweep_base_sgl (from results/sweep2d_base_sgl.json) to sweep_256_sgl (from results/sweep2d_256_sgl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------- [sweep_base_sgl vs. sweep_256_sgl]/4096 -0.7104 -0.7104 2872377 831729 2865660 829784 [sweep_base_sgl vs. sweep_256_sgl]/8192 -0.7079 -0.7079 11506013 3361005 11479107 3353127 [sweep_base_sgl vs. sweep_256_sgl]/16384 -0.7016 -0.7016 46017767 13732233 45909943 13700120 [sweep_base_sgl vs. sweep_256_sgl]/32768 -0.6624 -0.6624 184173481 62176512 183742778 62031113 OVERALL_GEOMEAN -0.6962 -0.6962 0 0 0 0 and double precision:\nComparing sweep_base_sgl (from results/sweep2d_base_sgl.json) to sweep_256_dbl (from results/sweep2d_256_dbl.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------ [sweep_base_sgl vs. sweep_256_dbl]/4096 -0.4926 -0.4926 2872377 1457335 2865660 1453927 [sweep_base_sgl vs. sweep_256_dbl]/8192 -0.4868 -0.4868 11506013 5905197 11479107 5891388 [sweep_base_sgl vs. sweep_256_dbl]/16384 -0.4857 -0.4857 46017767 23664846 45909943 23609391 [sweep_base_sgl vs. sweep_256_dbl]/32768 -0.4381 -0.4381 184173481 103495512 183742778 103253479 OVERALL_GEOMEAN -0.4762 -0.4762 0 0 0 0 Now the speedup has barely changed, and has, in fact, gotten slower for the single precision version! Profiling this code reveals that a big chunk of time is spent on the loading of the vdiff vector to the tmp variable and then adding 8 (single precision) or 4 (double precision) elements to bx/by[i].\nSwapping out the code would change the load and add combination:\n_mm_storeu_ps(tmp, vdiff); bx[i] += tmp[0]+tmp[1]+tmp[2]+tmp[3]; and replacing it with something like\nbx[i] += reduce_256_sgl(vdiff); where reduce_256_sgl, uses the function definitions:\nstatic inline __attribute__((always_inline)) float reduce_128_sgl(__m128 a) { __m128 shuf = _mm_movehdup_ps(a); __m128 sums = _mm_add_ps(a, shuf); shuf = _mm_movehl_ps(shuf, sums); sums = _mm_add_ss(sums, shuf); return _mm_cvtss_f32(sums); } static inline __attribute__((always_inline)) float reduce_256_sgl(__m256 a) { __m128 a_lo = _mm256_castps256_ps128(a); __m128 a_hi = _mm256_extractf128_ps(a, 1); a_lo = _mm_add_ps(a_lo, a_hi); return reduce_128_sgl(a_lo); } where inline __attribute__((always_inline)) ensures that the compiler inlines the function no matter what. The double precision version would be that described in the faster sum reductions page. The performance results from using these functions for the single precision version:\nComparing sweep_base_sgl (from results/sweep2d_base_sgl.json) to sweep_256_sgl_reduce2 (from results/sweep2d_256_sgl_reduce2.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------- [sweep_base_sgl vs. sweep_256_sgl_reduce2]/4096 -0.8014 -0.8014 2872377 570401 2865660 569065 [sweep_base_sgl vs. sweep_256_sgl_reduce2]/8192 -0.7940 -0.7940 11506013 2370599 11479107 2365055 [sweep_base_sgl vs. sweep_256_sgl_reduce2]/16384 -0.7642 -0.7642 46017767 10848874 45909943 10823504 [sweep_base_sgl vs. sweep_256_sgl_reduce2]/32768 -0.7883 -0.7883 184173481 38997633 183742778 38906450 OVERALL_GEOMEAN -0.7874 -0.7874 0 0 0 0 which raises the performance from ~3.5x to ~5x! Not optimal, but a big improvement, and potentially providing sufficient motivation for the 256-bit manual vectorisation. The double precision performance results also show a similar improvement, raising the speedup from ~2x to ~2.4x.\nComparing sweep_base_sgl (from results/sweep2d_base_sgl.json) to sweep_256_dbl_reduce2 (from results/sweep2d_256_dbl_reduce2.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ------------------------------------------------------------------------------------------------------------------ [sweep_base_sgl vs. sweep_256_dbl_reduce2]/4096 -0.5912 -0.5912 2872377 1174242 2865660 1171496 [sweep_base_sgl vs. sweep_256_dbl_reduce2]/8192 -0.5372 -0.5372 11506013 5324447 11479107 5311997 [sweep_base_sgl vs. sweep_256_dbl_reduce2]/16384 -0.5334 -0.5334 46017767 21473573 45909943 21423354 [sweep_base_sgl vs. sweep_256_dbl_reduce2]/32768 -0.5680 -0.5680 184173481 79555471 183742778 79369428 OVERALL_GEOMEAN -0.5581 -0.5581 0 0 0 0 Performance of 3D version # For the sake of completeness, we\u0026rsquo;ll include the 3D performance numbers too. Hopefully, from the 2D version, you\u0026rsquo;ll understand that this is just duplicating some code a second time. The results are summarized below for the sake of brevity.\nCase Min Speedup Over Base (g++ 10.3) Max Speedup Over Base (g++ 10.3) Min Speedup Over Base (clang++ 12.0.1) Max Speedup Over Base (clang++ 12.0.1) 128_sgl_reduce0 2.24 2.61 2.66 3.24 128_sgl_reduce1 2.35 2.59 2.69 3.24 128_sgl_reduce2 2.43 2.68 2.76 3.35 128_dbl_reduce0 1.32 1.38 1.42 1.69 128_dbl_reduce1 1.33 1.38 1.48 1.68 128_dbl_reduce2 1.34 1.38 1.42 1.53 256_sgl_reduce0 1.97 2.18 2.55 2.80 256_sgl_reduce1 2.56 2.79 3.05 3.49 256_sgl_reduce2 2.59 2.87 2.94 3.64 256_dbl_reduce0 1.44 1.49 1.49 1.76 256_dbl_reduce1 1.45 1.50 1.69 1.85 256_dbl_reduce2 1.45 1.49 1.75 1.91 The 3D version shows significantly less speedup compared its 2D and 1D counterparts using both g++ and clang++ compilers. The best speedup obtained speedups are ~3.6x and ~1.9x for the single and double precision versions, respectively. The speedup of using 256-bit vectors is comparable to using 128-bit vectors. But, despite the lesser performance of the 3D version, it is still probably worth manually vectorizing this algorith, given the inability of the Clang and GNU compilers from automatically vectorizing the nested loop (at the time of writing).\nCompleting the Sweep # The Sweep algorithm discussed so far, takes shortcuts to mitigate the discussion of dealing with \u0026ldquo;residuals\u0026rdquo; - the leftover itrations. Just for reference, the double precision 256-bit manually vectorized version of a full sweep, taking care of the residuals, is shown below:\nvoid sweep_256_dbl_reduce2(const int nelem, double* ax, double *ay, double* az, double* bx, double* by, double* bz) { for (int i = 0; i \u0026lt; nelem-1; i += 8) { // loading ith value into vector __m256d vix = _mm256_set1_pd(ax[i]); __m256d viy = _mm256_set1_pd(ay[i]); __m256d viz = _mm256_set1_pd(az[i]); int len = nelem-(i+1); // number of iterations in inner loop int end = (len % 4 == 0) ? nelem : nelem - 4; // truncating inner loop int j; // declaring counter variable for (j = i+1; j \u0026lt; nelem; j += 4) { __m256d vj = _mm256_loadu_pd(\u0026amp;ax[j]); __m256d vdiff = _mm256_sub_pd(vix, vj); __m256d vbj = _mm256_loadu_pd(\u0026amp;bx[j]); vbj = _mm256_sub_pd(vbj, vdiff); _mm256_storeu_pd(\u0026amp;bx[j], vbj); bx[i] += reduce_256_dbl(vdiff); vj = _mm256_loadu_pd(\u0026amp;ay[j]); vdiff = _mm256_sub_pd(viy, vj); vbj = _mm256_loadu_pd(\u0026amp;by[j]); vbj = _mm256_sub_pd(vbj, vdiff); _mm256_storeu_pd(\u0026amp;by[j], vbj); by[i] += reduce_256_dbl(vdiff); vj = _mm256_loadu_pd(\u0026amp;az[j]); vdiff = _mm256_sub_pd(viz, vj); vbj = _mm256_loadu_pd(\u0026amp;bz[j]); vbj = _mm256_sub_pd(vbj, vdiff); _mm256_storeu_pd(\u0026amp;bz[j], vbj); bz[i] += reduce_256_dbl(vdiff); } // taking care of residual iterations from inner loop for (; j \u0026lt; nelem; ++j) { double tmp = ax[i]-ax[j]; bx[i] += tmp; bx[j] -= tmp; tmp = ay[i]-ay[j]; by[i] += tmp; by[j] -= tmp; tmp = az[i]-az[j]; bz[i] += tmp; bz[j] -= tmp; } } } The approach to dealing with the residual iterations is intentionally naive as SIMD intrinsics are a little tricky to implement efficiently using blend instructions. And, given that at most, only a few iterations in the inner loop will make use of the simple, it\u0026rsquo;s probably not really worth the added complexity for a likely modest performance gain.\nThe corresponding performance (compiled with g++):\nComparing sweep_base_dbl (from results/sweep3dfull_base_dbl.json) to sweep_256_dbl_reduce2 (from results/sweep3dfull_256_dbl_reduce2.json) Benchmark Time CPU Time Old Time New CPU Old CPU New ---------------------------------------------------------------------------------------------------------------------- [sweep_base_dbl vs. sweep_256_dbl_reduce2]/4096 -0.3532 -0.3530 23663467 15305544 23599375 15269741 [sweep_base_dbl vs. sweep_256_dbl_reduce2]/8192 -0.2939 -0.2936 102167385 72144190 101891762 71975441 [sweep_base_dbl vs. sweep_256_dbl_reduce2]/16384 -0.3422 -0.3421 411281918 270557558 410259459 269924670 [sweep_base_dbl vs. sweep_256_dbl_reduce2]/32768 -0.4680 -0.4678 1838934215 978258653 1833970828 975969961 OVERALL_GEOMEAN -0.3677 -0.3675 0 0 0 0 The speedup varies significantly, but is roughly similar to the simplified 3D version shown previously. Interestingly, the speedup of the largest test case is quite good for reasons unnknown to me. This effect is present with both g++ and clang++.\n"},{"id":24,"href":"/worknotes/docs/useful/z-curve-spatial-hashing/","title":"Z-curve hashing","section":"Useful code snippets","content":" Hashing grid cell indices based on Z-order curve # Description # Code (Fortran) # Coming soon... "},{"id":25,"href":"/worknotes/docs/cuda/reference-codes/peakBandwidth/","title":"peakBandwidth","section":"nvfortran and nvc++ reference codes","content":" peakBandwidth # Description # Code to obtain theoretical peak memory bandwidth of GPUs on the system.\nEffective bandwidth can be obtained with\n\\[bw_e = (r_B \u0026#43; w_B)/(t\\cdot10^9)\\] where \\(bw_e\\) is the effective bandiwdth, \\(r_B\\) is the number of Bytes read, \\(w_B\\) is the number of Bytes written, and \\(t\\) is elapsed wall time in seconds.\nThe wall time of the simple memory kernel written in the limitingFactor code can be used.\nAnother note: the three memory bandwidth values we care about are:\nTheoretical: calculated in the code below Effective: can be measured with the approach above Actual: the realised bandwidth which is affected by memory access patterns Code (C++) # #include \u0026lt;stdio.h\u0026gt; int main() { int nDevices; cudaGetDeviceCount(\u0026amp;nDevices); cudaDeviceProp prop; for (int i = 0; i \u0026lt; nDevices; ++i) { cudaGetDeviceProperties(\u0026amp;prop, i); printf(\u0026#34; Device Number: %d\\n\u0026#34;, i); printf(\u0026#34; Memory Clock Rate (kHz): %d\\n\u0026#34;, prop.memoryClockRate); printf(\u0026#34; Memory Bush Width (bits): %d\\n\u0026#34;, prop.memoryBusWidth); printf(\u0026#34; Peak Memory Bandwidth (GB/s): %f\\n\\n\u0026#34;, 2.0*prop.memoryClockRate*(prop.memoryBusWidth / 8)*1.e-6); } } Code (Fortran) # program peakBandwidth use cudafor implicit none integer:: i, istat, nDevices=0 type(cudaDeviceProp):: prop istat = cudaGetDeviceCount(nDevices) do i = 0, nDevices-1 istat = cudaGetDeviceProperties(prop, i) write(*,\u0026#34;(\u0026#39; Device Number: \u0026#39;,i0)\u0026#34;) i write(*,\u0026#34;(\u0026#39; Device name: \u0026#39;,a)\u0026#34;) trim(prop%name) write(*,\u0026#34;(\u0026#39; Memory Clock Rate (KHz): \u0026#39;, i0)\u0026#34;) prop%memoryClockRate write(*,\u0026#34;(\u0026#39; Memory Bush Width (bits): \u0026#39;, i0)\u0026#34;) prop%memoryBusWidth write(*,\u0026#34;(\u0026#39; Peak Memory Bandwidth (GB/s): \u0026#39;, f6.2)\u0026#34;) \u0026amp; 2. *prop%memoryClockRate * (prop%memoryBusWidth/8.) * 1.e-6 write(*,*) end do end program peakBandwidth "},{"id":26,"href":"/worknotes/docs/manual-vectorization/cll-avx512/","title":"Vectorizing Cell-base Pair Search with AVX512","section":"Investigating Manual Vectorization for SPH","content":" Vectorizing Cell-base Pair Search with AVX512 # A step that all SPH codes must perform, is finding the pairs of interacting particles within the simulation domain. In the context of a uniform kernel radius (most common in weakly-compressible CFD schemes), this amounts to finding pairs of points that are within a fixed cutoff of eachother. If you\u0026rsquo;re familiar with the SciPy suite of tools, the scipy.spatial.cKDTree.query_pairs ( docs) function can be used here.\nThe drawback with the SciPy KDTree generic pair searching algorithm, is that they are suboptimal for the case where the cutoff length is fixed throughout the spatial domain.\n"},{"id":27,"href":"/worknotes/docs/cuda/reference-codes/bandwidthTest/","title":"bandwidthTest","section":"nvfortran and nvc++ reference codes","content":" bandwidthTest # Description # Demonstrates the increase of bandwidth when reading/writing from/to pinned host memory instead of normal \u0026ldquo;pageable\u0026rdquo; memory.\nCode (C++) # #include \u0026lt;stdio.h\u0026gt; #include \u0026lt;cstdlib\u0026gt; int main() { const int nElements = 4*1024*1024; size_t nbytes = nElements*sizeof(float); float *a_pageable, *b_pageable; float *a_pinned, *b_pinned; float *a_d; int ierr_a, ierr_b; cudaDeviceProp prop; cudaEvent_t startEvent, stopEvent; float time = 0.0; // pageable host memory a_pageable = (float*)malloc(nbytes); b_pageable = (float*)malloc(nbytes); // pinned host memory ierr_a = cudaMallocHost((void**)\u0026amp;a_pinned, nbytes); ierr_b = cudaMallocHost((void**)\u0026amp;b_pinned, nbytes); if (ierr_a != 0 || ierr_b != 0) { printf(\u0026#34;Allocation of a_pinned/b_pinned failed\\n\u0026#34;); std::exit(1); } // initializing for (int i = 0; i \u0026lt; nElements; ++i) a_pageable[i] = i; memcpy(a_pinned, a_pageable, nbytes); memset(b_pageable, 0.0, nbytes); memset(b_pinned, 0.0, nbytes); // device memory cudaMalloc((void**)\u0026amp;a_d, nbytes); // output device info and transfer size cudaGetDeviceProperties(\u0026amp;prop, 0); printf(\u0026#34;\\nDevice: %s\\n\u0026#34;, prop.name); printf(\u0026#34;Transfer size (MB): %f\\n\u0026#34;, nbytes/1024./1024.); cudaEventCreate(\u0026amp;startEvent); cudaEventCreate(\u0026amp;stopEvent); // pageable data transfers printf(\u0026#34;\\nPageable transfers\\n\u0026#34;); cudaEventRecord(startEvent, 0); cudaMemcpy(a_d, a_pageable, nbytes, cudaMemcpyHostToDevice); cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34; Host to Device bandwidth (GB/s): %f\\n\u0026#34;, nbytes/time/1.e+6); cudaEventRecord(startEvent, 0); cudaMemcpy(b_pageable, a_d, nbytes, cudaMemcpyDeviceToHost); cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34; Device to Host bandwidth (GB/s): %f\\n\u0026#34;, nbytes/time/1.e+6); for (int i = 0; i \u0026lt; nElements; ++i) { if (a_pageable[i] != b_pageable[i]) { printf(\u0026#34;*** Pageable transfers failed ***\\n\u0026#34;); break; } } cudaMemset(a_d, 0.0, nbytes); // pinned data transfers printf(\u0026#34;\\nPinned transfers\\n\u0026#34;); cudaEventRecord(startEvent, 0); cudaMemcpy(a_d, a_pinned, nbytes, cudaMemcpyHostToDevice); cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34; Host to Device bandwidth (GB/s): %f\\n\u0026#34;, nbytes/time/1.e+6); cudaEventRecord(startEvent, 0); cudaMemcpy(b_pinned, a_d, nbytes, cudaMemcpyDeviceToHost); cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34; Device to Host bandwidth (GB/s): %f\\n\u0026#34;, nbytes/time/1.e+6); for (int i = 0; i \u0026lt; nElements; ++i) { if (a_pinned[i] != a_pinned[i]) { printf(\u0026#34;*** Pinned transfers failed ***\\n\u0026#34;); break; } } printf(\u0026#34;\\n\u0026#34;); // cleanup cudaFree(a_d); cudaFreeHost(a_pinned); cudaFreeHost(b_pinned); free(a_pageable); free(b_pageable); cudaEventDestroy(startEvent); cudaEventDestroy(stopEvent); } Code (Fortran) # program BandwidthTest use cudafor implicit none integer, parameter:: nElements = 4*1024*1024 ! host arrays real(4):: a_pageable(nElements), b_pageable(nElements) real(4), allocatable, pinned:: a_pinned(:), b_pinned(:) ! device arrays real(4), device:: a_d(nElements) ! events for timing type(cudaEvent):: startEvent, stopEvent ! misc type(cudaDeviceProp):: prop real(4):: time integer:: istat, i logical:: pinnedFlag ! allocate and initialize do i = 1, nElements a_pageable(i) = i end do b_pageable = 0. allocate(a_pinned(nElements), b_pinned(nElements), STAT=istat, PINNED=pinnedFlag) if (istat /= 0) then write(*,*) \u0026#39;Allocation of a_pinned/b_pinned failed\u0026#39; pinnedFlag = .false. else if (.not. pinnedFlag) write(*,*) \u0026#39;Pinned allocation failed\u0026#39; end if if (pinnedFlag) then a_pinned = a_pageable b_pinned = 0. end if istat = cudaEventCreate(startEvent) istat = cudaEventCreate(stopEvent) ! output device info and transfer size istat = cudaGetDeviceProperties(prop, 0) write(*,*) write(*,*) \u0026#39;Device: \u0026#39;, trim(prop%name) write(*,*) \u0026#39;Transfer size (MB): \u0026#39;, 4*nElements/1024./1024. ! pageable data transfer write(*,*) write(*,*) \u0026#39;Pageable transfer\u0026#39; istat = cudaEventRecord(startEvent, 0) a_d = a_pageable istat = cudaEventRecord(stopEvent, 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) \u0026#39; Host to Device bandwidth (GB/s): \u0026#39;, nElements*4/time/1.e+6 istat = cudaEventRecord(startEvent, 0) b_pageable = a_d istat = cudaEventRecord(stopEvent, 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) \u0026#39; Device to Host bandwidth (GB/s): \u0026#39;, nElements*4/time/1.e+6 if (any(a_pageable /= b_pageable)) write(*,*) \u0026#39;*** Pageable transfers failed ***\u0026#39; ! pinned data transfers if (pinnedFlag) then write(*,*) write(*,*) \u0026#39;Pinned transfer\u0026#39; istat = cudaEventRecord(startEvent, 0) a_d = a_pinned istat = cudaEventRecord(stopEvent, 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time , startEvent , stopEvent) write(*,*) \u0026#39; Host to Device bandwidth (GB/s): \u0026#39;, nElements*4/time/1.e+6 istat = cudaEventRecord(startEvent, 0) b_pinned = a_d istat = cudaEventRecord(stopEvent, 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) \u0026#39; Device to Host bandwidth (GB/s): \u0026#39;, nElements*4/time/1.e+6 if (any(a_pinned /= b_pinned)) write(*,*) \u0026#39;*** Pinned transfers failed ***\u0026#39; end if write(*,*) ! cleanup if (allocated(a_pinned)) deallocate(a_pinned) if (allocated(b_pinned)) deallocate(b_pinned) istat = cudaEventDestroy(startEvent) istat = cudaEventDestroy(stopEvent) end program BandwidthTest "},{"id":28,"href":"/worknotes/docs/cuda/reference-codes/MemCpy/","title":"MemCpy","section":"nvfortran and nvc++ reference codes","content":" Memcpy # Description # Alternatives to array assignment to transfer data between host and device.\nCalls to the Memcpy function may be beneifical as transfers by array assignment can be implicitly broken up into multiple transfers, slowing down the transfer.\ncudaMemcpy is used heaps in the other C++ example codes, so I didn\u0026rsquo;t bother including a sample here.\nCode (Fortran) # ! for contiguous data istat = cudaMemcpy(a_d , a_pageable , nElements) ! for 2D data istat = cudaMemcpy2D(a_d(n1_l , n2_l), n, \u0026amp; a(n1_l , n2_l), n, \u0026amp; n1_u - n1_l + 1, n2_u - n2_l +1) ! there is also cudaMemcpy3D "},{"id":29,"href":"/worknotes/docs/cuda/reference-codes/testAsync/","title":"testAsync","section":"nvfortran and nvc++ reference codes","content":" testAsync # Description # Program to demonstrate the run-time improvements when performing asynchronous memory transfers and execution.\nIn the book, they show three different batching approaches, which yield different results based on the GPU. On my NVIDIA 1650 (Turing architecture), I see V1 producing best results most of the time. On my workplace\u0026rsquo;s P100 (Keplar), A30 (Ampere), and A100 (Ampere) GPUs, the lowest run times are reliably obtained with V2 (but only about 1% difference).\nCode (C++) # #include \u0026lt;stdio.h\u0026gt; /* this code demonstrates strategies hiding data transfer via asynchronous data copies in multiple streams */ __global__ void kernel(float* __restrict__ a, int offset) { int i = offset + threadIdx.x + blockIdx.x * blockDim.x; float x = i, s = sin(x), c = cos(x); a[i] += sqrt(s*s + c*c); } int main() { const int blockSize = 256, nStreams = 4; const int n = 4*1024*blockSize*nStreams, streamSize = n/nStreams; size_t nbytes = n*sizeof(float), streamBytes = streamSize*sizeof(float); int ierr; float *a, *a_d; cudaEvent_t startEvent, stopEvent, dummyEvent; cudaStream_t stream[nStreams]; float time, err; int offset; cudaDeviceProp prop; cudaGetDeviceProperties(\u0026amp;prop, 0); printf(\u0026#34; Device: %s\\n\\n\u0026#34;, prop.name); // allocate pinned host memory and device memory ierr = cudaMallocHost(\u0026amp;a, nbytes); if (ierr != 0) { printf(\u0026#34;Allocation of a failed\\n\u0026#34;); exit(1); } cudaMalloc(\u0026amp;a_d, nbytes); // create events and streams cudaEventCreate(\u0026amp;startEvent); cudaEventCreate(\u0026amp;stopEvent); cudaEventCreate(\u0026amp;dummyEvent); for (int i = 0; i \u0026lt; nStreams; ++i) { cudaStreamCreate(\u0026amp;stream[i]); } // baseline case - sequential transfer and execute memset(a, 0, nbytes); cudaEventRecord(startEvent, 0); cudaMemcpy(a_d, a, nbytes, cudaMemcpyHostToDevice); kernel\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, 0); cudaMemcpy(a, a_d, nbytes, cudaMemcpyDeviceToHost); cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34;Time for sequential transfer and execute (ms): %f\\n\u0026#34;, time); err = 0.0; for (int i = 0; i \u0026lt; n; ++i) { if (abs(a[i]-1.0) \u0026gt; err) err = abs(a[i]-1.0); } printf(\u0026#34; max error: %f\\n\u0026#34;, err); // asynchronous version 1: loop over (copy, kernel, copy) memset(a, 0, nbytes); cudaEventRecord(startEvent, 0); for (int i = 0; i \u0026lt; nStreams; ++i) { offset = i*streamSize; cudaMemcpyAsync(\u0026amp;a_d[offset], \u0026amp;a[offset], streamBytes, cudaMemcpyHostToDevice, stream[i]); kernel\u0026lt;\u0026lt;\u0026lt;streamSize/blockSize, blockSize, 0, stream[i]\u0026gt;\u0026gt;\u0026gt;(a_d, offset); cudaMemcpyAsync(\u0026amp;a[offset], \u0026amp;a_d[offset], streamBytes, cudaMemcpyDeviceToHost, stream[i]); } cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34;Time for asynchronous V1 transfer and execute (ms): %f\\n\u0026#34;, time); err = 0.0; for (int i = 0; i \u0026lt; n; ++i) { if (abs(a[i]-1.0) \u0026gt; err) err = abs(a[i]-1.0); } printf(\u0026#34; max error: %f\\n\u0026#34;, err); // asynchronous version 2: // loop over copy, loop over kernel, loop over copy memset(a, 0, nbytes); cudaEventRecord(startEvent, 0); for (int i = 0; i \u0026lt; nStreams; ++i) { offset = i*streamSize; cudaMemcpyAsync(\u0026amp;a_d[offset], \u0026amp;a[offset], streamBytes, cudaMemcpyHostToDevice, stream[i]); } for (int i = 0; i \u0026lt; nStreams; ++i) { offset = i*streamSize; kernel\u0026lt;\u0026lt;\u0026lt;streamSize/blockSize, blockSize, 0, stream[i]\u0026gt;\u0026gt;\u0026gt;(a_d, offset); } for (int i = 0; i \u0026lt; nStreams; ++i) { offset = i*streamSize; cudaMemcpyAsync(\u0026amp;a[offset], \u0026amp;a_d[offset], streamBytes, cudaMemcpyDeviceToHost, stream[i]); } cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34;Time for asynchronous V2 transfer and execute (ms): %f\\n\u0026#34;, time); err = 0.0; for (int i = 0; i \u0026lt; n; ++i) { if (abs(a[i]-1.0) \u0026gt; err) err = abs(a[i]-1.0); } printf(\u0026#34; max error: %f\\n\u0026#34;, err); // asynchronous version 3: // loop over copy, loop over (kernel, event), loop over copy memset(a, 0, nbytes); cudaEventRecord(startEvent, 0); for (int i = 0; i \u0026lt; nStreams; ++i) { offset = i*streamSize; cudaMemcpyAsync(\u0026amp;a_d[offset], \u0026amp;a[offset], streamBytes, cudaMemcpyHostToDevice, stream[i]); } for (int i = 0; i \u0026lt; nStreams; ++i) { offset = i*streamSize; kernel\u0026lt;\u0026lt;\u0026lt;streamSize/blockSize, blockSize, 0, stream[i]\u0026gt;\u0026gt;\u0026gt;(a_d, offset); cudaEventRecord(dummyEvent, stream[i]); } for (int i = 0; i \u0026lt; nStreams; ++i) { offset = i*streamSize; cudaMemcpyAsync(\u0026amp;a[offset], \u0026amp;a_d[offset], streamBytes, cudaMemcpyDeviceToHost, stream[i]); } cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34;Time for asynchronous V3 transfer and execute (ms): %f\\n\u0026#34;, time); err = 0.0; for (int i = 0; i \u0026lt; n; ++i) { if (abs(a[i]-1.0) \u0026gt; err) err = abs(a[i]-1.0); } printf(\u0026#34; max error: %f\\n\u0026#34;, err); cudaFree(a_d); free(a); } Code (Fortran) # ! This code demonstrates strategies hiding data transfers via ! asynchronous data copies in multiple streams module kernels_m contains attributes(global) subroutine kernel(a, offset) implicit none real:: a(*) integer, value:: offset integer:: i real:: c, s, x i = offset + threadIdx%x + (blockIdx%x-1)* blockDim%x x = i; s = sin(x); c = cos(x) a(i) = a(i) + sqrt(s**2+c**2) end subroutine kernel end module kernels_m program testAsync use cudafor use kernels_m implicit none integer, parameter:: blockSize = 256, nStreams = 4 integer, parameter:: n = 4*1024* blockSize*nStreams real, pinned, allocatable:: a(:) real, device:: a_d(n) integer(kind=cuda_stream_kind):: stream(nStreams) type(cudaEvent) :: startEvent, stopEvent, dummyEvent real:: time integer:: i, istat, offset, streamSize = n/nStreams logical:: pinnedFlag type(cudaDeviceProp):: prop istat = cudaGetDeviceProperties(prop , 0) write(*,\u0026#34;(\u0026#39; Device: \u0026#39;, a,/)\u0026#34;) trim(prop%name) ! allocate pinned host memory allocate(a(n), STAT=istat , PINNED=pinnedFlag) if (istat /= 0) then write(*,*) \u0026#39;Allocation of a failed\u0026#39; stop else if (.not. pinnedFlag) write(*,*) \u0026#39;Pinned allocation failed\u0026#39; end if ! create events and streams istat = cudaEventCreate(startEvent) istat = cudaEventCreate(stopEvent) istat = cudaEventCreate(dummyEvent) do i = 1, nStreams istat = cudaStreamCreate(stream(i)) end do ! baseline case - sequential transfer and execute a = 0 istat = cudaEventRecord(startEvent, 0) a_d = a call kernel \u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize \u0026gt;\u0026gt;\u0026gt;(a_d, 0) a = a_d istat = cudaEventRecord(stopEvent , 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) \u0026#39;Time for sequential \u0026#39;, \u0026#39;transfer and execute (ms): \u0026#39;, time write(*,*) \u0026#39; max error: \u0026#39;, maxval(abs(a-1.0)) ! asynchronous version 1: loop over {copy , kernel , copy} a = 0 istat = cudaEventRecord(startEvent, 0) do i = 1, nStreams offset = (i-1)* streamSize istat = cudaMemcpyAsync(a_d(offset+1), a(offset+1), streamSize, \u0026amp; stream(i)) call kernel \u0026lt;\u0026lt;\u0026lt;streamSize/blockSize, blockSize, 0, stream(i)\u0026gt;\u0026gt;\u0026gt;(a_d, \u0026amp; offset) istat = cudaMemcpyAsync(a(offset+1), a_d(offset+1), streamSize, \u0026amp; stream(i)) end do istat = cudaEventRecord(stopEvent, 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time , startEvent , stopEvent) write(*,*) \u0026#39;Time for asynchronous V1 \u0026#39;, \u0026#39;transfer and execute (ms): \u0026#39;, time write(*,*) \u0026#39; max error: \u0026#39;, maxval(abs(a-1.)) ! asynchronous version 2: ! loop over copy , loop over kernel , loop over copy a = 0 istat = cudaEventRecord(startEvent, 0) do i = 1, nStreams offset = (i-1)*streamSize istat = cudaMemcpyAsync(a_d(offset+1), a(offset+1), streamSize, \u0026amp; stream(i)) end do do i = 1, nStreams offset = (i-1)*streamSize call kernel \u0026lt;\u0026lt;\u0026lt;streamSize/blockSize, blockSize, 0, stream(i)\u0026gt;\u0026gt;\u0026gt;(a_d, \u0026amp; offset) end do do i = 1, nStreams offset = (i-1)*streamSize istat = cudaMemcpyAsync (a(offset +1), a_d(offset+1), streamSize, \u0026amp; stream(i)) end do istat = cudaEventRecord(stopEvent, 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time , startEvent , stopEvent) write(*,*) \u0026#39;Time for asynchronous V2 \u0026#39;, \u0026#39;transfer and execute (ms): \u0026#39;, time write(*,*) \u0026#39; max error: \u0026#39;, maxval(abs(a-1.)) ! asynchronous version 3: ! loop over copy , loop over {kernel, event}, ! loop over copy a = 0 istat = cudaEventRecord(startEvent ,0) do i = 1, nStreams offset = (i-1)* streamSize istat = cudaMemcpyAsync(a_d(offset+1), a(offset+1), streamSize, \u0026amp; stream(i)) end do do i = 1, nStreams offset = (i-1)* streamSize call kernel \u0026lt;\u0026lt;\u0026lt;streamSize/blockSize, blockSize, 0, stream(i)\u0026gt;\u0026gt;\u0026gt;(a_d, \u0026amp; offset) istat = cudaEventRecord(dummyEvent, stream(i)) end do do i = 1, nStreams offset = (i-1)* streamSize istat = cudaMemcpyAsync(a(offset+1), a_d(offset+1), streamSize, \u0026amp; stream(i)) end do istat = cudaEventRecord(stopEvent, 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) \u0026#39;Time for asynchronous V3 \u0026#39;, \u0026#39;transfer and execute (ms): \u0026#39;, time write(*,*) \u0026#39; max error: \u0026#39;, maxval(abs(a-1.)) ! cleanup istat = cudaEventDestroy(startEvent) istat = cudaEventDestroy(stopEvent) istat = cudaEventDestroy(dummyEvent) do i = 1, nStreams istat = cudaStreamDestroy(stream(i)) end do deallocate(a) end program testAsync "},{"id":30,"href":"/worknotes/docs/cuda/reference-codes/offsetNStride/","title":"offsetNStride","section":"nvfortran and nvc++ reference codes","content":" offsetNStride # Description # Demonstration of how coalesced access to GPU global memory i.e., accessing memory in strides of 16 (half-warp) or 32 (warp) can reduce the number of transactions made and reduce run times.\nMy NVIDIA 1650 behaves like K20 and C2050 used in the book, where 0-stride accesses are fastest, and everything else is worse (by only a little).\nCode (C++) # #include \u0026lt;stdio.h\u0026gt; #include \u0026lt;precision.h\u0026gt; __global__ void offset(userfp_t* a, int s) { int i = blockDim.x*blockIdx.x + threadIdx.x + s; a[i] += 1.0; } __global__ void stride(userfp_t* a, int s) { int i = (blockDim.x * blockIdx.x + threadIdx.x)*s; a[i] += 1.0; } int main() { const int nMB = 4, n = nMB*1024*1024/sizeof(userfp_t), blockSize = 256; const size_t nbytes = n*sizeof(userfp_t); userfp_t *a_d, *b_d; cudaDeviceProp prop; cudaEvent_t startEvent, stopEvent; float time; // array dimensions are 33*n for stride cases cudaMalloc(\u0026amp;a_d, 33*nbytes); cudaMalloc(\u0026amp;b_d, 33*nbytes); cudaGetDeviceProperties(\u0026amp;prop, 0); printf(\u0026#34;Device: %s\\n\u0026#34;, prop.name); printf(\u0026#34;Transfer size (MB): %d\\n\u0026#34;, nMB); if (sizeof(userfp_t)==sizeof(double)) { printf(\u0026#34;Double Precision\\n\u0026#34;); } else { printf(\u0026#34;Single Precision\\n\u0026#34;); } cudaEventCreate(\u0026amp;startEvent); cudaEventCreate(\u0026amp;stopEvent); printf(\u0026#34;Offset, Bandwidth (GB/s):\\n\u0026#34;); offset\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(b_d, 0); for (int i = 0; i \u0026lt; 33; ++i) { cudaMemset(\u0026amp;a_d, 0, 33*nbytes); cudaEventRecord(startEvent, 0); offset\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, i); cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34;%d %f\\n\u0026#34;, i, 2.0*nbytes/time*1.e-6); } printf(\u0026#34;\\nStrid, Bandwidth (GB/s):\\n\u0026#34;); stride\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(b_d, 1); for (int i = 1; i \u0026lt; 33; ++i) { cudaMemset(\u0026amp;a_d, 0, 33*nbytes); cudaEventRecord(startEvent, 0); stride\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(a_d, i); cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); printf(\u0026#34;%d %f\\n\u0026#34;, i, 2.0*nbytes/time*1.e-6); } cudaFree(a_d); cudaFree(b_d); cudaEventDestroy(startEvent); cudaEventDestroy(stopEvent); } Code (Fortran) # module kernels_m use precision_m contains attributes(global) subroutine offset(a, s) real(f):: a(*) integer, value:: s integer:: i i = blockDim%x*(blockIdx%x-1)+ threadIdx%x + s a(i) = a(i) + 1._f end subroutine offset attributes(global) subroutine stride(a, s) real(f):: a(*) integer, value:: s integer:: i i = (blockDim%x*(blockIdx%x-1)+ threadIdx%x) * s a(i) = a(i) + 1._f end subroutine stride end module kernels_m program offsetNStride use cudafor use kernels_m implicit none integer, parameter:: nMB = 4 ! transfer size in MB integer, parameter:: n = nMB*1024*1024/f integer, parameter:: blockSize = 256 ! array dimensions are 33*n for stride cases real(f), device:: a_d(33*n), b_d(33*n) type(cudaEvent):: startEvent, stopEvent type(cudaDeviceProp):: prop integer:: i, istat real:: time istat = cudaGetDeviceProperties(prop, 0) write(*,\u0026#39;(/,\u0026#34; Device: \u0026#34;,a)\u0026#39;) trim(prop%name) write(*,\u0026#39;(\u0026#34; Transfer size (MB): \u0026#34;,i0)\u0026#39;) nMB if (kind(a_d) == sf) then write(*,\u0026#39;(a,/)\u0026#39;) \u0026#39;Single Precision \u0026#39; else write(*,\u0026#39;(a,/)\u0026#39;) \u0026#39;Double Precision \u0026#39; end if istat = cudaEventCreate(startEvent) istat = cudaEventCreate(stopEvent) write(*,*) \u0026#39;Offset, Bandwidth (GB/s):\u0026#39; call offset \u0026lt;\u0026lt;\u0026lt;n/blockSize,blockSize \u0026gt;\u0026gt;\u0026gt;(b_d, 0) do i = 0, 32 a_d = 0._f istat = cudaEventRecord(startEvent,0) call offset \u0026lt;\u0026lt;\u0026lt;n/blockSize,blockSize \u0026gt;\u0026gt;\u0026gt;(a_d, i) istat = cudaEventRecord(stopEvent,0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) i, 2*n*f/time *1.e-6 end do write(*,*) write(*,*) \u0026#39;Stride, Bandwidth (GB/s):\u0026#39; call stride \u0026lt;\u0026lt;\u0026lt;n/blockSize,blockSize \u0026gt;\u0026gt;\u0026gt;(b_d, 1) do i = 1, 32 a_d = 0._f istat = cudaEventRecord(startEvent,0) call stride \u0026lt;\u0026lt;\u0026lt;n/blockSize,blockSize \u0026gt;\u0026gt;\u0026gt;(a_d, i) istat = cudaEventRecord(stopEvent,0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) i, 2*n*f/time*1.e-6 end do istat = cudaEventDestroy(startEvent) istat = cudaEventDestroy(stopEvent) end program offsetNStride "},{"id":31,"href":"/worknotes/docs/cuda/reference-codes/strideTexture/","title":"strideTexture","section":"nvfortran and nvc++ reference codes","content":" strideTexture # Description # A demonstration showing how the use of textured memory pointers can improve strided global memory access.\nI find that using textured memory pointers didn\u0026rsquo;t improve anything reliably on my NVIDIA 1650.\nThe deprecation is also why a C++ version is not provided here.\nCode (Fortran) # module kernels_m real, texture, pointer:: aTex (:) contains attributes(global) subroutine stride(b, a, s) real:: b(*), a(*) integer, value:: s integer:: i, is i = blockDim%x*( blockIdx%x-1)+ threadIdx%x is = (blockDim%x*( blockIdx%x-1)+ threadIdx%x) * s b(i) = a(is)+1 end subroutine stride attributes(global) subroutine strideTex(b, s) real:: b(*) integer, value:: s integer:: i, is i = blockDim%x*( blockIdx%x-1)+ threadIdx%x is = (blockDim%x*( blockIdx%x-1)+ threadIdx%x) * s b(i) = aTex(is)+1 end subroutine strideTex end module kernels_m program strideTexture use cudafor use kernels_m implicit none integer, parameter:: nMB = 4 ! transfer size in MB integer, parameter:: n = nMB *1024*1024/4 integer, parameter:: blockSize = 256 real, device, allocatable, target:: a_d(:), b_d (:) type(cudaEvent):: startEvent, stopEvent type(cudaDeviceProp):: prop integer:: i, istat, ib real:: time istat = cudaGetDeviceProperties(prop, 0) write(*,\u0026#39;(/,\u0026#34; Device: \u0026#34;,a)\u0026#39;) trim(prop%name) write(*,\u0026#39;(\u0026#34; Transfer size (MB): \u0026#34;,i0,/)\u0026#39;) nMB allocate(a_d(n*33), b_d(n)) istat = cudaEventCreate(startEvent) istat = cudaEventCreate(stopEvent) write(*,*) \u0026#39;Global version \u0026#39; write(*,*) \u0026#39;Stride, Bandwidth (GB/s)\u0026#39; call stride\u0026lt;\u0026lt;\u0026lt;n/blockSize,blockSize\u0026gt;\u0026gt;\u0026gt;(b_d, a_d, 1) do i = 1, 32 a_d = 0.0 istat = cudaEventRecord(startEvent,0) call stride\u0026lt;\u0026lt;\u0026lt;n/blockSize, blockSize\u0026gt;\u0026gt;\u0026gt;(b_d, a_d, i) istat = cudaEventRecord(stopEvent,0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) i, 2*n*4/time*1.e-6 enddo ! bind the texture aTex =\u0026gt; a_d write(*,*) \u0026#39;Texture version \u0026#39; write(*,*) \u0026#39;Stride, Bandwidth (GB/s)\u0026#39; call strideTex\u0026lt;\u0026lt;\u0026lt;n/blockSize,blockSize\u0026gt;\u0026gt;\u0026gt;(b_d, 1) do i = 1, 32 a_d = 0.0 istat = cudaEventRecord(startEvent,0) call strideTex\u0026lt;\u0026lt;\u0026lt;n/blockSize,blockSize\u0026gt;\u0026gt;\u0026gt;(b_d, i) istat = cudaEventRecord(stopEvent,0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) write(*,*) i, 2*n*4/time*1.e-6 enddo ! unbind the texture nullify(aTex) istat = cudaEventDestroy(startEvent) istat = cudaEventDestroy(stopEvent) deallocate(a_d, b_d) end program strideTexture "},{"id":32,"href":"/worknotes/docs/cuda/reference-codes/sharedExample/","title":"sharedExample","section":"nvfortran and nvc++ reference codes","content":" sharedExample # Description # A sample code to demonstrate how the compiler uses various types of memory. This information is availed when compiling with -Mcuda=ptxinfo.\nCode (C++) # /* This code shows how dynamically and statically allocated shared memory are used to reverse a small array */ #include \u0026lt;stdio.h\u0026gt; __global__ void staticReverse(float* d, int n) { __shared__ float s[64]; int t = threadIdx.x, tr = n - t - 1; s[t] = d[t]; __syncthreads(); d[t] = s[tr]; } __global__ void dynamicReverse1(float* d, int n) { extern __shared__ float s[]; int t = threadIdx.x, tr = n - t - 1; s[t] = d[t]; __syncthreads(); d[t] = s[tr]; } // only one thread can initialize the shared memory buffer __global__ void dynamicReverse2(float* d, int n) { __shared__ float *s ; int t = threadIdx.x, tr = n - t - 1; if (t==0) s = new float[n]; __syncthreads(); s[t] = d[t]; __syncthreads(); d[t] = s[tr]; if (t==0) delete(s); } // The Fortran dynamic Reverse3 doesn\u0026#39;t have an equivalent in C++ int main() { const int n = 64, nbytes = n*sizeof(float); float *a, *r, *d, *d_d; dim3 grid, tBlock; float maxerror; a = (float*)malloc(nbytes); r = (float*)malloc(nbytes); d = (float*)malloc(nbytes); cudaMalloc(\u0026amp;d_d, nbytes); tBlock = dim3(n, 1, 1); grid = dim3(1, 1, 1); for (int i = 0; i \u0026lt; n; ++i) { a[i] = i; r[i] = n-i-1; } // run version with static shared memory cudaMemcpy(d_d, a, nbytes, cudaMemcpyHostToDevice); staticReverse\u0026lt;\u0026lt;\u0026lt;grid, tBlock\u0026gt;\u0026gt;\u0026gt;(d_d, n); cudaMemcpy(d, d_d, nbytes, cudaMemcpyDeviceToHost); maxerror = 0.; for (int i = 0; i \u0026lt; n; ++i) { if (abs(r[i]-d[i]) \u0026gt; maxerror) maxerror = abs(r[i]-d[i]); } printf(\u0026#34;Static case max error: %f\\n\u0026#34;, maxerror); // run dynamic shared memory version 1 cudaMemcpy(d_d, a, nbytes, cudaMemcpyHostToDevice); dynamicReverse1\u0026lt;\u0026lt;\u0026lt;grid, tBlock\u0026gt;\u0026gt;\u0026gt;(d_d, n); cudaMemcpy(d, d_d, nbytes, cudaMemcpyDeviceToHost); maxerror = 0.; for (int i = 0; i \u0026lt; n; ++i) { if (abs(r[i]-d[i]) \u0026gt; maxerror) maxerror = abs(r[i]-d[i]); } printf(\u0026#34;Static case max error: %f\\n\u0026#34;, maxerror); // run dynamic shared memory version 2 cudaMemcpy(d_d, a, nbytes, cudaMemcpyHostToDevice); dynamicReverse2\u0026lt;\u0026lt;\u0026lt;grid, tBlock\u0026gt;\u0026gt;\u0026gt;(d_d, n); cudaMemcpy(d, d_d, nbytes, cudaMemcpyDeviceToHost); maxerror = 0.; for (int i = 0; i \u0026lt; n; ++i) { if (abs(r[i]-d[i]) \u0026gt; maxerror) maxerror = abs(r[i]-d[i]); } printf(\u0026#34;Static case max error: %f\\n\u0026#34;, maxerror); free(a); free(r); free(d); cudaFree(d_d); } Code (Fortran) # ! This code shows how dynamically and statically allocated ! shared memory are used to reverse a small array module reverse_m implicit none integer, device:: n_d contains attributes(global) subroutine staticReverse(d) real:: d(:) integer:: t, tr real, shared:: s(64) t = threadIdx%x tr = size(d)-t+1 s(t) = d(t) call syncthreads() d(t) = s(tr) end subroutine staticReverse attributes(global) subroutine dynamicReverse1(d) real:: d(:) integer:: t, tr real, shared:: s(*) t = threadIdx%x tr = size(d)-t+1 s(t) = d(t) call syncthreads() d(t) = s(tr) end subroutine dynamicReverse1 attributes(global) subroutine dynamicReverse2(d, nSize) real:: d(nSize) integer, value:: nSize integer:: t, tr real, shared:: s(nSize) t = threadIdx%x tr = nSize -t+1 s(t) = d(t) call syncthreads() d(t) = s(tr) end subroutine dynamicReverse2 attributes(global) subroutine dynamicReverse3(d) real:: d(n_d) real, shared:: s(n_d) integer:: t, tr t = threadIdx%x tr = n_d -t+1 s(t) = d(t) call syncthreads() d(t) = s(tr) end subroutine dynamicReverse3 end module reverse_m program sharedExample use cudafor use reverse_m implicit none integer, parameter:: n = 64 real:: a(n), r(n), d(n) real, device:: d_d(n) type(dim3):: grid, tBlock integer:: i, sizeInBytes tBlock = dim3(n,1,1) grid = dim3 (1,1,1) do i = 1, n a(i) = i r(i) = n-i+1 enddo sizeInBytes = sizeof(a(1))* tBlock%x ! run version with static shared memory d_d = a call staticReverse\u0026lt;\u0026lt;\u0026lt;grid,tBlock\u0026gt;\u0026gt;\u0026gt;(d_d) d = d_d write(*,*) \u0026#39;Static case max error:\u0026#39;, maxval(abs(r-d)) ! run dynamic shared memory version 1 d_d = a call dynamicReverse1\u0026lt;\u0026lt;\u0026lt;grid,tBlock,sizeInBytes\u0026gt;\u0026gt;\u0026gt;(d_d) d = d_d write(*,*) \u0026#39;Dynamic case 1 max error:\u0026#39;, maxval(abs(r-d)) ! run dynamic shared memory version 2 d_d = a call dynamicReverse2\u0026lt;\u0026lt;\u0026lt;grid,tBlock,sizeInBytes\u0026gt;\u0026gt;\u0026gt;(d_d,n) d = d_d write(*,*) \u0026#39;Dynamic case 2 max error:\u0026#39;, maxval(abs(r-d)) ! run dynamic shared memory version 3 n_d = n ! n_d declared in reverse_m d_d = a call dynamicReverse3\u0026lt;\u0026lt;\u0026lt;grid,tBlock,sizeInBytes\u0026gt;\u0026gt;\u0026gt;(d_d) d = d_d write(*,*) \u0026#39;Dynamic case 3 max error:\u0026#39;, maxval(abs(r-d)) end program sharedExample "},{"id":33,"href":"/worknotes/docs/cuda/reference-codes/checkP2pAccess/","title":"checkP2PAccess","section":"nvfortran and nvc++ reference codes","content":" checkP2pAccess # Description # Tool to check peer-to-peer connectivity between GPUs connected to the motherboard.\nCode (C++) # #include \u0026lt;stdio.h\u0026gt; int main() { int nDevices; cudaGetDeviceCount(\u0026amp;nDevices); printf(\u0026#34;Number of CUDA-capable devices: %d\\n\u0026#34;, nDevices); cudaDeviceProp prop; for (int i = 0; i \u0026lt; nDevices; ++i) { cudaGetDeviceProperties(\u0026amp;prop, i); printf(\u0026#34;Device %d: %s\\n\u0026#34;, i, prop.name); } int p2pOK[nDevices][nDevices]; for (int i = 0; i \u0026lt; nDevices; ++i) { for (int j = i+1; j \u0026lt; nDevices; ++j) { cudaDeviceCanAccessPeer(\u0026amp;p2pOK[i][j], i, j); p2pOK[j][i] = p2pOK[i][j]; } } printf(\u0026#34;\\n\u0026#34;); for (int i = 0; i \u0026lt; nDevices; ++i) { printf(\u0026#34; %3d\u0026#34;, i); } printf(\u0026#34;\\n\u0026#34;); for (int j = 0; j \u0026lt; nDevices; ++j) { printf(\u0026#34;%3d\u0026#34;,j); for (int i = 0; i \u0026lt; nDevices; ++i) { if (i==j) { printf(\u0026#34; - \u0026#34;); } else if (p2pOK[i][j] == 1) { printf(\u0026#34; Y \u0026#34;); } else { printf(\u0026#34; \u0026#34;); } } printf(\u0026#34;\\n\u0026#34;); } } Code (Fortran) # program checkP2pAccess use cudafor implicit none integer, allocatable:: p2pOK(:,:) integer:: nDevices, i, j, istat type (cudaDeviceProp):: prop istat = cudaGetDeviceCount(nDevices) write(*,\u0026#34;(\u0026#39;Number of CUDA -capable devices: \u0026#39;, i0,/)\u0026#34;) nDevices do i = 0, nDevices -1 istat = cudaGetDeviceProperties(prop, i) write(*,\u0026#34;(\u0026#39;Device \u0026#39;, i0, \u0026#39;: \u0026#39;, a)\u0026#34;) i, trim(prop%name) end do write(*,*) allocate(p2pOK (0: nDevices -1, 0: nDevices -1)) p2pOK = 0 do j = 0, nDevices -1 do i = j+1, nDevices -1 istat = cudaDeviceCanAccessPeer(p2pOK(i,j), i, j) p2pOK(j,i) = p2pOK(i,j) end do end do do i = 0, nDevices -1 write(*,\u0026#34;(3x,i3)\u0026#34;, advance=\u0026#39;no\u0026#39;) i end do write(*,*) do j = 0, nDevices -1 write(*,\u0026#34;(i3)\u0026#34;, advance=\u0026#39;no\u0026#39;) j do i = 0, nDevices -1 if (i == j) then write(*,\u0026#34;(2x,\u0026#39;-\u0026#39;,3x)\u0026#34;, advance=\u0026#39;no\u0026#39;) else if (p2pOK(i,j) == 1) then write(*,\u0026#34;(2x, \u0026#39;Y\u0026#39;,3x)\u0026#34;,advance=\u0026#39;no\u0026#39;) else write(*,\u0026#34;(6x)\u0026#34;,advance=\u0026#39;no\u0026#39;) end if end do write(*,*) end do end program checkP2pAccess "},{"id":34,"href":"/worknotes/docs/cuda/reference-codes/directTransfer/","title":"directTransfer","section":"nvfortran and nvc++ reference codes","content":" directTransfer # Description # Demonstration code showing the difference in data transfer rates when transfering directly between peer GPUs, versus without p2p transfer.\nRunning this on the GPUs at work, showed the transfer between GPUs with p2p disabled was maybe 5% slower, but this wasn\u0026rsquo;t produced reliably. The differnece was not nearly as dramatic as those in the book.\nCode (C++) # To do... Code (Fortran) # program directTransfer use cudafor implicit none integer, parameter:: N = 4*1024*1024 real, pinned, allocatable:: a(:), b(:) real, device, allocatable:: a_d(:), b_d(:) ! these hold free and total memory before and after ! allocation, used to verify allocation is happening ! on proper devices integer(int_ptr_kind()), allocatable:: freeBefore(:), totalBefore(:), freeAfter(:), totalAfter(:) integer:: istat, nDevices, i, accessPeer, timingDev type(cudaDeviceProp):: prop type(cudaEvent):: startEvent, stopEvent real:: time ! allocate host arrays allocate(a(N), b(N)) allocate(freeBefore(0: nDevices-1), totalBefore(0: nDevices-1)) allocate(freeAfter(0: nDevices-1), totalAfter(0: nDevices-1)) ! get device info(including total and free memory) ! before allocating a_d and b_d on devices 0 and 1 istat = cudaGetDeviceCount(nDevices) if(nDevices \u0026lt; 2) then write(*,*) \u0026#39;Need at least two CUDA capable devices \u0026#39; stop end if write(*,\u0026#34;(\u0026#39;Number of CUDA -capable devices: \u0026#39;, i0,/)\u0026#34;) nDevices do i = 0, nDevices-1 istat = cudaGetDeviceProperties(prop, i) istat = cudaSetDevice(i) istat = cudaMemGetInfo(freeBefore(i), totalBefore(i)) end do istat = cudaSetDevice(0) allocate(a_d(N)) istat = cudaSetDevice(1) allocate(b_d(N)) ! print out free memory before and after allocation write(*,\u0026#34;(\u0026#39;Allocation summary \u0026#39;)\u0026#34;) do i = 0, nDevices-1 istat = cudaGetDeviceProperties(prop, i) write(*,\u0026#34;(\u0026#39; Device \u0026#39;, i0, \u0026#39;: \u0026#39;, a)\u0026#34;) i, trim(prop%name) istat = cudaSetDevice(i) istat = cudaMemGetInfo(freeAfter(i), totalAfter(i)) write(*,\u0026#34;(\u0026#39; Free memory before: \u0026#39;, i0, \u0026#39;, after: \u0026#39;, i0, \u0026#39;, difference: \u0026#39;,i0,/)\u0026#34;) \u0026amp; freeBefore(i), freeAfter(i), freeBefore(i)-freeAfter(i) end do ! check whether devices 0 and 1 can use P2P if(nDevices \u0026gt; 1) then istat = cudaDeviceCanAccessPeer(accessPeer, 0, 1) if(accessPeer == 1) then write(*,*) \u0026#39;Peer access available between 0 and 1\u0026#39; else write(*,*) \u0026#39;Peer access not available between 0 and 1\u0026#39; end if end if ! initialize a = 1.0 istat = cudaSetDevice(0) a_d = a ! perform test twice, timing on both sending GPU ! and receiving GPU do timingDev = 0, 1 write(*,\u0026#34;(/,\u0026#39;Timing on device \u0026#39;, i0, /)\u0026#34;) timingDev ! create events on the timing device istat = cudaSetDevice(timingDev) istat = cudaEventCreate(startEvent) istat = cudaEventCreate(stopEvent) if(accessPeer == 1) then ! enable P2P communication istat = cudaSetDevice(0) istat = cudaDeviceEnablePeerAccess(1, 0) istat = cudaSetDevice(1) istat = cudaDeviceEnablePeerAccess(0, 0) ! transfer(implicitly) across devices b_d = -1.0 istat = cudaSetDevice(timingDev) istat = cudaEventRecord(startEvent,0) b_d = a_d istat = cudaEventRecord(stopEvent,0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) b = b_d if(any(b /= a)) then write(*,\u0026#34;(\u0026#39;Transfer failed \u0026#39;)\u0026#34;) else write(*,\u0026#34;(\u0026#39;b_d=a_d transfer(GB/s): \u0026#39;, f)\u0026#34;) N*4/time/1.0E+6 end if end if ! transfer via cudaMemcpyPeer() if(accessPeer == 0) istat = cudaSetDevice(1) b_d = -1.0 istat = cudaSetDevice(timingDev) istat = cudaEventRecord(startEvent,0) istat = cudaMemcpyPeer(b_d, 1, a_d, 0, N) istat = cudaEventRecord(stopEvent,0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) if(accessPeer == 0) istat = cudaSetDevice(1) b = b_d if(any(b /= a)) then write(*,\u0026#34;(\u0026#39;Transfer failed \u0026#39;)\u0026#34;) else write(*,\u0026#34;(\u0026#39;cudaMemcpyPeer transfer(GB/s): \u0026#39;, f)\u0026#34;) N*4/time/1.0E+6 end if ! cudaMemcpyPeer with P2P disabled if(accessPeer == 1) then istat = cudaSetDevice(0) istat = cudaDeviceDisablePeerAccess(1) istat = cudaSetDevice(1) istat = cudaDeviceDisablePeerAccess(0) b_d = -1.0 istat = cudaSetDevice(timingDev) istat = cudaEventRecord(startEvent,0) istat = cudaMemcpyPeer(b_d, 1, a_d, 0, N) istat = cudaEventRecord(stopEvent,0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) istat = cudaSetDevice(1) b = b_d if(any(b /= a)) then write(*,\u0026#34;(\u0026#39;Transfer failed \u0026#39;)\u0026#34;) else write(*,\u0026#34;(\u0026#39;cudaMemcpyPeer transfer w/ P2P \u0026#39;, \u0026#39; disabled(GB/s): \u0026#39;, f)\u0026#34;) N*4/time/1.0E+6 end if end if ! destroy events associated with timingDev istat = cudaEventDestroy(startEvent) istat = cudaEventDestroy(stopEvent) end do ! clean up deallocate(freeBefore, totalBefore, freeAfter, totalAfter) deallocate(a, b, a_d, b_d) end program directTransfer "},{"id":35,"href":"/worknotes/docs/cuda/reference-codes/p2pBandwidth/","title":"p2pBandwidth","section":"nvfortran and nvc++ reference codes","content":" p2pBandwidth # Description # Program to measure the bandwidth of memory transfer between a GPUs on a multi-GPU host.\nA useful technique shown here, is the use of derived types with device member arrays to manage each GPU\u0026rsquo;s instance of the device arrays.\nCode (C++) # #include \u0026lt;stdio.h\u0026gt; #include \u0026lt;cstdlib\u0026gt; struct distributedArray { float* a_d; }; __global__ void setVal(float* __restrict__ array, float val) { int i = threadIdx.x + blockIdx.x * blockDim.x; array[i] = val; } int main() { const int N = 4*1024*1024; const size_t nbytes = N*sizeof(float); // displaying number of GPUs int nDevices; cudaGetDeviceCount(\u0026amp;nDevices); // displaying GPU names cudaDeviceProp prop; for (int i = 0; i \u0026lt; nDevices; ++i) { cudaGetDeviceProperties(\u0026amp;prop, i); printf(\u0026#34;Device %d: %s\\n\u0026#34;, i, prop.name); } printf(\u0026#34;\\n\u0026#34;); distributedArray distArray[nDevices]; // enable p2p access between GPUs (if possible) int access; for (int j = 0; j \u0026lt; nDevices; ++j) { // sets GPU cudaSetDevice(j); // allocate device array on the set GPU cudaMalloc(\u0026amp;distArray[j].a_d, nbytes); setVal\u0026lt;\u0026lt;\u0026lt;N/1024, 1024\u0026gt;\u0026gt;\u0026gt;(distArray[j].a_d, j); for (int i = j+1; i \u0026lt; nDevices; ++i) { cudaDeviceCanAccessPeer(\u0026amp;access, j, i); if (access == 1) { cudaSetDevice(j); cudaDeviceEnablePeerAccess(i, 0); cudaSetDevice(i); cudaDeviceEnablePeerAccess(j, 0); } } } // allocating array to record inter-GPU bandwidths float bandwidth[nDevices][nDevices]; for (int i = 0; i \u0026lt; nDevices; ++i) { for (int j = 0; j \u0026lt; nDevices; ++j) { bandwidth[i][j] = 0.0; } } // measure p2p bandwidth between each pair of GPUs cudaEvent_t startEvent, stopEvent; float time; float *array = (float*)malloc(nbytes); for (int j = 0; j \u0026lt; nDevices; ++j) { cudaSetDevice(j); cudaEventCreate(\u0026amp;startEvent); cudaEventCreate(\u0026amp;stopEvent); for (int i = 0; i \u0026lt; nDevices; ++i) { if (i == j) continue; cudaEventRecord(startEvent, 0); cudaMemcpyPeer(distArray[j].a_d, j, distArray[i].a_d, i, nbytes); cudaEventRecord(stopEvent, 0); cudaEventSynchronize(stopEvent); cudaEventElapsedTime(\u0026amp;time, startEvent, stopEvent); cudaMemcpy(array, distArray[j].a_d, nbytes, cudaMemcpyDeviceToHost); for (int k = 0; k \u0026lt; N; ++k) { if (array[k] != i) { printf(\u0026#34;Transfer between GPUs %d and %d failed!\\n\u0026#34;, j, i); std::exit(1); } } bandwidth[j][i] = N*sizeof(float)/time/1.0e6; } setVal\u0026lt;\u0026lt;\u0026lt;N/1024, 1024\u0026gt;\u0026gt;\u0026gt;(distArray[j].a_d, j); cudaEventDestroy(startEvent); cudaEventDestroy(stopEvent); } printf(\u0026#34;Bandwidth (GB/s) for transfer size (MB): %f\\n\u0026#34;, nbytes/1024.0/1024.0); printf(\u0026#34; S\\\\R 0\u0026#34;); for (int i = 1; i \u0026lt; nDevices; ++i) printf(\u0026#34; %3d\u0026#34;, i); printf(\u0026#34;\\n\u0026#34;); for (int j = 0; j \u0026lt; nDevices; ++j) { printf(\u0026#34;%3d\u0026#34;, j); for (int i = 0; i \u0026lt; nDevices; ++i) { if (i==j) { printf(\u0026#34; 0 \u0026#34;); } else { printf(\u0026#34;%8.2f\u0026#34;, bandwidth[j][i]); } } printf(\u0026#34;\\n\u0026#34;); } // cleanup for (int j = 0; j \u0026lt; nDevices; ++j) { cudaFree(distArray[j].a_d); } } Code (Fortran) # program p2pBandwidth use cudafor implicit none integer, parameter:: N = 4*1024*1024 type distributedArray real, device, allocatable:: a_d(:) end type distributedArray type(distributedArray), allocatable:: distArray(:) real, allocatable:: bandwidth(:,:) real:: array(N), time integer:: nDevices, access, i, j, istat type(cudaDeviceProp):: prop type(cudaEvent):: startEvent, stopEvent ! displaying number of GPUs istat = cudaGetDeviceCount(nDevices) write(*,\u0026#34;(\u0026#39;Number of CUDA-capable devices: \u0026#39;, i0,/)\u0026#34;) nDevices ! displaying GPU names do i = 0, nDevices-1 istat = cudaGetDeviceProperties(prop, i) write(*,\u0026#34;(\u0026#39;Device \u0026#39;, i0, \u0026#39;: \u0026#39;, a)\u0026#34;) i, trim(prop%name) end do write(*,*) ! creating device array for each GPU allocate(distArray(0:nDevices-1)) ! enable p2p access between GPUs (if possible) do j = 0, nDevices-1 istat = cudaSetDevice(j) ! sets GPU allocate(distArray(j)%a_d(N)) ! allocates device array on the set GPU distArray(j)%a_d = j do i = j+1, nDevices-1 istat = cudaDeviceCanAccessPeer(access, j, i) if (access == 1) then istat = cudaSetDevice(j) istat = cudaDeviceEnablePeerAccess(i, 0) istat = cudaSetDevice(i) istat = cudaDeviceEnablePeerAccess(j, 0) end if end do end do ! allocating array to record inter-GPU bandwidths allocate(bandwidth(0:nDevices-1, 0:nDevices-1)) bandwidth(:,:) = 0. ! measure p2p bandwidth between each pair of GPUs do j = 0, nDevices-1 istat = cudaSetDevice(j) istat = cudaEventCreate(startEvent) istat = cudaEventCreate(stopEvent) do i = 0, nDevices-1 if (i == j) cycle istat = cudaMemcpyPeer(distArray(j)%a_d, j, distArray(i)%a_d, i, N) istat = cudaEventRecord(startEvent, 0) istat = cudaMemcpyPeer(distArray(j)%a_d, j, distArray(i)%a_d, i, N) istat = cudaEventRecord(stopEvent, 0) istat = cudaEventSynchronize(stopEvent) istat = cudaEventElapsedTime(time, startEvent, stopEvent) array = distArray(j)%a_d if (all(array == i)) bandwidth(j,i) = N*4/time/1.0E+6 end do distArray(j)%a_d = j istat = cudaEventDestroy(startEvent) istat = cudaEventDestroy(stopEvent) end do write(*,\u0026#34;(\u0026#39;Bandwidth (GB/s) for transfer size (MB): \u0026#39;, f9.3,/)\u0026#34;) N*4./1024**2 write(*,\u0026#34;(\u0026#39; S\\\\R 0\u0026#39;)\u0026#34;, advance=\u0026#39;no\u0026#39;) do i = 1, nDevices-1 write(*,\u0026#34;(5x,i3)\u0026#34;, advance=\u0026#39;no\u0026#39;) i end do write(*,*) do j = 0, nDevices-1 write(*,\u0026#34;(i3)\u0026#34;, advance=\u0026#39;no\u0026#39;) j do i = 0, nDevices-1 if (i==j) then write(*,\u0026#34;(4x,\u0026#39;0\u0026#39;,3x)\u0026#34;, advance=\u0026#39;no\u0026#39;) else write(*,\u0026#34;(f8.2)\u0026#34;, advance=\u0026#39;no\u0026#39;) bandwidth(j,i) end if end do write(*,*) end do ! cleanup do j = 0, nDevices-1 deallocate(distArray(j)%a_d) end do deallocate(distArray, bandwidth) end program p2pBandwidth "},{"id":36,"href":"/worknotes/docs/cuda/reference-codes/mpiDevices/","title":"mpiDevices","section":"nvfortran and nvc++ reference codes","content":" mpiDevices # Description # Code to get started with using MPI with nvfortran. All it does is check compute mode:\ndefault (0): multiple host threads can use a single GPU exclusive (1): one host thread can use a single GPU at a time prohibited (2): No host threads can use the GPU exclusive process (3): Single contect cna be created by a single process, but that process can be current to all threads of that process. GPUs in default mode will allow for multiple MPI processes to be assigned to a single GPU, whereas exclusive and exclusive process will only allow one MPI process per GPU.\nCode (C++) # To do... Code (Fortran) # program mpiDevices use cudafor use mpi implicit none ! global array size integer, parameter:: n = 1024*1024 ! MPI variables integer:: procid, numprocs, ierr ! device type(cudaDeviceProp):: prop integer(int_ptr_kind()):: freeB, totalB, freeA, totalA real, device, allocatable:: d(:) integer:: i, j, istat, devid ! MPI initialization call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, procid, ierr) call MPI_COMM_SIZE(MPI_COMM_WORLD, numprocs, ierr) ! print compute mode for device istat = cudaGetDevice(devid) istat = cudaGetDeviceProperties(prop, devid) do i = 1, numprocs call MPI_BARRIER(MPI_COMM_WORLD, ierr) if (procid == i) write(*, \u0026amp; \u0026#34;(\u0026#39;[\u0026#39;,i0,\u0026#39;] using device: \u0026#39;, i0, \u0026#39; in compute mode: \u0026#39;, i0)\u0026#34;) \u0026amp; procid, devid, prop%computeMode end do ! get memory use before large allocations call MPI_BARRIER(MPI_COMM_WORLD, ierr) istat = cudaMemGetInfo(freeB, totalB) ! now allocate arrays, one rank at a time do j = 0, numprocs-1 ! allocate on device associated with rank j call MPI_BARRIER(MPI_COMM_WORLD, ierr) if (procid == j) allocate(d(n)) ! Get free memory after allocation call MPI_BARRIER(MPI_COMM_WORLD, ierr) istat = cudaMemGetInfo(freeA, totalA) write(*, \u0026#34;(\u0026#39; [\u0026#39;,i0,\u0026#39;] after allocation on rank: \u0026#39;, i0, \u0026amp; \u0026#39;, device arrays allocated: \u0026#39;, i0)\u0026#34;) \u0026amp; procid, devid, (freeB-freeA)/n/4 end do deallocate(d) call MPI_FINALIZE(ierr) end program mpiDevices "},{"id":37,"href":"/worknotes/docs/cuda/reference-codes/mpiDeviceUtil/","title":"mpiDeviceUtil","section":"nvfortran and nvc++ reference codes","content":" mpiDeviceUtil # Description # Basic module to assign MPI processes to unique GPUs. This is modified from the code in the book to use MPI_ALLGATHER, an explicit definition of the quicksort subroutine, and the use of hostnm Fortran intrinsic function instead of the MPI_GET_PROCESSOR_NAME.\nCode (C++) # To do... Code (Fortran) # module mpiDeviceUtil contains subroutine assignDevice(procid, numprocs, dev) use mpi use cudafor implicit none integer:: numprocs, procid, dev character(len=100), allocatable:: hosts(:) character(len=100):: hostname integer:: namelength, color, i integer:: newComm, newProcid, ierr logical:: mpiInitialized ! allocate array of hostnames allocate(hosts(0:numprocs-1)) ! every process collects the hostname of all the nodes call hostnm(hostname) call MPI_ALLGATHER(hostname, 100, MPI_CHARACTER, hosts, 100, \u0026amp; MPI_CHARACTER, MPI_COMM_WORLD, ierr) call MPI_BARRIER(MPI_COMM_WORLD,ierr) ! sort the list of names call quicksort(hosts, 100, 1, numprocs) call MPI_BARRIER(MPI_COMM_WORLD,ierr) ! assign the same color to the same node color = 0 do i = 0, numprocs-1 if (i \u0026gt; 0) then if ( hosts(i-1) /= hosts(i) ) color = color + 1 end if if ( hostname \u0026lt;= hosts(i) ) exit end do call MPI_COMM_SPLIT(MPI_COMM_WORLD, color, 0, newComm, ierr) call MPI_COMM_RANK(newComm, newProcid, ierr) dev = newProcid ierr = cudaSetDevice(dev) deallocate(hosts) end subroutine assignDevice ! quicksort.f -*-f90-*- ! Author: t-nissie ! License: GPLv3 ! Gist: https://gist.github.com/t-nissie/479f0f16966925fa29ea !! recursive subroutine quicksort(a, strlen, first, last) implicit none integer, intent(in):: strlen, first, last character(strlen), intent(inout):: a(*) character(strlen):: x, t integer:: i, j x = a( (first+last) / 2 ) i = first j = last do do while (a(i) \u0026lt; x) i=i+1 end do do while (x \u0026lt; a(j)) j=j-1 end do if (i \u0026gt;= j) exit t = a(i); a(i) = a(j); a(j) = t i=i+1 j=j-1 end do if (first \u0026lt; i-1) call quicksort(a, strlen, first, i-1) if (j+1 \u0026lt; last) call quicksort(a, strlen, j+1, last) end subroutine quicksort end module mpiDeviceUtil program main use mpi use cudafor use mpiDeviceUtil implicit none integer, parameter:: n = 1024*1024 ! mpi character(len=100):: hostname integer:: procid, numprocs, ierr, namelength ! device type(cudaDeviceProp):: prop integer(int_ptr_kind()):: freeB, totalB, freeA, totalA real, device, allocatable:: d(:) integer:: deviceID, i, istat call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, procid, ierr) call MPI_COMM_SIZE(MPI_COMM_WORLD, numprocs, ierr) ! get and set unique device call assignDevice(procid, numprocs, deviceID) ! print hostname and device ID for each rank call hostnm(hostname) do i = 0, numprocs-1 call MPI_BARRIER(MPI_COMM_WORLD, ierr) if (i == procid) write(*, \u0026amp; \u0026#34;(\u0026#39;[\u0026#39;,i0,\u0026#39;] host: \u0026#39;, a, \u0026#39;, device: \u0026#39;, i0)\u0026#34;) \u0026amp; procid, trim(hostname), deviceID end do ! get memory use before large allocations call MPI_BARRIER(MPI_COMM_WORLD, ierr) istat = cudaMemGetInfo(freeB, totalB) ! allocate memory on each device call MPI_BARRIER(MPI_COMM_WORLD, ierr) allocate(d(n)) ! get free memory after allocation call MPI_BARRIER(MPI_COMM_WORLD, ierr) istat = cudaMemGetInfo(freeA, totalA) do i = 0, numprocs-1 call MPI_BARRIER(MPI_COMM_WORLD, ierr) if (i == procid) write(*, \u0026amp; \u0026#34;(\u0026#39; [\u0026#39;, i0, \u0026#39;] \u0026#39;, \u0026#39;device arrays allocated: \u0026#39;, i0)\u0026#34;) \u0026amp; procid, (freeB-freeA)/n/4 end do deallocate(d) call MPI_FINALIZE(ierr) end program main "},{"id":38,"href":"/worknotes/docs/cuda/reference-codes/transposeMPI/","title":"transposeMPI","section":"nvfortran and nvc++ reference codes","content":" transposeMPI # Description # Code showing the use of tiling and shared memory in transposing a matrix. The book uses it as an example of the performance difference between CUDA aware MPI vs non-MPI transfers (transfers between GPUs via their respective host CPUs). The code will work on GPUs communicating across node boundaries.\nCode (C++) # To do... Code (Fortran) # module transpose_m implicit none integer, parameter:: cudaTileDim = 32 integer, parameter:: blockRows = 8 contains attributes(global) subroutine cudaTranspose(odata, ldo, idata, ldi) real, intent(out):: odata(ldo, *) real, intent(in):: idata(ldi, *) integer, value, intent(in):: ldo, ldi real, shared:: tile(cudaTileDim+1, cudaTileDim) integer:: x, y, j x = (blockIdx%x-1) * cudaTileDim + threadIdx%x y = (blockIdx%y-1) * cudaTileDim + threadIdx%y do j = 0, cudaTileDim-1, blockRows tile(threadIdx%x, threadIdx%y+j) = idata(x, y+j) end do call syncthreads() x = (blockIdx%y-1) * cudaTileDim + threadIdx%x y = (blockIdx%x-1) * cudaTileDim + threadIdx%y do j = 0, cudaTileDim-1, blockRows odata(x, y+j) = tile(threadIdx%y+j, threadIdx%x) end do end subroutine end module transpose_m ! ! Main code ! program transpose_MPI use cudafor use mpi use transpose_m use mpiDeviceUtil implicit none ! global array size integer, parameter:: nx = 2048, ny = 2048 ! host arrays (global) real:: idata_h(nx, ny), tdata_h(ny, nx), gold(ny, nx) ! CUDA vars and device arrays integer:: deviceID, nDevices type(dim3):: dimGrid, dimBlock real, device, allocatable:: idata_d(:, :), tdata_d(:, :), sTile_d(:, :), \u0026amp; rTile_d(:, :) ! MPI stuff integer:: mpiTileDimX, mpiTileDimY integer:: procid, numprocs, tag, ierr, localRank integer:: nstages, stage, sRank, rRank integer:: status(MPI_STATUS_SIZE) double precision:: timeStart, timeStop character(len=10):: localRankStr integer:: i, j, nyl, jl, jg, p integer:: xOffset, yOffset ! MPI initialization call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, procid, ierr) call MPI_COMM_SIZE(MPI_COMM_WORLD, numprocs, ierr) ierr = cudaGetDeviceCount(nDevices) ! check parameters and calculate execution configuration if (mod(nx, numprocs) == 0 .and. mod(ny, numprocs) == 0) then mpiTileDimX = nx/numprocs mpiTileDimY = ny/numprocs else write(*,*) \u0026#34;ny must be an integral multiple of numprocs\u0026#34; call MPI_ABORT(MPI_COMM_WORLD, 1, ierr) end if if (mod(mpiTileDimX, cudaTileDim) /= 0 .or. mod(mpiTileDimY, cudaTileDim) \u0026amp; /= 0) then write(*,*) \u0026#34;mpiTileDimX and mpiTileDimY must be an integral multiple\u0026#34;, \u0026amp; \u0026#34; of cudaTileDim\u0026#34; call MPI_ABORT(MPI_COMM_WORLD, 1, ierr) end if if (numprocs \u0026gt; nDevices) then write(*,*) \u0026#34;numprcs must be \u0026lt;= number of GPUs on the system!\u0026#34; call MPI_ABORT(MPI_COMM_WORLD, 1, ierr) end if ! each MPI process call assignDevice(procid, numprocs, deviceID) dimGrid = dim3(mpiTileDimX/cudaTileDim, mpiTileDimY/cudaTileDim, 1) dimBlock = dim3(cudaTileDim, blockRows, 1) ! write parameters to terminal if (procid == 0) then write(*,*) write(*,\u0026#34;(\u0026#39;Array size: \u0026#39;, i0,\u0026#39; x \u0026#39;,i0,/)\u0026#34;) nx, ny write(*,\u0026#34;(\u0026#39;CUDA block size: \u0026#39;, i0,\u0026#39; x \u0026#39;,i0,\u0026#39;, CUDA tile size: \u0026#39;, \u0026amp; i0,\u0026#39; x \u0026#39;,i0,/)\u0026#34;) cudaTileDim, blockRows, cudaTileDim, cudaTileDim write(*,\u0026#34;(\u0026#39;dimGrid: \u0026#39;, i0,\u0026#39; x \u0026#39;,i0,\u0026#39; x \u0026#39;,i0,\u0026#39; dimBlock: \u0026#39;, i0,\u0026amp; \u0026#39; x \u0026#39;,i0,\u0026#39; x \u0026#39;,i0,/)\u0026#34;) dimGrid%x, dimGrid%y, dimGrid%x, \u0026amp; dimBlock%x, dimBlock%y, dimBlock%z write(*,\u0026#34;(\u0026#39;numprocs: \u0026#39;, i0, \u0026#39;, Local input array size: \u0026#39;, \u0026amp; i0,\u0026#39; x \u0026#39;,i0,/)\u0026#34;) numprocs, nx, mpiTileDimY write(*,\u0026#34;(\u0026#39;mpiTileDim: \u0026#39;, i0,\u0026#39; x \u0026#39;,i0,/)\u0026#34;) mpiTileDimX, mpiTileDimY end if ! initalize data ! host - each process has entire array on host (for now) do p = 0, numprocs-1 do jl = 1, mpiTileDimY jg = p*mpiTileDimY + jl do i = 1, nx idata_h(i, jg) = i + (jg-1)*nx end do end do end do gold = transpose(idata_h) ! device - each process has nx*mpiTileDimY = ny*mpiTileDimX elements allocate( idata_d(nx, mpiTileDimY), \u0026amp; tdata_d(ny, mpiTileDimX), \u0026amp; sTile_d(mpiTileDimX, mpiTileDimY), \u0026amp; rTile_d(mpiTileDimX, mpiTileDimY)) yOffset = procid*mpiTileDimY idata_d(1:nx, 1:mpiTileDimY) = idata_h(1:nx, yOffset+1:yOffset+mpiTileDimY) tdata_d = -1.0 ! --------- ! transpose ! --------- call MPI_BARRIER(MPI_COMM_WORLD, ierr) timeStart = MPI_WTIME() ! 0th stage - local transpose call cudaTranspose\u0026lt;\u0026lt;\u0026lt;dimGrid, dimBlock\u0026gt;\u0026gt;\u0026gt;\u0026amp; (tdata_d(procid*mpiTileDimY+1, 1), ny, \u0026amp; idata_d(procid*mpiTileDimX+1, 1), nx) ! other stages that involve MPI transfers do stage = 1, numprocs-1 ! sRank = the rank to which procid send data to ! rRank = the rank from which myrank receives data sRank = modulo(procid-stage, numprocs) rRank = modulo(procid+stage, numprocs) call MPI_BARRIER(MPI_COMM_WORLD, ierr) ! pack tile so data to be sent is contiguous !$cuf kernel do (2) \u0026lt;\u0026lt;\u0026lt;*, *\u0026gt;\u0026gt;\u0026gt; do j = 1, mpiTileDimY do i = 1, mpiTileDimX sTile_d(i, j) = idata_d(sRank*mpiTileDimX+i, j) end do end do call MPI_SENDRECV(sTile_d, mpiTileDimX*mpiTileDimY, MPI_REAL, sRank, \u0026amp; procid, rTile_d, mpiTileDimX*mpiTileDimY, MPI_REAL, rRank, rRank, \u0026amp; MPI_COMM_WORLD, status, ierr) ! do transpose from receive tile into final array ! (no need to unpack) call cudaTranspose\u0026lt;\u0026lt;\u0026lt;dimGrid, dimBlock\u0026gt;\u0026gt;\u0026gt;\u0026amp; (tdata_d(rRank*mpiTileDimY+1, 1), ny, rTile_d, mpiTileDimX) end do call MPI_BARRIER(MPI_COMM_WORLD, ierr) timeStop = MPI_WTIME() ! check results tdata_h = tdata_d xOffset = procid*mpiTileDimX if (all(tdata_h(1:ny, 1:mpiTileDimX) == \u0026amp; gold(1:ny, xOffset+1:xOffset+mpiTileDimX))) then if (procid == 0) then write(*,\u0026#34;(\u0026#39;Bandwidth (GB/s): \u0026#39;, f7.2,/)\u0026#34;) 2.*(nx*ny*4)/\u0026amp; (1.e9*(timeStop-timeStart)) end if else write(*,\u0026#34;(\u0026#39;[\u0026#39;,i0,\u0026#39;]\u0026#39;, \u0026#39;*** Failed ***\u0026#39;)\u0026#34;) procid end if deallocate(idata_d, tdata_d, sTile_d, rTile_d) call MPI_FINALIZE(ierr) end program transpose_MPI "},{"id":39,"href":"/worknotes/docs/cuda/hidden/","title":"Hidden","section":"CUDA Programming","content":" This page is hidden in menu # Quondam non pater est dignior ille Eurotas # Latent te facies # Lorem markdownum arma ignoscas vocavit quoque ille texit mandata mentis ultimus, frementes, qui in vel. Hippotades Peleus pennas conscia cuiquam Caeneus quas.\nPater demittere evincitque reddunt Maxime adhuc pressit huc Danaas quid freta Soror ego Luctus linguam saxa ultroque prior Tatiumque inquit Saepe liquitur subita superata dederat Anius sudor Cum honorum Latona # O fallor in sustinui iussorum equidem. Nymphae operi oris alii fronde parens dumque, in auro ait mox ingenti proxima iamdudum maius?\nreality(burnDocking(apache_nanometer), pad.property_data_programming.sectorBrowserPpga(dataMask, 37, recycleRup)); intellectualVaporwareUser += -5 * 4; traceroute_key_upnp /= lag_optical(android.smb(thyristorTftp)); surge_host_golden = mca_compact_device(dual_dpi_opengl, 33, commerce_add_ppc); if (lun_ipv) { verticalExtranet(1, thumbnail_ttl, 3); bar_graphics_jpeg(chipset - sector_xmp_beta); } Fronde cetera dextrae sequens pennis voce muneris # Acta cretus diem restet utque; move integer, oscula non inspirat, noctisque scelus! Nantemque in suas vobis quamvis, et labori!\nvar runtimeDiskCompiler = home - array_ad_software; if (internic \u0026gt; disk) { emoticonLockCron += 37 + bps - 4; wan_ansi_honeypot.cardGigaflops = artificialStorageCgi; simplex -= downloadAccess; } var volumeHardeningAndroid = pixel + tftp + onProcessorUnmount; sector(memory(firewire + interlaced, wired)); "}]
\ No newline at end of file
diff --git a/public/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js b/public/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js
new file mode 100644
index 0000000..3c16ad2
--- /dev/null
+++ b/public/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js
@@ -0,0 +1 @@
+"use strict";(function(){const o="/worknotes/en.search-data.min.bfd72454f23cce5f215a08402e127f32c4fb44ac8cdb9e40d8d0c733fd85a1b6.json",i=Object.assign({cache:!0},{doc:{id:"id",field:["title","content"],store:["title","href","section"]}}),e=document.querySelector("#book-search-input"),t=document.querySelector("#book-search-results");if(!e)return;e.addEventListener("focus",n),e.addEventListener("keyup",s),document.addEventListener("keypress",a);function a(t){if(t.target.value!==void 0)return;if(e===document.activeElement)return;const n=String.fromCharCode(t.charCode);if(!r(n))return;e.focus(),t.preventDefault()}function r(t){const n=e.getAttribute("data-hotkeys")||"";return n.indexOf(t)>=0}function n(){e.removeEventListener("focus",n),e.required=!0,fetch(o).then(e=>e.json()).then(e=>{window.bookSearchIndex=FlexSearch.create("balance",i),window.bookSearchIndex.add(e)}).then(()=>e.required=!1).then(s)}function s(){for(;t.firstChild;)t.removeChild(t.firstChild);if(!e.value)return;const n=window.bookSearchIndex.search(e.value,10);n.forEach(function(e){const n=c("<li><a href></a><small></small></li>"),s=n.querySelector("a"),o=n.querySelector("small");s.href=e.href,s.textContent=e.title,o.textContent=e.section,t.appendChild(n)})}function c(e){const t=document.createElement("div");return t.innerHTML=e,t.firstChild}})()
\ No newline at end of file
diff --git a/public/worknotes/index.html b/public/worknotes/index.html
index e5d10ba..dea9655 100644
--- a/public/worknotes/index.html
+++ b/public/worknotes/index.html
@@ -17,7 +17,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/index.xml" title="Ed's Space - Notes" />
diff --git a/public/worknotes/sitemap.xml b/public/worknotes/sitemap.xml
index 0a1d06f..f6134c4 100644
--- a/public/worknotes/sitemap.xml
+++ b/public/worknotes/sitemap.xml
@@ -69,7 +69,7 @@
     <lastmod>2023-03-18T01:36:43+11:00</lastmod>
   </url><url>
     <loc>/worknotes/docs/useful/grid-rows-spatial-hashing/</loc>
-    <lastmod>2023-03-18T01:36:43+11:00</lastmod>
+    <lastmod>2023-09-10T15:17:43+10:00</lastmod>
   </url><url>
     <loc>/worknotes/docs/pi/</loc>
     <lastmod>2023-03-18T01:36:43+11:00</lastmod>
@@ -136,7 +136,7 @@
     <loc>/worknotes/categories/</loc>
   </url><url>
     <loc>/worknotes/docs/</loc>
-    <lastmod>2023-09-09T12:42:53+10:00</lastmod>
+    <lastmod>2023-09-10T15:17:43+10:00</lastmod>
   </url><url>
     <loc>/worknotes/</loc>
     <lastmod>2023-03-18T01:43:41+11:00</lastmod>
diff --git a/public/worknotes/tags/index.html b/public/worknotes/tags/index.html
index c155297..6fd6622 100644
--- a/public/worknotes/tags/index.html
+++ b/public/worknotes/tags/index.html
@@ -14,7 +14,7 @@
 <link rel="icon" href="/worknotes/favicon.png" type="image/x-icon">
 <link rel="stylesheet" href="/worknotes/book.min.16fde4d05cd1cf758f445b62581a359a2b0aa3a8b1be4966bf73546658560a86.css" >
   <script defer src="/worknotes/flexsearch.min.js"></script>
-  <script defer src="/worknotes/en.search.min.ccdff01d8c2f60d0d214910635ea86a7eaff4df870afc24daf6dd740bfdb0152.js" ></script>
+  <script defer src="/worknotes/en.search.min.c7e0237ad85bf0edc4423653a6946844778e5633aa3fa9785d2dab3a945175fe.js" ></script>
 
   <script defer src="/worknotes/sw.min.97afbbb840f8817a158b86f13f6a071d4ddf793bff2b4832f0bea2832fcea52b.js" ></script>
 <link rel="alternate" type="application/rss+xml" href="/worknotes/tags/index.xml" title="Ed's Space - Notes" />
diff --git a/src/worknotes/content.en/docs/useful/fixed-cutoff-cell-lists-pair-search.md b/src/worknotes/content.en/docs/useful/fixed-cutoff-cell-lists-pair-search.md
index 40a65bc..355cf99 100644
--- a/src/worknotes/content.en/docs/useful/fixed-cutoff-cell-lists-pair-search.md
+++ b/src/worknotes/content.en/docs/useful/fixed-cutoff-cell-lists-pair-search.md
@@ -54,6 +54,7 @@ contains
 
    !---------------------------------------------------------------------------
    function coordsToCell(dim, xi, minx, cutoff) result(icell)
+      ! function to convert coordinates to grid cells
 
       implicit none
       real(f), intent(in):: xi(dim), minx(3), cutoff
@@ -76,20 +77,24 @@ contains
       real(f):: r2, minx(3), maxx(3)
       integer, allocatable:: pincell(:, :, :), cells(:, :, :, :)
 
+
+      ! determine grid min-max coordinates, with concessions made for dim=2 case
       minx(1:dim) = minval(x, dim=2)
       if (dim == 2) minx(3) = 0.d0
       maxx(1:dim) = maxval(x, dim=2)
       if (dim == 2) maxx(3) = 0.d0
+      
+      ! creating buffer layers
       minx(:) = minx(:) - 2.d0*cutoff
       maxx(:) = maxx(:) + 2.d0*cutoff
-      ngridx(:) = int((maxx(:) - minx(:))/cutoff) + 1
-      maxx(:) = maxx(:) + ngridx(:)*cutoff
 
+      ! initializing grid arrays
       allocate (pincell(ngridx(1), ngridx(2), ngridx(3)), &
                 cells(maxpercell, ngridx(1), ngridx(2), ngridx(3)))
 
       pincell(:, :, :) = 0
 
+      ! populating grid with point information
       do i = 1, npoints
          icell = coordsToCell(dim, x(:, i), minx, cutoff)
          pincell(icell(1), icell(2), icell(3)) = &
@@ -105,6 +110,7 @@ contains
          cells(pincell(icell(1), icell(2), icell(3)), icell(1), icell(2), icell(3)) = i
       end do
 
+      ! perform pair search
       npairs = 0
       do i = 1, npoints
          icell(:) = coordsToCell(dim, x(:, i), minx, cutoff)
diff --git a/src/worknotes/content.en/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search.md b/src/worknotes/content.en/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search.md
index 99f26eb..242c4c7 100644
--- a/src/worknotes/content.en/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search.md
+++ b/src/worknotes/content.en/docs/useful/fixed-cutoff-cell-lists-pair-searchhalf-search.md
@@ -118,20 +118,23 @@ contains
                                                          0, -1,  1, &
                                                          0,  0, -1], [3, 13])
 
+      ! determine grid min-max coordinates, with concessions made for dim=2 case
       minx(1:dim) = minval(x, dim=2)
       if (dim == 2) minx(3) = 0.d0
       maxx(1:dim) = maxval(x, dim=2)
       if (dim == 2) maxx(3) = 0.d0
+      
+      ! creating buffer layers
       minx(:) = minx(:) - 2.d0*cutoff
       maxx(:) = maxx(:) + 2.d0*cutoff
-      ngridx(:) = int((maxx(:) - minx(:))/cutoff) + 1
-      maxx(:) = maxx(:) + ngridx(:)*cutoff
 
+      ! initializing grid arrays
       allocate (pincell(ngridx(1), ngridx(2), ngridx(3)), &
                 cells(maxpercell, ngridx(1), ngridx(2), ngridx(3)))
 
       pincell(:, :, :) = 0
 
+      ! populating grid with point information
       do i = 1, npoints
          icell = coordsToCell(dim, x(:, i), minx, cutoff)
          pincell(icell(1), icell(2), icell(3)) = &
diff --git a/src/worknotes/content.en/docs/useful/fixed-cutoff-direct-pair-search.md b/src/worknotes/content.en/docs/useful/fixed-cutoff-direct-pair-search.md
index cda9f80..ea82a77 100644
--- a/src/worknotes/content.en/docs/useful/fixed-cutoff-direct-pair-search.md
+++ b/src/worknotes/content.en/docs/useful/fixed-cutoff-direct-pair-search.md
@@ -50,6 +50,8 @@ contains
 
    !---------------------------------------------------------------------------
    subroutine dsearch_dp(dim, npoints, x, cutoff, maxnpair, npairs, pair_i, pair_j)
+      ! subroutine to find pairs of points using direct search. Output is 
+      ! two lists describing which points are in the pair
 
       implicit none
       integer, intent(in):: dim, npoints, maxnpair
@@ -75,6 +77,8 @@ contains
 
    !---------------------------------------------------------------------------
    subroutine dsearch_compact_dp(dim, npoints, x, cutoff, maxnpair, npairs, endpos, pair_j)
+      ! subroutine to find pairs using direct search. Output is two lists. One
+      ! list describes the last pair with point `i` as the "left-sided" point.
 
       implicit none
       integer, intent(in):: dim, npoints, maxnpair
@@ -93,6 +97,7 @@ contains
                pair_j(npairs) = j
             end if
          end do
+         ! update end position when done with point i
          endpos(i) = npairs
       end do
 
diff --git a/src/worknotes/content.en/docs/useful/grid-rows-spatial-hashing.md b/src/worknotes/content.en/docs/useful/grid-rows-spatial-hashing.md
index 71f86ad..e4881f5 100644
--- a/src/worknotes/content.en/docs/useful/grid-rows-spatial-hashing.md
+++ b/src/worknotes/content.en/docs/useful/grid-rows-spatial-hashing.md
@@ -40,7 +40,7 @@ Another benefit is the lower memory usage by the algorithm. This is because it d
 
 Save the C++ file as `sort.cpp`, and Fortran code as `cll3.F90`. Compile the code with
 
-```bash
+```bash {linenos=false,style=tango}
 g++ -c sort.cpp
 gfortran -o cll3.x cll3.F90 sort.o -lstdc++
 ```