prov/{rxd/shm/sm2/udp/util}: util_cq ep_list_lock to genlock

Change util_cq->ep_list_lock from ofi_mutex to ofi_genlock. This will
allow us to do less locking on the critical path, which will make every
provider that uses common utilities faster.

Signed-off-by: Seth Zegelstein <szegel@amazon.com>

include/ofi_util.h

@@ -504,7 +504,7 @@ struct util_cq {
 	struct util_wait	*wait;
 	ofi_atomic32_t		ref;
 	struct dlist_entry	ep_list;
-	ofi_mutex_t		ep_list_lock;
+	struct ofi_genlock	ep_list_lock;
 	struct ofi_genlock	cq_lock;
 	uint64_t		flags;
 

prov/rxd/src/rxd_cq.c

@@ -52,15 +52,15 @@ static const char *rxd_cq_strerror(struct fid_cq *cq_fid, int prov_errno,
 
 	cq = container_of(cq_fid, struct rxd_cq, util_cq.cq_fid);
 
-	ofi_mutex_lock(&cq->util_cq.ep_list_lock);
+	ofi_genlock_lock(&cq->util_cq.ep_list_lock);
 	assert(!dlist_empty(&cq->util_cq.ep_list));
 	fid_entry = container_of(cq->util_cq.ep_list.next,
 				struct fid_list_entry, entry);
 	util_ep = container_of(fid_entry->fid, struct util_ep, ep_fid.fid);
 	ep = container_of(util_ep, struct rxd_ep, util_ep);
 
 	str = fi_cq_strerror(ep->dg_cq, prov_errno, err_data, buf, len);
-	ofi_mutex_unlock(&cq->util_cq.ep_list_lock);
+	ofi_genlock_unlock(&cq->util_cq.ep_list_lock);
 	return str;
 }
 
@@ -1257,7 +1257,7 @@ ssize_t rxd_cq_sreadfrom(struct fid_cq *cq_fid, void *buf, size_t count,
 		}
 
 		ep_retry = -1;
-		ofi_mutex_lock(&cq->ep_list_lock);
+		ofi_genlock_lock(&cq->ep_list_lock);
 		dlist_foreach_container(&cq->ep_list, struct fid_list_entry,
 					fid_entry, entry) {
 			ep = container_of(fid_entry->fid, struct rxd_ep,
@@ -1267,7 +1267,7 @@ ssize_t rxd_cq_sreadfrom(struct fid_cq *cq_fid, void *buf, size_t count,
 			ep_retry = ep_retry == -1 ? ep->next_retry :
 					MIN(ep_retry, ep->next_retry);
 		}
-		ofi_mutex_unlock(&cq->ep_list_lock);
+		ofi_genlock_unlock(&cq->ep_list_lock);
 
 		ret = fi_wait(&cq->wait->wait_fid, ep_retry == -1 ?
 			      timeout : rxd_get_timeout(ep_retry));

prov/shm/src/smr_ep.c

@@ -879,7 +879,7 @@ static int smr_ep_bind_cq(struct smr_ep *ep, struct util_cq *cq, uint64_t flags)
 			return ret;
 	}
 
-	ret = fid_list_insert(&cq->ep_list,
+	ret = fid_list_insert2(&cq->ep_list,
 			      &cq->ep_list_lock,
 			      &ep->util_ep.ep_fid.fid);
 

prov/sm2/src/sm2_ep.c

@@ -311,8 +311,8 @@ static int sm2_ep_bind_cq(struct sm2_ep *ep, struct util_cq *cq, uint64_t flags)
 			return ret;
 	}
 
-	ret = fid_list_insert(&cq->ep_list, &cq->ep_list_lock,
-			      &ep->util_ep.ep_fid.fid);
+	ret = fid_list_insert2(&cq->ep_list, &cq->ep_list_lock,
+			       &ep->util_ep.ep_fid.fid);
 
 	return ret;
 }

prov/udp/src/udpx_ep.c

@@ -563,7 +563,7 @@ static int udpx_ep_close(struct fid *fid)
 					    struct util_wait_fd, util_wait);
 			ofi_epoll_del(wait->epoll_fd, (int)ep->sock);
 		}
-		fid_list_remove(&ep->util_ep.rx_cq->ep_list,
+		fid_list_remove2(&ep->util_ep.rx_cq->ep_list,
 				&ep->util_ep.rx_cq->ep_list_lock,
 				&ep->util_ep.ep_fid.fid);
 	}
@@ -614,7 +614,7 @@ static int udpx_ep_bind_cq(struct udpx_ep *ep, struct util_cq *cq,
 				udpx_rx_src_comp : udpx_rx_comp;
 		}
 
-		ret = fid_list_insert(&cq->ep_list,
+		ret = fid_list_insert2(&cq->ep_list,
 				      &cq->ep_list_lock,
 				      &ep->util_ep.ep_fid.fid);
 		if (ret)

prov/util/src/util_cq.c

@@ -421,7 +421,7 @@ int ofi_cq_cleanup(struct util_cq *cq)
 	}
 
 	ofi_genlock_destroy(&cq->cq_lock);
-	ofi_mutex_destroy(&cq->ep_list_lock);
+	ofi_genlock_destroy(&cq->ep_list_lock);
 	ofi_atomic_dec32(&cq->domain->ref);
 	return 0;
 }
@@ -491,14 +491,14 @@ void ofi_cq_progress(struct util_cq *cq)
 	struct fid_list_entry *fid_entry;
 	struct dlist_entry *item;
 
-	ofi_mutex_lock(&cq->ep_list_lock);
+	ofi_genlock_lock(&cq->ep_list_lock);
 	dlist_foreach(&cq->ep_list, item) {
 		fid_entry = container_of(item, struct fid_list_entry, entry);
 		ep = container_of(fid_entry->fid, struct util_ep, ep_fid.fid);
 		ep->progress(ep);
 
 	}
-	ofi_mutex_unlock(&cq->ep_list_lock);
+	ofi_genlock_unlock(&cq->ep_list_lock);
 }
 
 static ssize_t util_peer_cq_write(struct fid_peer_cq *cq, void *context,
@@ -696,9 +696,6 @@ int ofi_cq_init(const struct fi_provider *prov, struct fid_domain *domain,
 	ofi_atomic_initialize32(&cq->ref, 0);
 	ofi_atomic_initialize32(&cq->wakeup, 0);
 	dlist_init(&cq->ep_list);
-	ret = ofi_mutex_init(&cq->ep_list_lock);
-	if (ret)
-		return ret;
 
 	if (cq->domain->threading == FI_THREAD_COMPLETION ||
 	    cq->domain->threading == FI_THREAD_DOMAIN)
@@ -710,6 +707,10 @@ int ofi_cq_init(const struct fi_provider *prov, struct fid_domain *domain,
 	if (ret)
 		goto destroy1;
 
+	ret = ofi_genlock_init(&cq->ep_list_lock, lock_type);
+	if (ret)
+		return ret;
+
 	cq->flags = attr->flags;
 	cq->cq_fid.fid.fclass = FI_CLASS_CQ;
 	cq->cq_fid.fid.context = context;
@@ -769,7 +770,7 @@ int ofi_cq_init(const struct fi_provider *prov, struct fid_domain *domain,
 destroy2:
 	ofi_genlock_destroy(&cq->cq_lock);
 destroy1:
-	ofi_mutex_destroy(&cq->ep_list_lock);
+	ofi_genlock_destroy(&cq->ep_list_lock);
 	return ret;
 }
 

prov/util/src/util_ep.c

@@ -64,7 +64,7 @@ int ofi_ep_bind_cq(struct util_ep *ep, struct util_cq *cq, uint64_t flags)
 	}
 
 	if (flags & (FI_TRANSMIT | FI_RECV)) {
-		return fid_list_insert(&cq->ep_list,
+		return fid_list_insert2(&cq->ep_list,
 				       &cq->ep_list_lock,
 				       &ep->ep_fid.fid);
 	}
@@ -272,14 +272,14 @@ int ofi_endpoint_close(struct util_ep *util_ep)
 	int i;
 
 	if (util_ep->tx_cq) {
-		fid_list_remove(&util_ep->tx_cq->ep_list,
+		fid_list_remove2(&util_ep->tx_cq->ep_list,
 				&util_ep->tx_cq->ep_list_lock,
 				&util_ep->ep_fid.fid);
 		ofi_atomic_dec32(&util_ep->tx_cq->ref);
 	}
 
 	if (util_ep->rx_cq) {
-		fid_list_remove(&util_ep->rx_cq->ep_list,
+		fid_list_remove2(&util_ep->rx_cq->ep_list,
 				&util_ep->rx_cq->ep_list_lock,
 				&util_ep->ep_fid.fid);
 		ofi_atomic_dec32(&util_ep->rx_cq->ref);