diff --git a/include/ofi_util.h b/include/ofi_util.h index 1ec3ae4fbe7..64248a4f83c 100644 --- a/include/ofi_util.h +++ b/include/ofi_util.h @@ -504,7 +504,7 @@ struct util_cq { struct util_wait *wait; ofi_atomic32_t ref; struct dlist_entry ep_list; - ofi_mutex_t ep_list_lock; + struct ofi_genlock ep_list_lock; struct ofi_genlock cq_lock; uint64_t flags; diff --git a/prov/rxd/src/rxd_cq.c b/prov/rxd/src/rxd_cq.c index 2d73474c948..4eb33994937 100644 --- a/prov/rxd/src/rxd_cq.c +++ b/prov/rxd/src/rxd_cq.c @@ -52,7 +52,7 @@ static const char *rxd_cq_strerror(struct fid_cq *cq_fid, int prov_errno, cq = container_of(cq_fid, struct rxd_cq, util_cq.cq_fid); - ofi_mutex_lock(&cq->util_cq.ep_list_lock); + ofi_genlock_lock(&cq->util_cq.ep_list_lock); assert(!dlist_empty(&cq->util_cq.ep_list)); fid_entry = container_of(cq->util_cq.ep_list.next, struct fid_list_entry, entry); @@ -60,7 +60,7 @@ static const char *rxd_cq_strerror(struct fid_cq *cq_fid, int prov_errno, ep = container_of(util_ep, struct rxd_ep, util_ep); str = fi_cq_strerror(ep->dg_cq, prov_errno, err_data, buf, len); - ofi_mutex_unlock(&cq->util_cq.ep_list_lock); + ofi_genlock_unlock(&cq->util_cq.ep_list_lock); return str; } @@ -1257,7 +1257,7 @@ ssize_t rxd_cq_sreadfrom(struct fid_cq *cq_fid, void *buf, size_t count, } ep_retry = -1; - ofi_mutex_lock(&cq->ep_list_lock); + ofi_genlock_lock(&cq->ep_list_lock); dlist_foreach_container(&cq->ep_list, struct fid_list_entry, fid_entry, entry) { ep = container_of(fid_entry->fid, struct rxd_ep, @@ -1267,7 +1267,7 @@ ssize_t rxd_cq_sreadfrom(struct fid_cq *cq_fid, void *buf, size_t count, ep_retry = ep_retry == -1 ? ep->next_retry : MIN(ep_retry, ep->next_retry); } - ofi_mutex_unlock(&cq->ep_list_lock); + ofi_genlock_unlock(&cq->ep_list_lock); ret = fi_wait(&cq->wait->wait_fid, ep_retry == -1 ? timeout : rxd_get_timeout(ep_retry)); diff --git a/prov/shm/src/smr_ep.c b/prov/shm/src/smr_ep.c index e1f4a7b7f88..4973882fde3 100644 --- a/prov/shm/src/smr_ep.c +++ b/prov/shm/src/smr_ep.c @@ -879,7 +879,7 @@ static int smr_ep_bind_cq(struct smr_ep *ep, struct util_cq *cq, uint64_t flags) return ret; } - ret = fid_list_insert(&cq->ep_list, + ret = fid_list_insert2(&cq->ep_list, &cq->ep_list_lock, &ep->util_ep.ep_fid.fid); diff --git a/prov/sm2/src/sm2_ep.c b/prov/sm2/src/sm2_ep.c index 555c7384166..6ae2741f7a4 100644 --- a/prov/sm2/src/sm2_ep.c +++ b/prov/sm2/src/sm2_ep.c @@ -311,8 +311,8 @@ static int sm2_ep_bind_cq(struct sm2_ep *ep, struct util_cq *cq, uint64_t flags) return ret; } - ret = fid_list_insert(&cq->ep_list, &cq->ep_list_lock, - &ep->util_ep.ep_fid.fid); + ret = fid_list_insert2(&cq->ep_list, &cq->ep_list_lock, + &ep->util_ep.ep_fid.fid); return ret; } diff --git a/prov/udp/src/udpx_ep.c b/prov/udp/src/udpx_ep.c index 97d574bd1ca..8f08a7bc6bf 100644 --- a/prov/udp/src/udpx_ep.c +++ b/prov/udp/src/udpx_ep.c @@ -563,7 +563,7 @@ static int udpx_ep_close(struct fid *fid) struct util_wait_fd, util_wait); ofi_epoll_del(wait->epoll_fd, (int)ep->sock); } - fid_list_remove(&ep->util_ep.rx_cq->ep_list, + fid_list_remove2(&ep->util_ep.rx_cq->ep_list, &ep->util_ep.rx_cq->ep_list_lock, &ep->util_ep.ep_fid.fid); } @@ -614,7 +614,7 @@ static int udpx_ep_bind_cq(struct udpx_ep *ep, struct util_cq *cq, udpx_rx_src_comp : udpx_rx_comp; } - ret = fid_list_insert(&cq->ep_list, + ret = fid_list_insert2(&cq->ep_list, &cq->ep_list_lock, &ep->util_ep.ep_fid.fid); if (ret) diff --git a/prov/util/src/util_cq.c b/prov/util/src/util_cq.c index 5c3197c5871..1c9dff0bf94 100644 --- a/prov/util/src/util_cq.c +++ b/prov/util/src/util_cq.c @@ -421,7 +421,7 @@ int ofi_cq_cleanup(struct util_cq *cq) } ofi_genlock_destroy(&cq->cq_lock); - ofi_mutex_destroy(&cq->ep_list_lock); + ofi_genlock_destroy(&cq->ep_list_lock); ofi_atomic_dec32(&cq->domain->ref); return 0; } @@ -491,14 +491,14 @@ void ofi_cq_progress(struct util_cq *cq) struct fid_list_entry *fid_entry; struct dlist_entry *item; - ofi_mutex_lock(&cq->ep_list_lock); + ofi_genlock_lock(&cq->ep_list_lock); dlist_foreach(&cq->ep_list, item) { fid_entry = container_of(item, struct fid_list_entry, entry); ep = container_of(fid_entry->fid, struct util_ep, ep_fid.fid); ep->progress(ep); } - ofi_mutex_unlock(&cq->ep_list_lock); + ofi_genlock_unlock(&cq->ep_list_lock); } static ssize_t util_peer_cq_write(struct fid_peer_cq *cq, void *context, @@ -696,9 +696,6 @@ int ofi_cq_init(const struct fi_provider *prov, struct fid_domain *domain, ofi_atomic_initialize32(&cq->ref, 0); ofi_atomic_initialize32(&cq->wakeup, 0); dlist_init(&cq->ep_list); - ret = ofi_mutex_init(&cq->ep_list_lock); - if (ret) - return ret; if (cq->domain->threading == FI_THREAD_COMPLETION || cq->domain->threading == FI_THREAD_DOMAIN) @@ -710,6 +707,10 @@ int ofi_cq_init(const struct fi_provider *prov, struct fid_domain *domain, if (ret) goto destroy1; + ret = ofi_genlock_init(&cq->ep_list_lock, lock_type); + if (ret) + return ret; + cq->flags = attr->flags; cq->cq_fid.fid.fclass = FI_CLASS_CQ; cq->cq_fid.fid.context = context; @@ -769,7 +770,7 @@ int ofi_cq_init(const struct fi_provider *prov, struct fid_domain *domain, destroy2: ofi_genlock_destroy(&cq->cq_lock); destroy1: - ofi_mutex_destroy(&cq->ep_list_lock); + ofi_genlock_destroy(&cq->ep_list_lock); return ret; } diff --git a/prov/util/src/util_ep.c b/prov/util/src/util_ep.c index 889b34f809f..eade45d46db 100644 --- a/prov/util/src/util_ep.c +++ b/prov/util/src/util_ep.c @@ -64,7 +64,7 @@ int ofi_ep_bind_cq(struct util_ep *ep, struct util_cq *cq, uint64_t flags) } if (flags & (FI_TRANSMIT | FI_RECV)) { - return fid_list_insert(&cq->ep_list, + return fid_list_insert2(&cq->ep_list, &cq->ep_list_lock, &ep->ep_fid.fid); } @@ -272,14 +272,14 @@ int ofi_endpoint_close(struct util_ep *util_ep) int i; if (util_ep->tx_cq) { - fid_list_remove(&util_ep->tx_cq->ep_list, + fid_list_remove2(&util_ep->tx_cq->ep_list, &util_ep->tx_cq->ep_list_lock, &util_ep->ep_fid.fid); ofi_atomic_dec32(&util_ep->tx_cq->ref); } if (util_ep->rx_cq) { - fid_list_remove(&util_ep->rx_cq->ep_list, + fid_list_remove2(&util_ep->rx_cq->ep_list, &util_ep->rx_cq->ep_list_lock, &util_ep->ep_fid.fid); ofi_atomic_dec32(&util_ep->rx_cq->ref);