Skip to content

Commit

Permalink
prov/efa: Add setopt/getopt support for remaining EP sizes
Browse files Browse the repository at this point in the history 
This adds support for FI_OPT_{MSG,INJECT}_{TAGGED,ATOMIC}_SIZE

Signed-off-by: Darryl Abbate <drl@amazon.com>
  • Loading branch information
@darrylabbate
darrylabbate committed Sep 27, 2024
1 parent 7adbcf5 commit 5d2b9f6
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 0 deletions.
4 changes: 4 additions & 0 deletions prov/efa/src/rdm/efa_rdm_ep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,13 @@ struct efa_rdm_ep {
size_t rx_iov_limit;
size_t tx_iov_limit;
size_t max_msg_size; /**< #FI_OPT_MAX_MSG_SIZE */
size_t max_tagged_size; /**< #FI_OPT_MAX_TAGGED_SIZE */
size_t max_rma_size; /**< #FI_OPT_MAX_RMA_SIZE */
size_t max_atomic_size; /**< #FI_OPT_MAX_ATOMIC_SIZE */
size_t inject_msg_size; /**< #FI_OPT_INJECT_MSG_SIZE */
size_t inject_tagged_size; /**< #FI_OPT_INJECT_TAGGED_SIZE */
size_t inject_rma_size; /**< #FI_OPT_INJECT_RMA_SIZE */
size_t inject_atomic_size; /**< #FI_OPT_INJECT_ATOMIC_SIZE */

/* Endpoint's capability to support zero-copy rx */
bool use_zcpy_rx;
Expand Down
40 changes: 40 additions & 0 deletions prov/efa/src/rdm/efa_rdm_ep_fiops.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,9 +562,13 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
efa_rdm_ep->rx_iov_limit = info->rx_attr->iov_limit;
efa_rdm_ep->tx_iov_limit = info->tx_attr->iov_limit;
efa_rdm_ep->max_msg_size = info->ep_attr->max_msg_size;
efa_rdm_ep->max_tagged_size = info->ep_attr->max_msg_size;
efa_rdm_ep->max_rma_size = info->ep_attr->max_msg_size;
efa_rdm_ep->max_atomic_size = info->ep_attr->max_msg_size;
efa_rdm_ep->inject_msg_size = info->tx_attr->inject_size;
efa_rdm_ep->inject_tagged_size = info->tx_attr->inject_size;
efa_rdm_ep->inject_rma_size = info->tx_attr->inject_size;
efa_rdm_ep->inject_atomic_size = info->tx_attr->inject_size;
efa_rdm_ep->efa_max_outstanding_tx_ops = efa_domain->device->rdm_info->tx_attr->size;
efa_rdm_ep->efa_max_outstanding_rx_ops = efa_domain->device->rdm_info->rx_attr->size;
efa_rdm_ep->use_device_rdma = efa_rdm_get_use_device_rdma(info->fabric_attr->api_version);
Expand Down Expand Up @@ -1666,15 +1670,27 @@ static int efa_rdm_ep_setopt(fid_t fid, int level, int optname,
case FI_OPT_MAX_MSG_SIZE:
EFA_RDM_EP_SETOPT_THRESHOLD(MAX_MSG_SIZE, efa_rdm_ep->max_msg_size, efa_rdm_ep->base_ep.info->ep_attr->max_msg_size)
break;
case FI_OPT_MAX_TAGGED_SIZE:
EFA_RDM_EP_SETOPT_THRESHOLD(MAX_TAGGED_SIZE, efa_rdm_ep->max_tagged_size, efa_rdm_ep->base_ep.info->ep_attr->max_msg_size)
break;
case FI_OPT_MAX_RMA_SIZE:
EFA_RDM_EP_SETOPT_THRESHOLD(MAX_RMA_SIZE, efa_rdm_ep->max_rma_size, efa_rdm_ep->base_ep.info->ep_attr->max_msg_size)
break;
case FI_OPT_MAX_ATOMIC_SIZE:
EFA_RDM_EP_SETOPT_THRESHOLD(MAX_ATOMIC_SIZE, efa_rdm_ep->max_atomic_size, efa_rdm_ep->base_ep.info->ep_attr->max_msg_size)
break;
case FI_OPT_INJECT_MSG_SIZE:
EFA_RDM_EP_SETOPT_THRESHOLD(INJECT_MSG_SIZE, efa_rdm_ep->inject_msg_size, efa_rdm_ep->base_ep.info->tx_attr->inject_size)
break;
case FI_OPT_INJECT_TAGGED_SIZE:
EFA_RDM_EP_SETOPT_THRESHOLD(INJECT_TAGGED_SIZE, efa_rdm_ep->inject_tagged_size, efa_rdm_ep->base_ep.info->tx_attr->inject_size)
break;
case FI_OPT_INJECT_RMA_SIZE:
EFA_RDM_EP_SETOPT_THRESHOLD(INJECT_RMA_SIZE, efa_rdm_ep->inject_rma_size, efa_rdm_ep->base_ep.info->tx_attr->inject_size)
break;
case FI_OPT_INJECT_ATOMIC_SIZE:
EFA_RDM_EP_SETOPT_THRESHOLD(INJECT_ATOMIC_SIZE, efa_rdm_ep->inject_atomic_size, efa_rdm_ep->base_ep.info->tx_attr->inject_size)
break;
case FI_OPT_EFA_USE_DEVICE_RDMA:
if (optlen != sizeof(bool))
return -FI_EINVAL;
Expand Down Expand Up @@ -1757,24 +1773,48 @@ static int efa_rdm_ep_getopt(fid_t fid, int level, int optname, void *optval,
*(size_t *) optval = efa_rdm_ep->max_msg_size;
*optlen = sizeof (size_t);
break;
case FI_OPT_MAX_TAGGED_SIZE:
if (*optlen < sizeof (size_t))
return -FI_ETOOSMALL;
*(size_t *) optval = efa_rdm_ep->max_tagged_size;
*optlen = sizeof (size_t);
break;
case FI_OPT_MAX_RMA_SIZE:
if (*optlen < sizeof (size_t))
return -FI_ETOOSMALL;
*(size_t *) optval = efa_rdm_ep->max_rma_size;
*optlen = sizeof (size_t);
break;
case FI_OPT_MAX_ATOMIC_SIZE:
if (*optlen < sizeof (size_t))
return -FI_ETOOSMALL;
*(size_t *) optval = efa_rdm_ep->max_atomic_size;
*optlen = sizeof (size_t);
break;
case FI_OPT_INJECT_MSG_SIZE:
if (*optlen < sizeof (size_t))
return -FI_ETOOSMALL;
*(size_t *) optval = efa_rdm_ep->inject_msg_size;
*optlen = sizeof (size_t);
break;
case FI_OPT_INJECT_TAGGED_SIZE:
if (*optlen < sizeof (size_t))
return -FI_ETOOSMALL;
*(size_t *) optval = efa_rdm_ep->inject_tagged_size;
*optlen = sizeof (size_t);
break;
case FI_OPT_INJECT_RMA_SIZE:
if (*optlen < sizeof (size_t))
return -FI_ETOOSMALL;
*(size_t *) optval = efa_rdm_ep->inject_rma_size;
*optlen = sizeof (size_t);
break;
case FI_OPT_INJECT_ATOMIC_SIZE:
if (*optlen < sizeof (size_t))
return -FI_ETOOSMALL;
*(size_t *) optval = efa_rdm_ep->inject_atomic_size;
*optlen = sizeof (size_t);
break;
case FI_OPT_EFA_EMULATED_READ:
if (*optlen < sizeof(bool))
return -FI_ETOOSMALL;
Expand Down
2 changes: 2 additions & 0 deletions prov/efa/src/rdm/efa_rdm_msg.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,7 @@ ssize_t efa_rdm_msg_tinject(struct fid_ep *ep_fid, const void *buf, size_t len,
struct efa_rdm_peer *peer;

efa_rdm_ep = container_of(ep_fid, struct efa_rdm_ep, base_ep.util_ep.ep_fid.fid);
assert(len <= efa_rdm_ep->inject_tagged_size);

peer = efa_rdm_ep_get_peer(efa_rdm_ep, dest_addr);
assert(peer);
Expand All @@ -585,6 +586,7 @@ ssize_t efa_rdm_msg_tinjectdata(struct fid_ep *ep_fid, const void *buf, size_t l
struct efa_rdm_peer *peer;

efa_rdm_ep = container_of(ep_fid, struct efa_rdm_ep, base_ep.util_ep.ep_fid.fid);
assert(len <= efa_rdm_ep->inject_tagged_size);

peer = efa_rdm_ep_get_peer(efa_rdm_ep, dest_addr);
assert(peer);
Expand Down

0 comments on commit 5d2b9f6

Please sign in to comment.