prov/efa: Raise error when receiver gets rtm pkt while zcpy recv is on

When zero copy recv is turned on, the ep cannot
handle rtm pkts delivered to the internal bounce buffer,
because the user recv buffer has been posted to the other
QP and we cannot cancel that.

Signed-off-by: Shi Jin <sjina@amazon.com>

prov/efa/src/rdm/efa_rdm_cq.c

@@ -354,12 +354,19 @@ static void efa_rdm_cq_handle_recv_completion(struct efa_ibv_cq *ibv_cq, struct
 	/* Proc receives with pkt hdrs (posted to ctrl QPs)*/
 	base_hdr = efa_rdm_pke_get_base_hdr(pkt_entry);
 	pkt_type = base_hdr->type;
-	if (pkt_type >= EFA_RDM_EXTRA_REQ_PKT_END) {
+	/**
+	 * When zero copy recv is turned on, the ep cannot
+	 * handle rtm pkts delivered to the internal bounce buffer,
+	 * because the user recv buffer has been posted to the other
+	 * QP and we cannot cancel that.
+	 */
+	if ((pkt_type >= EFA_RDM_EXTRA_REQ_PKT_END) ||
+		(ep->use_zcpy_rx && efa_rdm_pkt_type_is_rtm(pkt_type))) {
 		EFA_WARN(FI_LOG_CQ,
 			"Peer %d is requesting feature %d, which this EP does not support.\n",
 			(int)pkt_entry->addr, base_hdr->type);
 
-		assert(0 && "invalid REQ packet type");
+		assert(0 && "invalid packet type");
 		efa_base_ep_write_eq_error(&ep->base_ep, FI_EIO, FI_EFA_ERR_INVALID_PKT_TYPE);
 		efa_rdm_pke_release_rx(pkt_entry);
 		return;

prov/efa/src/rdm/efa_rdm_pkt_type.h

@@ -121,6 +121,38 @@ bool efa_rdm_pkt_type_is_longcts_req(int pkt_type)
 	}
 }
 
+/**
+ * @brief determine whether a req pkt type is RTM
+ *
+ * @param[in]		pkt_type		REQ packet type
+ * @return		a boolean
+ */
+static inline
+bool efa_rdm_pkt_type_is_rtm(int pkt_type)
+{
+	switch(pkt_type) {
+	case EFA_RDM_EAGER_MSGRTM_PKT:
+	case EFA_RDM_EAGER_TAGRTM_PKT:
+	case EFA_RDM_DC_EAGER_MSGRTM_PKT:
+	case EFA_RDM_DC_EAGER_TAGRTM_PKT:
+	case EFA_RDM_MEDIUM_MSGRTM_PKT:
+	case EFA_RDM_MEDIUM_TAGRTM_PKT:
+	case EFA_RDM_DC_MEDIUM_MSGRTM_PKT:
+	case EFA_RDM_DC_MEDIUM_TAGRTM_PKT:
+	case EFA_RDM_LONGCTS_MSGRTM_PKT:
+	case EFA_RDM_LONGCTS_TAGRTM_PKT:
+	case EFA_RDM_DC_LONGCTS_MSGRTM_PKT:
+	case EFA_RDM_DC_LONGCTS_TAGRTM_PKT:
+	case EFA_RDM_LONGREAD_MSGRTM_PKT:
+	case EFA_RDM_LONGREAD_TAGRTM_PKT:
+	case EFA_RDM_RUNTREAD_MSGRTM_PKT:
+	case EFA_RDM_RUNTREAD_TAGRTM_PKT:
+		return 1;
+	default:
+		return 0;
+	}
+}
+
 /**
  * @brief determine whether a req pkt type is RTA
  *