DAOS-11516 cart: relax progress and HG thread safety using thread mode

src/cart/README.env

@@ -202,6 +202,10 @@ This file lists the environment variables used in CaRT.
    start copying data in an effort to release multi-recv buffers. Copy will occur when at
    most D_MRECV_BUF_COPY buffers remain.
 
+   D_THREAD_MODE_SINGLE
+   Relax thread safety when user can guarantee that only a single thread will access
+   CART context at the same time. Setting this variable may improve performance.
+
    SWIM_TRAFFIC_CLASS
    (server only) Select a traffic class for the SWIM protocol to use and prevent potential
    traffic congestion. Available options are: "unspec" (default), "best_effort",

src/cart/crt_hg.c

@@ -825,7 +825,8 @@ crt_sep_hg_class_set(crt_provider_t provider, hg_class_t *class)
 }
 
 static int
-crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_idx, hg_class_t **ret_hg_class)
+crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_idx,
+		  bool thread_mode_single, hg_class_t **ret_hg_class)
 {
 	char			*info_string = NULL;
 	struct hg_init_info	init_info = HG_INIT_INFO_INITIALIZER;
@@ -866,6 +867,8 @@ crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_
 	/* Separate SWIM traffic in an effort to prevent potential congestion. */
 	if (crt_is_service() && ctx_idx == crt_gdata.cg_swim_ctx_idx)
 		init_info.traffic_class = (enum na_traffic_class)crt_gdata.cg_swim_tc;
+	if (thread_mode_single)
+		init_info.na_init_info.thread_mode = NA_THREAD_MODE_SINGLE;
 
 	hg_class = HG_Init_opt2(info_string, crt_is_service(), HG_VERSION(2, 4), &init_info);
 	if (hg_class == NULL) {
@@ -917,13 +920,16 @@ crt_hg_ctx_init(struct crt_hg_context *hg_ctx, crt_provider_t provider, int idx,
 	crt_ctx = container_of(hg_ctx, struct crt_context, cc_hg_ctx);
 
 	hg_ctx->chc_provider = provider;
-	sep_mode = crt_provider_is_sep(true, provider);
+	sep_mode             = crt_provider_is_sep(true, provider);
+	if ((!sep_mode && crt_is_service()) || crt_gdata.cg_thread_mode_single)
+		hg_ctx->chc_thread_mode_single = true;
 
 	/* In SEP mode all contexts share same hg_class*/
 	if (sep_mode) {
 		/* Only initialize class for context0 */
 		if (idx == 0) {
-			rc = crt_hg_class_init(provider, idx, primary, iface_idx, &hg_class);
+			rc = crt_hg_class_init(provider, idx, primary, iface_idx,
+					       hg_ctx->chc_thread_mode_single, &hg_class);
 			if (rc != 0)
 				D_GOTO(out, rc);
 
@@ -932,7 +938,8 @@ crt_hg_ctx_init(struct crt_hg_context *hg_ctx, crt_provider_t provider, int idx,
 			hg_class = crt_sep_hg_class_get(provider);
 		}
 	} else {
-		rc = crt_hg_class_init(provider, idx, primary, iface_idx, &hg_class);
+		rc = crt_hg_class_init(provider, idx, primary, iface_idx,
+				       hg_ctx->chc_thread_mode_single, &hg_class);
 		if (rc != 0)
 			D_GOTO(out, rc);
 	}

src/cart/crt_hg.h

@@ -114,13 +114,14 @@ struct crt_hg_pool {
 /** HG context */
 struct crt_hg_context {
 	/* Flag indicating whether hg class is shared; true for SEP mode */
-	bool			 chc_shared_hg_class;
-	hg_class_t		*chc_hgcla; /* HG class */
-	hg_context_t		*chc_hgctx; /* HG context */
-	hg_class_t		*chc_bulkcla; /* bulk class */
-	hg_context_t		*chc_bulkctx; /* bulk context */
-	struct crt_hg_pool	 chc_hg_pool; /* HG handle pool */
-	int			 chc_provider; /* provider */
+	bool               chc_shared_hg_class;
+	hg_class_t        *chc_hgcla;              /* HG class */
+	hg_context_t      *chc_hgctx;              /* HG context */
+	hg_class_t        *chc_bulkcla;            /* bulk class */
+	hg_context_t      *chc_bulkctx;            /* bulk context */
+	struct crt_hg_pool chc_hg_pool;            /* HG handle pool */
+	int                chc_provider;           /* provider */
+	bool               chc_thread_mode_single; /* thread safety */
 };
 
 /* crt_hg.c */

src/cart/crt_init.c

@@ -91,6 +91,8 @@ dump_opt(crt_init_options_t *opt)
 	/* Handle similar to D_PROVIDER_AUTH_KEY */
 	if (opt->cio_auth_key)
 		D_INFO("auth_key is set\n");
+	if (opt->cio_thread_mode_single)
+		D_INFO("thread mode single is set\n");
 }
 
 static int
@@ -704,6 +706,14 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt)
 				D_DEBUG(DB_ALL, "set group_config_path as %s.\n", path);
 		}
 
+		if (opt && opt->cio_thread_mode_single) {
+			crt_gdata.cg_thread_mode_single = opt->cio_thread_mode_single;
+		} else {
+			bool thread_mode_single = false;
+			crt_env_get(D_THREAD_MODE_SINGLE, &thread_mode_single);
+			crt_gdata.cg_thread_mode_single = thread_mode_single;
+		}
+
 		if (opt && opt->cio_auth_key)
 			auth_key = opt->cio_auth_key;
 		else {

src/cart/crt_internal_types.h

@@ -147,6 +147,9 @@ struct crt_gdata {
 	/** whether we are on a primary provider */
 	unsigned int             cg_provider_is_primary : 1;
 
+	/** use single thread to access context */
+	bool                     cg_thread_mode_single;
+
 	ATOMIC uint64_t		cg_rpcid; /* rpc id */
 
 	/* protects crt_gdata (see the lock order comment on crp_mutex) */
@@ -222,6 +225,7 @@ struct crt_event_cb_priv {
 	ENV(D_POLL_TIMEOUT)                                                                        \
 	ENV_STR(D_PORT)                                                                            \
 	ENV(D_PORT_AUTO_ADJUST)                                                                    \
+	ENV(D_THREAD_MODE_SINGLE)                                                                  \
 	ENV(D_POST_INCR)                                                                           \
 	ENV(D_POST_INIT)                                                                           \
 	ENV(D_MRECV_BUF)                                                                           \

src/include/cart/types.h

@@ -90,9 +90,11 @@ typedef struct crt_init_options {
 	char		*cio_port;
 
 	/** If set, used as the authentication key instead of D_PROVIDER_AUTH_KEY env */
-	char		*cio_auth_key;
-} crt_init_options_t;
+	char            *cio_auth_key;
 
+	/** use single thread to access context */
+	bool             cio_thread_mode_single;
+} crt_init_options_t;
 
 typedef int		crt_status_t;
 /**