Backward compatibility for old drivers

New configuration parameter (max_rate_enum) allows to manually limit the
rate of subnet when using old drivers which support up to a limited rate.
The value it takes is Rate Enumeration, defined at IB specification chapter
15.2.5.16.1.

Previously, a boolean parameter (use_original_extended_sa_rates_only) was
used to indicate rate limit of up to 12xEDR. However it cannot be used to
indicate of other rate limitations, thus max_rate_enum is required.

In case use_original_extended_sa_rates_only is TRUE (equivalent to limited
rate enumeration of 18), SA uses the minimum between 18 (up to 12xEDR) and
max_rate_enum as the rate limitation.

Signed-off-by: Or Nechemia <orn@nvidia.com>

include/opensm/osm_helper.h

@@ -895,6 +895,34 @@ int ib_path_rate_2x_hdr_fixups(IN const ib_port_info_t * p_pi,
 * SEE ALSO
 *********/
 
+/****f* IBA Base: Types/ib_path_get_reduced_rate
+* NAME
+*	ib_path_get_reduced_rate
+*
+* DESCRIPTION
+*	Obtains encoded rate for a reduced rate, subsequent
+*	to input maximal rate.
+*
+* SYNOPSIS
+*/
+int ib_path_get_reduced_rate(IN const uint8_t rate, IN const uint8_t limit);
+/*
+* PARAMETERS
+*	rate
+*		[in] Encoded path rate.
+*
+*	limit
+*		[in] Encoded maximal rate supported.
+*
+* RETURN VALUES
+*	Returns an int indicating reduced encoded rate supported,
+*	or minimal rate if none can be found.
+*
+* NOTES
+*
+* SEE ALSO
+*********/
+
 /****f* OpenSM: Helper/sprint_uint8_arr
 * NAME
 *	sprint_uint8_arr

include/opensm/osm_sa.h

@@ -66,6 +66,8 @@
 #  define END_C_DECLS
 #endif				/* __cplusplus */
 
+#define SA_RATE_MAX_ENUM 63
+
 BEGIN_C_DECLS
 /****h* OpenSM/SA
 * NAME
@@ -613,5 +615,31 @@ void osm_pr_process_half(IN osm_sa_t * sa, IN const ib_sa_mad_t * sa_mad,
 				IN const ib_gid_t * p_dgid,
 				IN cl_qlist_t * p_list);
 
+/****f* OpenSM: SA/osm_sa_limit_rate
+* NAME
+*	osm_sa_limit_rate
+*
+* DESCRIPTION
+*	Find reduced rate of input rate that does not exceed the maximal
+*	rate value of subnet.
+*
+* SYNOPSIS
+*/
+uint8_t osm_sa_limit_rate(IN osm_sa_t * sa, IN const uint8_t rate);
+/*
+* PARAMETERS
+*	sa
+*		[in] Pointer to a SA object.
+*
+*	rate
+*		[in] Rate to be adjusted to maximal rate value of subnet.
+*
+* RETURN VALUE
+*	The rate after adjusting to maximal rate, may be the same or lower.
+*
+* SEE ALSO
+*	SA object, osm_sa_construct, osm_sa_init
+*********/
+
 END_C_DECLS
 #endif				/* _OSM_SA_H_ */

include/opensm/osm_subnet.h

@@ -338,6 +338,7 @@ typedef struct osm_subn_opt {
 	boolean_t ipoib_mcgroup_creation_validation;
 	boolean_t mcgroup_join_validation;
 	boolean_t use_original_extended_sa_rates_only;
+	uint8_t max_rate_enum;
 	boolean_t use_optimized_slvl;
 	boolean_t fsync_high_avail_files;
 	osm_qos_options_t qos_options;
@@ -641,6 +642,12 @@ typedef struct osm_subn_opt {
 *		old kernels/drivers that don't understand the
 *		new SA rates for 2x link width and/or HDR link speed (19-22).
 *
+*	max_rate_enum
+*		Enumeration of the maximal rate subnet supports. Option is
+*		required for subnets with old kernels/drivers that don't
+*		understand new SA rates.
+*		See also: use_original_extended_sa_rates_only.
+*
 *	use_optimized_slvl
 *		Use optimized SLtoVLMappingTable programming if
 *		device indicates it supports this.

libopensm/libopensm.map

@@ -91,5 +91,6 @@ OPENSM_1.5 {
 		sprint_uint8_arr;
 		ib_path_rate_max_12xedr;
 		ib_path_rate_2x_hdr_fixups;
+		ib_path_get_reduced_rate;
 	local: *;
 };

libopensm/osm_helper.c

@@ -3350,6 +3350,17 @@ int ib_path_rate_get_next(IN const int rate)
 	return find_ordered_rate(orate);
 }
 
+int ib_path_get_reduced_rate(IN const uint8_t rate, IN const uint8_t limit)
+{
+	int i = ib_path_rate_get_prev(rate);
+
+	while (i > IB_MIN_RATE &&
+	       (ordered_rates[i] > ordered_rates[limit] || i > limit))
+		i = ib_path_rate_get_prev(i);
+
+	return i ? i : IB_MIN_RATE;
+}
+
 int ib_path_rate_max_12xedr(IN const int rate)
 {
 	CL_ASSERT(rate >= IB_MIN_RATE && rate <= IB_MAX_RATE);

opensm/osm_sa.c

@@ -1375,3 +1375,11 @@ int osm_sa_db_file_load(osm_opensm_t * p_osm)
 	fclose(file);
 	return ret;
 }
+
+uint8_t osm_sa_limit_rate(IN osm_sa_t * sa, IN const uint8_t rate)
+{
+	if (sa->p_subn->opt.max_rate_enum < rate)
+		return ib_path_get_reduced_rate(rate, sa->p_subn->opt.max_rate_enum);
+
+	return rate;
+}

opensm/osm_sa_mcmember_record.c

@@ -775,8 +775,8 @@ static boolean_t mgrp_request_is_realizable(IN osm_sa_t * sa,
 			return FALSE;
 		}
 	}
-	if (sa->p_subn->opt.use_original_extended_sa_rates_only) {
-		new_rate = ib_path_rate_max_12xedr(rate);
+	if (sa->p_subn->opt.max_rate_enum < SA_RATE_MAX_ENUM) {
+		new_rate = osm_sa_limit_rate(sa, rate);
 		if (new_rate != rate) {
 			OSM_LOG(sa->p_log, OSM_LOG_VERBOSE,
 				"Rate decreased from %u to %u\n",

opensm/osm_sa_multipath_record.c

@@ -853,8 +853,8 @@ static void mpr_rcv_build_pr(IN osm_sa_t * sa,
 	ib_path_rec_set_sl(p_pr, p_parms->sl);
 	p_pr->mtu = (uint8_t) (p_parms->mtu | 0x80);
 	rate = p_parms->rate;
-	if (sa->p_subn->opt.use_original_extended_sa_rates_only) {
-		new_rate = ib_path_rate_max_12xedr(rate);
+	if (sa->p_subn->opt.max_rate_enum < SA_RATE_MAX_ENUM) {
+		new_rate = osm_sa_limit_rate(sa, rate);
 		if (new_rate != rate) {
 			OSM_LOG(sa->p_log, OSM_LOG_VERBOSE,
 				"Rate decreased from %u to %u\n",

opensm/osm_sa_path_record.c

@@ -921,8 +921,8 @@ static void pr_rcv_build_pr(IN osm_sa_t * sa,
 	ib_path_rec_set_qos_class(p_pr, 0);
 	p_pr->mtu = (uint8_t) (p_parms->mtu | 0x80);
 	rate = p_parms->rate;
-	if (sa->p_subn->opt.use_original_extended_sa_rates_only) {
-		new_rate = ib_path_rate_max_12xedr(rate);
+	if (sa->p_subn->opt.max_rate_enum < SA_RATE_MAX_ENUM) {
+		new_rate = osm_sa_limit_rate(sa, rate);
 		if (new_rate != rate) {
 			OSM_LOG(sa->p_log, OSM_LOG_VERBOSE,
 				"Rate decreased from %u to %u\n",

opensm/osm_subnet.c

@@ -851,6 +851,7 @@ static const opt_rec_t opt_tbl[] = {
 	{ "ipoib_mcgroup_creation_validation", OPT_OFFSET(ipoib_mcgroup_creation_validation), opts_parse_boolean, NULL, 1 },
 	{ "mcgroup_join_validation", OPT_OFFSET(mcgroup_join_validation), opts_parse_boolean, NULL, 1 },
 	{ "use_original_extended_sa_rates_only", OPT_OFFSET(use_original_extended_sa_rates_only), opts_parse_boolean, NULL, 1 },
+	{ "max_rate_enum", OPT_OFFSET(max_rate_enum), opts_parse_uint8, NULL, 1},
 	{ "use_optimized_slvl", OPT_OFFSET(use_optimized_slvl), opts_parse_boolean, NULL, 1 },
 	{ "fsync_high_avail_files", OPT_OFFSET(fsync_high_avail_files), opts_parse_boolean, NULL, 1 },
 #ifdef ENABLE_OSM_PERF_MGR
@@ -1686,6 +1687,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * p_opt)
 	p_opt->cc_cct.entries_len = 0;
 	p_opt->cc_cct.input_str = NULL;
 	p_opt->quasi_ftree_indexing = FALSE;
+	p_opt->max_rate_enum = SA_RATE_MAX_ENUM;
 }
 
 static char *clean_val(char *val)
@@ -2201,6 +2203,28 @@ int osm_subn_verify_config(IN osm_subn_opt_t * p_opts)
 		}
 	}
 
+	if (p_opts->use_original_extended_sa_rates_only &&
+	    p_opts->max_rate_enum > IB_PATH_RECORD_RATE_300_GBS) {
+		if (p_opts->max_rate_enum != SA_RATE_MAX_ENUM)
+			log_report(" Warning:"
+				   " use_original_extended_sa_rates_only"
+				   " defines a rate limit lower"
+				   " than max_rate_enum.\n\t "
+				   " Setting max_rate_enum to %d\n",
+				   IB_PATH_RECORD_RATE_300_GBS);
+
+		p_opts->max_rate_enum = IB_PATH_RECORD_RATE_300_GBS;
+	}
+
+	if (p_opts->max_rate_enum < IB_MIN_RATE ||
+	    (p_opts->max_rate_enum > IB_MAX_RATE &&
+	     p_opts->max_rate_enum != SA_RATE_MAX_ENUM)) {
+		log_report("Illegal max_rate_enum %u, setting to default "
+			   "value (%u), support all rates\n",
+			   p_opts->max_rate_enum, SA_RATE_MAX_ENUM);
+		p_opts->max_rate_enum = SA_RATE_MAX_ENUM;
+	}
+
 	return 0;
 }
 
@@ -2795,7 +2819,16 @@ void osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
 		"# Set to TRUE for subnets with old kernels/drivers that don't understand\n"
 		"# the new SA rates for 2x link width and/or HDR link speed (19-22)\n"
 		"# default is FALSE\n"
+		"# Notice: use_original_extended_sa_rates_only is deprecated by max_rate_enum!\n"
+		"# Use max_rate_enum 18 instead.\n"
 		"use_original_extended_sa_rates_only %s\n\n"
+		"# Enumeration of the maximal rate subnet supports. Option is needed for\n"
+		"# subnets with old kernels/drivers that don't understand new SA rates.\n"
+		"# For example:"
+		"# 18: Rate of 300Gbps - support speeds up to 12xEDR\n"
+		"# 22: Rate of 600Gbps - support speeds up to HDR\n"
+		"# For farther reference refer to IB specification chapter 15.2.5.16.1\n"
+		"max_rate_enum %u\n\n"
 		"# Use Optimized SLtoVLMapping programming if supported by device\n"
 		"use_optimized_slvl %s\n\n"
 		"# Sync in memory files used for high availability with storage\n"
@@ -2807,6 +2840,7 @@ void osm_subn_output_conf(FILE *out, IN osm_subn_opt_t * p_opts)
 		p_opts->ipoib_mcgroup_creation_validation ? "TRUE" : "FALSE",
 		p_opts->mcgroup_join_validation ? "TRUE" : "FALSE",
 		p_opts->use_original_extended_sa_rates_only ? "TRUE" : "FALSE",
+		p_opts->max_rate_enum,
 		p_opts->use_optimized_slvl ? "TRUE" : "FALSE",
 		p_opts->fsync_high_avail_files ? "TRUE" : "FALSE");