Merge pull request #25 from ornechemia/ndr_support

NDR support and backward compatibility rate support

include/iba/ib_types.h

@@ -3216,6 +3216,8 @@ ib_path_rec_rate(IN const ib_path_rec_t * const p_rec)
 *		20: 50 Gb/sec.
 *		21: 400 Gb/sec.
 *		22: 600 Gb/sec.
+*		23 : 800 Gb/sec.
+*		24 : 1200 Gb/sec.
 *		others: reserved
 *
 * NOTES
@@ -4702,6 +4704,7 @@ typedef struct _ib_port_info {
 #define IB_PORT_CAP2_IS_SWITCH_PORT_STATE_TBL_SUPP (CL_HTON16(0x0008))
 #define IB_PORT_CAP2_IS_LINK_WIDTH_2X_SUPPORTED (CL_HTON16(0x0010))
 #define IB_PORT_CAP2_IS_LINK_SPEED_HDR_SUPPORTED (CL_HTON16(0x0020))
+#define IB_PORT_CAP2_IS_LINK_SPEED_NDR_SUPPORTED (CL_HTON16(0x0400))
 
 /****s* IBA Base: Types/ib_port_info_ext_t
 * NAME
@@ -4721,7 +4724,9 @@ typedef struct _ib_port_info_ext {
 	ib_net16_t edr_fec_mode_enable;
 	ib_net16_t hdr_fec_mode_sup;
 	ib_net16_t hdr_fec_mode_enable;
-	uint8_t reserved[46];
+	ib_net16_t ndr_fec_mode_sup;
+	ib_net16_t ndr_fec_mode_enable;
+	uint8_t reserved[42];
 } ib_port_info_ext_t;
 /************/
 
@@ -5182,9 +5187,14 @@ ib_port_info_get_link_speed_active(IN const ib_port_info_t * const p_pi)
 #define IB_LINK_SPEED_EXT_ACTIVE_14		1
 #define IB_LINK_SPEED_EXT_ACTIVE_25		2
 #define IB_LINK_SPEED_EXT_ACTIVE_50		4
+#define IB_LINK_SPEED_EXT_ACTIVE_100		8
 #define IB_LINK_SPEED_EXT_14_25_OR_50		(IB_LINK_SPEED_EXT_ACTIVE_14 | \
 						 IB_LINK_SPEED_EXT_ACTIVE_25 | \
 						 IB_LINK_SPEED_EXT_ACTIVE_50)
+#define IB_LINK_SPEED_EXT_MAX_VALUE		(IB_LINK_SPEED_EXT_ACTIVE_14 | \
+						 IB_LINK_SPEED_EXT_ACTIVE_25 | \
+						 IB_LINK_SPEED_EXT_ACTIVE_50 | \
+						 IB_LINK_SPEED_EXT_ACTIVE_100)
 #define IB_LINK_SPEED_EXT_DISABLE		30
 #define IB_LINK_SPEED_EXT_SET_LSES		31
 
@@ -5210,10 +5220,13 @@ ib_port_info_get_link_speed_active(IN const ib_port_info_t * const p_pi)
 #define IB_PATH_RECORD_RATE_50_GBS		20
 #define IB_PATH_RECORD_RATE_400_GBS		21
 #define IB_PATH_RECORD_RATE_600_GBS		22
+/* following v1 ver1.5 p1044 */
+#define IB_PATH_RECORD_RATE_800_GBS		23
+#define IB_PATH_RECORD_RATE_1200_GBS		24
 
 #define IB_MIN_RATE    IB_PATH_RECORD_RATE_2_5_GBS
-#define IB_MAX_RATE    IB_PATH_RECORD_RATE_600_GBS
-#define IB_RATE_MAX    IB_PATH_RECORD_RATE_600_GBS
+#define IB_MAX_RATE    IB_PATH_RECORD_RATE_1200_GBS
+#define IB_RATE_MAX    IB_PATH_RECORD_RATE_1200_GBS
 
 static inline uint8_t OSM_API
 ib_port_info_get_link_speed_ext_active(IN const ib_port_info_t * const p_pi);
@@ -5316,6 +5329,33 @@ ib_port_info_compute_rate(IN const ib_port_info_t * const p_pi,
 				break;
 			}
 			break;
+		case IB_LINK_SPEED_EXT_ACTIVE_100:
+			switch (p_pi->link_width_active) {
+			case IB_LINK_WIDTH_ACTIVE_1X:
+				rate = IB_PATH_RECORD_RATE_100_GBS;
+				break;
+
+			case IB_LINK_WIDTH_ACTIVE_4X:
+				rate = IB_PATH_RECORD_RATE_400_GBS;
+				break;
+
+			case IB_LINK_WIDTH_ACTIVE_8X:
+				rate = IB_PATH_RECORD_RATE_800_GBS;
+				break;
+
+			case IB_LINK_WIDTH_ACTIVE_12X:
+				rate = IB_PATH_RECORD_RATE_1200_GBS;
+				break;
+
+			case IB_LINK_WIDTH_ACTIVE_2X:
+				rate = IB_PATH_RECORD_RATE_200_GBS;
+				break;
+
+			default:
+				rate = IB_PATH_RECORD_RATE_100_GBS;
+				break;
+			}
+			break;
 		/* IB_LINK_SPEED_EXT_ACTIVE_NONE and any others */
 		default:
 			break;
@@ -6456,7 +6496,9 @@ static inline uint8_t OSM_API ib_get_highest_link_speed(IN const uint8_t speeds)
 	uint8_t ret = 0;
 	uint8_t extspeeds = (speeds >> 4);
 
-	if (extspeeds & IB_LINK_SPEED_EXT_ACTIVE_50)
+	if (extspeeds & IB_LINK_SPEED_EXT_ACTIVE_100)
+		ret = IB_LINK_SPEED_EXT_ACTIVE_100 << 4;
+	else if (extspeeds & IB_LINK_SPEED_EXT_ACTIVE_50)
 		ret = IB_LINK_SPEED_EXT_ACTIVE_50 << 4;
 	else if (extspeeds & IB_LINK_SPEED_EXT_ACTIVE_25)
 		ret = IB_LINK_SPEED_EXT_ACTIVE_25 << 4;

include/opensm/osm_base.h

@@ -933,6 +933,17 @@ typedef enum _osm_thread_state {
  */
 #define OSM_CAP2_IS_LINK_SPEED_HDR_SUPPORTED (1 << 15)
 
+/****d* OpenSM: OSM_CAP2_IS_LINK_SPEED_NDR_SUPPORTED
+ * Name
+ *	OSM_CAP2_IS_LINK_SPEED_NDR_SUPPORTED
+ *
+ * DESCRIPTION
+ *	NDR link speed supported
+ *
+ * SYNOPSIS
+ */
+#define OSM_CAP2_IS_LINK_SPEED_NDR_SUPPORTED (1 << 17)
+
 /****d* OpenSM: osm_signal_t
 * NAME
 *	osm_signal_t

include/opensm/osm_helper.h

@@ -841,6 +841,9 @@ int ib_path_rate_get_next(IN const int rate);
 *	ib_path_rate_max_12xedr
 *
 * DESCRIPTION
+*	Function is deprecated and SHOULD NOT BE USED.
+*	Use osm_sa_limit_rate instead.
+*
 *	Obtains encoded rate from the set of "original" extended
 *	SA rates (up through and including 300 Gbps - 12x EDR).
 *
@@ -854,6 +857,9 @@ int ib_path_rate_max_12xedr(IN const int rate);
 *		[in] Encoded path rate.
 *
 * RETURN VALUES
+*	Function is deprecated and SHOULD NOT BE USED.
+*	Use osm_sa_limit_rate instead.
+*
 *	Returns an int indicating the encoded rate
 *	with a maximum of 300 Gbps (12x EDR).
 *	For new rates (relating to 2x and HDR), the
@@ -895,6 +901,34 @@ int ib_path_rate_2x_hdr_fixups(IN const ib_port_info_t * p_pi,
 * SEE ALSO
 *********/
 
+/****f* IBA Base: Types/ib_path_get_reduced_rate
+ * NAME
+ *	ib_path_get_reduced_rate
+ *
+ * DESCRIPTION
+ *	Obtains encoded rate for a reduced rate, subsequent
+ *	to input maximal rate.
+ *
+ * SYNOPSIS
+ */
+int ib_path_get_reduced_rate(IN const uint8_t rate, IN const uint8_t limit);
+/*
+ * PARAMETERS
+ *	rate
+ *		[in] Encoded path rate.
+ *
+ *	limit
+ *		[in] Encoded maximal rate supported.
+ *
+ * RETURN VALUES
+ *	Returns an int indicating reduced encoded rate supported,
+ *	or minimal rate if none can be found.
+ *
+ * NOTES
+ *
+ * SEE ALSO
+ *********/
+
 /****f* OpenSM: Helper/sprint_uint8_arr
 * NAME
 *	sprint_uint8_arr

include/opensm/osm_sa.h

@@ -66,6 +66,8 @@
 #  define END_C_DECLS
 #endif				/* __cplusplus */
 
+#define SA_RATE_MAX_ENUM 63
+
 BEGIN_C_DECLS
 /****h* OpenSM/SA
 * NAME
@@ -613,5 +615,31 @@ void osm_pr_process_half(IN osm_sa_t * sa, IN const ib_sa_mad_t * sa_mad,
 				IN const ib_gid_t * p_dgid,
 				IN cl_qlist_t * p_list);
 
+/****f* OpenSM: SA/osm_sa_limit_rate
+ * NAME
+ *	osm_sa_limit_rate
+ *
+ * DESCRIPTION
+ *	Find reduced rate of input rate that does not exceed the maximal
+ *	rate value of subnet.
+ *
+ * SYNOPSIS
+ */
+uint8_t osm_sa_limit_rate(IN osm_sa_t *sa, IN const uint8_t rate);
+/*
+ * PARAMETERS
+ *	sa
+ *		[in] Pointer to a SA object.
+ *
+ *	rate
+ *		[in] Rate to be adjusted to maximal rate value of subnet.
+ *
+ * RETURN VALUE
+ *	The rate after adjusting to maximal rate, may be the same or lower.
+ *
+ * SEE ALSO
+ *	SA object, osm_sa_construct, osm_sa_init
+ *********/
+
 END_C_DECLS
 #endif				/* _OSM_SA_H_ */

include/opensm/osm_subnet.h

@@ -338,6 +338,7 @@ typedef struct osm_subn_opt {
 	boolean_t ipoib_mcgroup_creation_validation;
 	boolean_t mcgroup_join_validation;
 	boolean_t use_original_extended_sa_rates_only;
+	uint8_t max_rate_enum;
 	boolean_t use_optimized_slvl;
 	boolean_t fsync_high_avail_files;
 	osm_qos_options_t qos_options;
@@ -641,6 +642,12 @@ typedef struct osm_subn_opt {
 *		old kernels/drivers that don't understand the
 *		new SA rates for 2x link width and/or HDR link speed (19-22).
 *
+*	max_rate_enum
+*		Enumeration of the maximal rate subnet supports. Option is
+*		required for subnets with old kernels/drivers that don't
+*		understand new SA rates.
+*		See also : use_original_extended_sa_rates_only.
+*
 *	use_optimized_slvl
 *		Use optimized SLtoVLMappingTable programming if
 *		device indicates it supports this.

libopensm/libopensm.map

@@ -91,5 +91,6 @@ OPENSM_1.5 {
 		sprint_uint8_arr;
 		ib_path_rate_max_12xedr;
 		ib_path_rate_2x_hdr_fixups;
+		ib_path_get_reduced_rate;
 	local: *;
 };

libopensm/osm_helper.c

@@ -464,6 +464,8 @@ static int ordered_rates[] = {
 	11,	/* 20 -  50 Gbps (62.5 Gbps equiv) */
 	20,	/* 21 - 400 Gbps (500 Gbps equiv) */
 	21,	/* 22 - 600 Gbps (750 Gbps equiv) */
+	22,	/* 23 - 800 Gbps (1000 Gbps equiv) */
+	23,	/* 24 - 1200 Gbps (1500 Gbps equiv) */
 };
 
 int sprint_uint8_arr(char *buf, size_t size,
@@ -3215,7 +3217,11 @@ static const char *lsea_str_fixed_width[] = {
 	"14  ",
 	"25  ",
 	"????",
-	"50  "
+	"50  ",
+	"????",
+	"????",
+	"????",
+	"100 "
 };
 
 const char *osm_get_lsa_str(IN uint8_t lsa, IN uint8_t lsea, IN uint8_t state,
@@ -3229,7 +3235,7 @@ const char *osm_get_lsa_str(IN uint8_t lsa, IN uint8_t lsea, IN uint8_t state,
 		else
 			return lsa_str_fixed_width[lsa];
 	}
-	if (lsea > IB_LINK_SPEED_EXT_ACTIVE_50)
+	if (lsea > IB_LINK_SPEED_EXT_ACTIVE_100)
 		return lsa_str_fixed_width[3];
 	return lsea_str_fixed_width[lsea];
 }
@@ -3344,6 +3350,17 @@ int ib_path_rate_get_next(IN const int rate)
 	return find_ordered_rate(orate);
 }
 
+int ib_path_get_reduced_rate(IN const uint8_t rate, IN const uint8_t limit)
+{
+	int i = ib_path_rate_get_prev(rate);
+
+	while (i > IB_MIN_RATE &&
+	       (ordered_rates[i] > ordered_rates[limit] || i > limit))
+		i = ib_path_rate_get_prev(i);
+
+	return i ? i : IB_MIN_RATE;
+}
+
 int ib_path_rate_max_12xedr(IN const int rate)
 {
 	CL_ASSERT(rate >= IB_MIN_RATE && rate <= IB_MAX_RATE);
@@ -3358,6 +3375,8 @@ int ib_path_rate_max_12xedr(IN const int rate)
 		return IB_PATH_RECORD_RATE_40_GBS;
 	case IB_PATH_RECORD_RATE_400_GBS:
 	case IB_PATH_RECORD_RATE_600_GBS:
+	case IB_PATH_RECORD_RATE_800_GBS:
+	case IB_PATH_RECORD_RATE_1200_GBS:
 		return IB_PATH_RECORD_RATE_300_GBS;
 	default:
 		break;
@@ -3397,10 +3416,28 @@ int ib_path_rate_2x_hdr_fixups(IN const ib_port_info_t * p_pi,
 	case IB_PATH_RECORD_RATE_600_GBS:
 		/* HDR not supported but HDR only rate */
 		if (!(p_pi->capability_mask & IB_PORT_CAP_HAS_CAP_MASK2) ||
-		    (p_pi->capability_mask & IB_PORT_CAP_HAS_CAP_MASK2 &&
-		    !(p_pi->capability_mask2 & IB_PORT_CAP2_IS_LINK_SPEED_HDR_SUPPORTED)))
+		    ((p_pi->capability_mask & IB_PORT_CAP_HAS_CAP_MASK2) &&
+		    !(p_pi->capability_mask2 &
+		    (IB_PORT_CAP2_IS_LINK_SPEED_HDR_SUPPORTED |
+		    IB_PORT_CAP2_IS_LINK_SPEED_NDR_SUPPORTED))))
 			new_rate = IB_PATH_RECORD_RATE_300_GBS;
 		break;
+	case IB_PATH_RECORD_RATE_800_GBS:
+	case IB_PATH_RECORD_RATE_1200_GBS:
+		/* NDR not supported but NDR only rate */
+		if (!(p_pi->capability_mask & IB_PORT_CAP_HAS_CAP_MASK2) ||
+		    (p_pi->capability_mask & IB_PORT_CAP_HAS_CAP_MASK2 &&
+		    !(p_pi->capability_mask2 & IB_PORT_CAP2_IS_LINK_SPEED_NDR_SUPPORTED))) {
+			if ((p_pi->capability_mask & IB_PORT_CAP_HAS_CAP_MASK2) &&
+			    !(p_pi->capability_mask2 & IB_PORT_CAP2_IS_LINK_SPEED_HDR_SUPPORTED)) {
+				/* HDR not supported */
+				new_rate = IB_PATH_RECORD_RATE_300_GBS;
+			} else {
+				/* HDR is supported */
+				new_rate = IB_PATH_RECORD_RATE_600_GBS;
+			}
+		}
+		break;
 	default:
 		break;
 	}

opensm/osm_console.c

@@ -730,6 +730,7 @@ typedef struct {
 	uint64_t ports_fdr;
 	uint64_t ports_edr;
 	uint64_t ports_hdr;
+	uint64_t ports_ndr;
 	uint64_t ports_unknown_speed;
 	port_report_t *unknown_speed_ports;
 	uint64_t ports_unenabled_speed;
@@ -996,6 +997,8 @@ static void portstatus_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
 		fprintf(out, "   %" PRIu64 " at 25.78125 Gbps\n", fs.ports_edr);
 	if (fs.ports_hdr)
 		fprintf(out, "   %" PRIu64 " at 53.125 Gbps\n", fs.ports_hdr);
+	if (fs.ports_ndr)
+		fprintf(out, "   %" PRIu64 " at 106.25 Gbps\n", fs.ports_ndr);
 
 	if (fs.ports_disabled + fs.ports_reduced_speed + fs.ports_reduced_width
 	    + fs.ports_unenabled_width + fs.ports_unenabled_speed

opensm/osm_sa.c

@@ -1375,3 +1375,11 @@ int osm_sa_db_file_load(osm_opensm_t * p_osm)
 	fclose(file);
 	return ret;
 }
+
+uint8_t osm_sa_limit_rate(IN osm_sa_t *sa, IN const uint8_t rate)
+{
+	if (sa->p_subn->opt.max_rate_enum < rate)
+		return ib_path_get_reduced_rate(rate, sa->p_subn->opt.max_rate_enum);
+
+	return rate;
+}

opensm/osm_sa_mcmember_record.c

@@ -775,8 +775,8 @@ static boolean_t mgrp_request_is_realizable(IN osm_sa_t * sa,
 			return FALSE;
 		}
 	}
-	if (sa->p_subn->opt.use_original_extended_sa_rates_only) {
-		new_rate = ib_path_rate_max_12xedr(rate);
+	if (sa->p_subn->opt.max_rate_enum < SA_RATE_MAX_ENUM) {
+		new_rate = osm_sa_limit_rate(sa, rate);
 		if (new_rate != rate) {
 			OSM_LOG(sa->p_log, OSM_LOG_VERBOSE,
 				"Rate decreased from %u to %u\n",

opensm/osm_sa_multipath_record.c

@@ -853,8 +853,8 @@ static void mpr_rcv_build_pr(IN osm_sa_t * sa,
 	ib_path_rec_set_sl(p_pr, p_parms->sl);
 	p_pr->mtu = (uint8_t) (p_parms->mtu | 0x80);
 	rate = p_parms->rate;
-	if (sa->p_subn->opt.use_original_extended_sa_rates_only) {
-		new_rate = ib_path_rate_max_12xedr(rate);
+	if (sa->p_subn->opt.max_rate_enum < SA_RATE_MAX_ENUM) {
+		new_rate = osm_sa_limit_rate(sa, rate);
 		if (new_rate != rate) {
 			OSM_LOG(sa->p_log, OSM_LOG_VERBOSE,
 				"Rate decreased from %u to %u\n",

opensm/osm_sa_path_record.c

@@ -921,8 +921,8 @@ static void pr_rcv_build_pr(IN osm_sa_t * sa,
 	ib_path_rec_set_qos_class(p_pr, 0);
 	p_pr->mtu = (uint8_t) (p_parms->mtu | 0x80);
 	rate = p_parms->rate;
-	if (sa->p_subn->opt.use_original_extended_sa_rates_only) {
-		new_rate = ib_path_rate_max_12xedr(rate);
+	if (sa->p_subn->opt.max_rate_enum < SA_RATE_MAX_ENUM) {
+		new_rate = osm_sa_limit_rate(sa, rate);
 		if (new_rate != rate) {
 			OSM_LOG(sa->p_log, OSM_LOG_VERBOSE,
 				"Rate decreased from %u to %u\n",