rdma: Add support for running the rdma tests over hardware

The blktests are using soft-RoCE (rdma_rxe) and soft-iWARP (siw) to run
RDMA related tests, this change add support for run nvme-rdma and SRP
tests with real RDMA hardware, this is needed to make sure that we don't
have issues when using real RDMA hardware.

Signed-off-by: Kamal Heib <kheib@redhat.com>

Documentation/running-tests.md

@@ -113,7 +113,7 @@ The NVMe tests can be additionally parameterized via environment variables.
 ### Running nvme-rdma and SRP tests
 
 These tests will use the siw (soft-iWARP) driver by default. The rdma_rxe
-(soft-RoCE) driver is also supported.
+(soft-RoCE) driver and hardware RDMA drivers are also supported.
 
 ```sh
 To use the siw driver:
@@ -123,6 +123,13 @@ nvme_trtype=rdma ./check nvme/
 To use the rdma_rxe driver:
 use_rxe=1 nvme_trtype=rdma ./check nvme/
 use_rxe=1 ./check srp/
+
+To use hardware RDMA drivers, set up hardware RDMA beforehand:
+use_hw_rdma=1 nvme_trtype=rdma ./check nvme/
+use_hw_rdma=1 ./check srp/
+
+Note: The variables use_rxe and use_hw_rdma must not be enabled at the same time.
+Also, Need to make sure to unload the hardware RDMA drivers when testing siw or rdma_rxe.
 ```
 
 ### Normal user

common/multipath-over-rdma

@@ -13,6 +13,7 @@ fio_aux_path=/tmp/fio-state-files
 memtotal=$(sed -n 's/^MemTotal:[[:blank:]]*\([0-9]*\)[[:blank:]]*kB$/\1/p' /proc/meminfo)
 max_ramdisk_size=$((1<<25))
 use_rxe=${use_rxe:-""}
+use_hw_rdma=${use_hw_rdma:-""}
 ramdisk_size=$((memtotal*(1024/16)))  # in bytes
 if [ $ramdisk_size -gt $max_ramdisk_size ]; then
 	ramdisk_size=$max_ramdisk_size
@@ -439,6 +440,23 @@ stop_soft_rdma() {
 	} >>"$FULL"
 }
 
+start_rdma() {
+	if [[ -n "$use_rxe" && -n "$use_hw_rdma" ]]; then
+		echo "The variables use_rxe and use_hw_rdma must not be enable at the same time."
+		return 1
+	fi
+
+	if [ -z "$use_hw_rdma" ]; then
+		start_soft_rdma
+	fi
+}
+
+stop_rdma() {
+	if [ -z "$use_hw_rdma" ]; then
+		stop_soft_rdma
+	fi
+}
+
 # Look up the block device below the filesystem for directory $1.
 block_dev_of_dir() {
 	df "$1" | {
@@ -580,7 +598,7 @@ configure_null_blk() {
 }
 
 setup_rdma() {
-	start_soft_rdma
+	start_rdma
 	(
 		echo "RDMA interfaces:"
 		cd /sys/class/infiniband &&
@@ -599,7 +617,7 @@ teardown_uncond() {
 	killall -9 multipathd >&/dev/null
 	rm -f /etc/multipath.conf
 	stop_target
-	stop_soft_rdma
+	stop_rdma
 	_exit_null_blk
 }
 

tests/nvme/rc

@@ -46,7 +46,7 @@ _nvme_requires() {
 		_have_program rdma
 		if [ -n "$use_rxe" ]; then
 			_have_driver rdma_rxe
-		else
+		elif [ -z "$use_hw_rdma" ]; then
 			_have_driver siw
 		fi
 		;;
@@ -372,7 +372,7 @@ _cleanup_nvmet() {
 	fi
 	modprobe -rq nvmet 2>/dev/null
 	if [[ "${nvme_trtype}" == "rdma" ]]; then
-		stop_soft_rdma
+		stop_rdma
 	fi
 
 	_cleanup_blkdev
@@ -386,7 +386,7 @@ _setup_nvmet() {
 	fi
 	modprobe -q nvme-"${nvme_trtype}"
 	if [[ "${nvme_trtype}" == "rdma" ]]; then
-		start_soft_rdma
+		start_rdma
 		for i in $(rdma_network_interfaces)
 		do
 			if [[ "${nvme_adrfam}" == "ipv6" ]]; then
@@ -554,7 +554,11 @@ _nvme_connect_subsys() {
 		ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
 	fi
 
-	nvme connect "${ARGS[@]}" 2> /dev/null
+	if ! nvme connect "${ARGS[@]}" 2> /dev/null; then
+		if [[ -n "$use_hw_rdma" ]]; then
+			echo "Check RDMA hardware set up; use_hw_rdma is enabledand 'nvme connect' failed."
+		fi
+	fi
 
 	# Wait until device file and uuid/wwid sysfs attributes get ready for
 	# all namespaces.

tests/srp/rc

@@ -53,7 +53,7 @@ group_requires() {
 	_have_module rdma_cm
 	if [ -n "$use_rxe" ]; then
 		_have_module rdma_rxe
-	else
+	elif [ -z "$use_hw_rdma" ]; then
 		_have_module siw
 		_have_kver 5 5
 		_have_iproute2 190404