diff --git a/Documentation/running-tests.md b/Documentation/running-tests.md index ae80860c..6bb71cb8 100644 --- a/Documentation/running-tests.md +++ b/Documentation/running-tests.md @@ -113,7 +113,7 @@ The NVMe tests can be additionally parameterized via environment variables. ### Running nvme-rdma and SRP tests These tests will use the siw (soft-iWARP) driver by default. The rdma_rxe -(soft-RoCE) driver is also supported. +(soft-RoCE) driver and hardware RDMA drivers are also supported. ```sh To use the siw driver: @@ -123,6 +123,13 @@ nvme_trtype=rdma ./check nvme/ To use the rdma_rxe driver: use_rxe=1 nvme_trtype=rdma ./check nvme/ use_rxe=1 ./check srp/ + +To use hardware RDMA drivers, set up hardware RDMA beforehand: +use_hw_rdma=1 nvme_trtype=rdma ./check nvme/ +use_hw_rdma=1 ./check srp/ + +Note: The variables use_rxe and use_hw_rdma must not be enabled at the same time. +Also, Need to make sure to unload the hardware RDMA drivers when testing siw or rdma_rxe. ``` ### Normal user diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma index ee051002..c72771eb 100644 --- a/common/multipath-over-rdma +++ b/common/multipath-over-rdma @@ -13,6 +13,7 @@ fio_aux_path=/tmp/fio-state-files memtotal=$(sed -n 's/^MemTotal:[[:blank:]]*\([0-9]*\)[[:blank:]]*kB$/\1/p' /proc/meminfo) max_ramdisk_size=$((1<<25)) use_rxe=${use_rxe:-""} +use_hw_rdma=${use_hw_rdma:-""} ramdisk_size=$((memtotal*(1024/16))) # in bytes if [ $ramdisk_size -gt $max_ramdisk_size ]; then ramdisk_size=$max_ramdisk_size @@ -439,6 +440,23 @@ stop_soft_rdma() { } >>"$FULL" } +start_rdma() { + if [[ -n "$use_rxe" && -n "$use_hw_rdma" ]]; then + echo "The variables use_rxe and use_hw_rdma must not be enable at the same time." + return 1 + fi + + if [ -z "$use_hw_rdma" ]; then + start_soft_rdma + fi +} + +stop_rdma() { + if [ -z "$use_hw_rdma" ]; then + stop_soft_rdma + fi +} + # Look up the block device below the filesystem for directory $1. block_dev_of_dir() { df "$1" | { @@ -580,7 +598,7 @@ configure_null_blk() { } setup_rdma() { - start_soft_rdma + start_rdma ( echo "RDMA interfaces:" cd /sys/class/infiniband && @@ -599,7 +617,7 @@ teardown_uncond() { killall -9 multipathd >&/dev/null rm -f /etc/multipath.conf stop_target - stop_soft_rdma + stop_rdma _exit_null_blk } diff --git a/tests/nvme/rc b/tests/nvme/rc index bac2db72..e22a73c7 100644 --- a/tests/nvme/rc +++ b/tests/nvme/rc @@ -46,7 +46,7 @@ _nvme_requires() { _have_program rdma if [ -n "$use_rxe" ]; then _have_driver rdma_rxe - else + elif [ -z "$use_hw_rdma" ]; then _have_driver siw fi ;; @@ -372,7 +372,7 @@ _cleanup_nvmet() { fi modprobe -rq nvmet 2>/dev/null if [[ "${nvme_trtype}" == "rdma" ]]; then - stop_soft_rdma + stop_rdma fi _cleanup_blkdev @@ -386,7 +386,7 @@ _setup_nvmet() { fi modprobe -q nvme-"${nvme_trtype}" if [[ "${nvme_trtype}" == "rdma" ]]; then - start_soft_rdma + start_rdma for i in $(rdma_network_interfaces) do if [[ "${nvme_adrfam}" == "ipv6" ]]; then @@ -554,7 +554,11 @@ _nvme_connect_subsys() { ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}") fi - nvme connect "${ARGS[@]}" 2> /dev/null + if ! nvme connect "${ARGS[@]}" 2> /dev/null; then + if [[ -n "$use_hw_rdma" ]]; then + echo "Check RDMA hardware set up; use_hw_rdma is enabledand 'nvme connect' failed." + fi + fi # Wait until device file and uuid/wwid sysfs attributes get ready for # all namespaces. diff --git a/tests/srp/rc b/tests/srp/rc index c77ef6c7..a2a8c46a 100755 --- a/tests/srp/rc +++ b/tests/srp/rc @@ -53,7 +53,7 @@ group_requires() { _have_module rdma_cm if [ -n "$use_rxe" ]; then _have_module rdma_rxe - else + elif [ -z "$use_hw_rdma" ]; then _have_module siw _have_kver 5 5 _have_iproute2 190404