]> www.infradead.org Git - users/sagi/blktests.git/commitdiff
Add NVMeOF dm-mpath tests
authorBart Van Assche <bvanassche@acm.org>
Mon, 17 Sep 2018 16:59:46 +0000 (09:59 -0700)
committerBart Van Assche <bvanassche@acm.org>
Thu, 27 Sep 2018 23:20:54 +0000 (16:20 -0700)
Add a series of tests for the NVMeOF drivers on top of the dm-mpath
driver. These tests are similar to the tests under tests/srp. Both
tests use the dm-mpath driver for multipath and the loopback
functionality of the rdma_rxe driver. The only difference is that the
nvmeof-mp tests use the NVMeOF initiator and target drivers instead
of the SRP initiator and target drivers.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
21 files changed:
tests/nvmeof-mp/.gitignore [new file with mode: 0644]
tests/nvmeof-mp/001 [new file with mode: 0755]
tests/nvmeof-mp/001.out [new file with mode: 0644]
tests/nvmeof-mp/002 [new file with mode: 0755]
tests/nvmeof-mp/002.out [new file with mode: 0644]
tests/nvmeof-mp/004 [new file with mode: 0755]
tests/nvmeof-mp/004.out [new file with mode: 0644]
tests/nvmeof-mp/005 [new file with mode: 0755]
tests/nvmeof-mp/005.out [new file with mode: 0644]
tests/nvmeof-mp/006 [new file with mode: 0755]
tests/nvmeof-mp/006.out [new file with mode: 0644]
tests/nvmeof-mp/009 [new file with mode: 0755]
tests/nvmeof-mp/009.out [new file with mode: 0644]
tests/nvmeof-mp/010 [new file with mode: 0755]
tests/nvmeof-mp/010.out [new file with mode: 0644]
tests/nvmeof-mp/011 [new file with mode: 0755]
tests/nvmeof-mp/011.out [new file with mode: 0644]
tests/nvmeof-mp/012 [new file with mode: 0755]
tests/nvmeof-mp/012.out [new file with mode: 0644]
tests/nvmeof-mp/multipath.conf [new file with mode: 0644]
tests/nvmeof-mp/rc [new file with mode: 0755]

diff --git a/tests/nvmeof-mp/.gitignore b/tests/nvmeof-mp/.gitignore
new file mode 100644 (file)
index 0000000..ecb6268
--- /dev/null
@@ -0,0 +1 @@
+multipath.conf
diff --git a/tests/nvmeof-mp/001 b/tests/nvmeof-mp/001
new file mode 100755 (executable)
index 0000000..01e3037
--- /dev/null
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Log in and log out"
+QUICK=1
+
+count_devices() {
+       local d devs=0
+
+       for d in /sys/class/nvme-fabrics/ctl/*/*/device; do
+               [ -d "$d" ] && ((devs++))
+       done
+       echo $devs
+}
+
+wait_for_devices() {
+       local expected=1 i devices
+
+       use_blk_mq y || return $?
+       for ((i=0;i<100;i++)); do
+               devices=$(count_devices)
+               [ "$devices" -ge $expected ] && break
+               sleep .1
+       done
+       echo "count_devices(): $devices <> $expected" >>"$FULL"
+       echo "count_devices(): $devices <> $expected"
+       [ "$devices" -ge $expected ]
+}
+
+test() {
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && wait_for_devices && echo Passed
+}
diff --git a/tests/nvmeof-mp/001.out b/tests/nvmeof-mp/001.out
new file mode 100644 (file)
index 0000000..2ce8d17
--- /dev/null
@@ -0,0 +1,3 @@
+Configured NVMe target driver
+count_devices(): 1 <> 1
+Passed
diff --git a/tests/nvmeof-mp/002 b/tests/nvmeof-mp/002
new file mode 100755 (executable)
index 0000000..8425357
--- /dev/null
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="File I/O on top of multipath concurrently with logout and login (mq)"
+TIMED=1
+
+test_disconnect_repeatedly() {
+       local dev fio_status m
+
+       use_blk_mq y || return $?
+       dev=$(get_bdev 0) || return $?
+       m=$(mountpoint 0) || return $?
+       create_filesystem "$dev" || return $?
+       mount_and_check "$dev" "$m" || return $?
+       # shellcheck disable=SC2064
+       trap "unmount_and_check $m" RETURN
+       simulate_network_failure_loop "$dev" "$TIMEOUT" &
+       run_fio --verify=md5 --rw=randwrite --bs=4K --loops=$((10**6)) \
+               --iodepth=64 --group_reporting --sync=1 --direct=1 \
+               --ioengine=libaio --directory="$m" --runtime="${TIMEOUT}" \
+               --name=data-integrity-test-mq --thread --numjobs=16 \
+               --output="${RESULTS_DIR}/nvmeof-mp/fio-output-002.txt" \
+               >>"$FULL"
+       fio_status=$?
+       wait
+       return $fio_status
+}
+
+test() {
+       : "${TIMEOUT:=30}"
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && test_disconnect_repeatedly && echo Passed
+}
diff --git a/tests/nvmeof-mp/002.out b/tests/nvmeof-mp/002.out
new file mode 100644 (file)
index 0000000..a7d4cb9
--- /dev/null
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
diff --git a/tests/nvmeof-mp/004 b/tests/nvmeof-mp/004
new file mode 100755 (executable)
index 0000000..87fe98d
--- /dev/null
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="File I/O on top of multipath concurrently with logout and login (sq-on-mq)"
+TIMED=1
+
+test_disconnect_repeatedly() {
+       local dev fio_status m
+
+       use_blk_mq n || return $?
+       dev=$(get_bdev 0) || return $?
+       m=$(mountpoint 0) || return $?
+       create_filesystem "$dev" || return $?
+       mount_and_check "$dev" "$m" || return $?
+       # shellcheck disable=SC2064
+       trap "unmount_and_check $m" RETURN
+       simulate_network_failure_loop "$dev" "$TIMEOUT" &
+       run_fio --verify=md5 --rw=randwrite --bs=4K --loops=$((10**6)) \
+               --iodepth=64 --group_reporting --sync=1 --direct=1 \
+               --ioengine=libaio --directory="$m" \
+               --name=data-integrity-test-02-sq-on-mq --thread \
+               --numjobs=16 --runtime="${TIMEOUT}" \
+               --output="${RESULTS_DIR}/nvmeof-mp/fio-output-004.txt" >>"$FULL"
+       fio_status=$?
+       wait
+       return $fio_status
+}
+
+test() {
+       : "${TIMEOUT:=30}"
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && test_disconnect_repeatedly && echo Passed
+}
diff --git a/tests/nvmeof-mp/004.out b/tests/nvmeof-mp/004.out
new file mode 100644 (file)
index 0000000..a7d4cb9
--- /dev/null
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
diff --git a/tests/nvmeof-mp/005 b/tests/nvmeof-mp/005
new file mode 100755 (executable)
index 0000000..3bd6934
--- /dev/null
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Direct I/O with large transfer sizes and bs=4M"
+QUICK=1
+
+test_large_transfer_size() {
+       local dev m
+
+       use_blk_mq y || return $?
+       dev=$(get_bdev 0) || return $?
+       run_fio --verify=md5 --rw=randwrite --bs=4M --loops=$((10**6)) \
+               --iodepth=4 --group_reporting --sync=1 --direct=1 \
+               --ioengine=libaio \
+               --filename="$dev" --name=large-io-test --thread --numjobs=1 \
+               --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-005.txt" \
+               >>"$FULL"
+}
+
+test() {
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && test_large_transfer_size && echo Passed
+}
diff --git a/tests/nvmeof-mp/005.out b/tests/nvmeof-mp/005.out
new file mode 100644 (file)
index 0000000..a7d4cb9
--- /dev/null
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
diff --git a/tests/nvmeof-mp/006 b/tests/nvmeof-mp/006
new file mode 100755 (executable)
index 0000000..796e7c6
--- /dev/null
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Direct I/O with large transfer sizes and bs=8M"
+QUICK=1
+
+test_large_transfer_size() {
+       local dev m
+
+       use_blk_mq y || return $?
+       dev=$(get_bdev 0) || return $?
+       run_fio --verify=md5 --rw=randwrite --bs=8M --loops=$((10**6)) \
+               --iodepth=4 --group_reporting --sync=1 --direct=1 \
+               --ioengine=libaio \
+               --filename="$dev" --name=large-io-test --thread --numjobs=1 \
+               --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-006.txt" \
+               >>"$FULL"
+}
+
+test() {
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && test_large_transfer_size && echo Passed
+}
diff --git a/tests/nvmeof-mp/006.out b/tests/nvmeof-mp/006.out
new file mode 100644 (file)
index 0000000..a7d4cb9
--- /dev/null
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
diff --git a/tests/nvmeof-mp/009 b/tests/nvmeof-mp/009
new file mode 100755 (executable)
index 0000000..00849d6
--- /dev/null
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Buffered I/O with large transfer sizes and bs=4M"
+QUICK=1
+
+test_large_transfer_size() {
+       local dev m
+
+       use_blk_mq y || return $?
+       dev=$(get_bdev 0) || return $?
+       run_fio --verify=md5 --rw=randwrite --bs=4M --loops=$((10**6)) \
+               --iodepth=4 --group_reporting --sync=1 --direct=0 \
+               --ioengine=libaio \
+               --filename="$dev" --name=large-io-test --thread --numjobs=1 \
+               --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-009.txt" \
+               >>"$FULL"
+}
+
+test() {
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && test_large_transfer_size && echo Passed
+}
diff --git a/tests/nvmeof-mp/009.out b/tests/nvmeof-mp/009.out
new file mode 100644 (file)
index 0000000..a7d4cb9
--- /dev/null
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
diff --git a/tests/nvmeof-mp/010 b/tests/nvmeof-mp/010
new file mode 100755 (executable)
index 0000000..70bc422
--- /dev/null
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Buffered I/O with large transfer sizes and bs=8M"
+QUICK=1
+
+test_large_transfer_size() {
+       local dev m
+
+       use_blk_mq y || return $?
+       dev=$(get_bdev 0) || return $?
+       run_fio --verify=md5 --rw=randwrite --bs=8M --loops=$((10**6)) \
+               --iodepth=4 --group_reporting --sync=1 --direct=0 \
+               --ioengine=libaio \
+               --filename="$dev" --name=large-io-test --thread --numjobs=1 \
+               --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-010.txt" \
+               >>"$FULL"
+}
+
+test() {
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && test_large_transfer_size && echo Passed
+}
diff --git a/tests/nvmeof-mp/010.out b/tests/nvmeof-mp/010.out
new file mode 100644 (file)
index 0000000..a7d4cb9
--- /dev/null
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
diff --git a/tests/nvmeof-mp/011 b/tests/nvmeof-mp/011
new file mode 100755 (executable)
index 0000000..4dfe275
--- /dev/null
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="Block I/O on top of multipath concurrently with logout and login"
+TIMED=1
+
+test_disconnect_repeatedly() {
+       local dev fio_status m
+
+       use_blk_mq y || return $?
+       dev=$(get_bdev 0) || return $?
+       simulate_network_failure_loop "$dev" "$TIMEOUT" &
+       run_fio --verify=md5 --rw=randwrite --bs=4K --loops=10000 \
+               --ioengine=libaio --iodepth=64 --iodepth_batch=32 \
+               --group_reporting --sync=1 --direct=1 --filename="$dev" \
+               --name=data-integrity-test-06 --thread --numjobs=1 \
+               --runtime="${TIMEOUT}" \
+               --output="${RESULTS_DIR}/nvmeof-mp/fio-output-011.txt" \
+               >>"$FULL"
+       fio_status=$?
+       wait
+       return $fio_status
+}
+
+test() {
+       : "${TIMEOUT:=30}"
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && test_disconnect_repeatedly && echo Passed
+}
diff --git a/tests/nvmeof-mp/011.out b/tests/nvmeof-mp/011.out
new file mode 100644 (file)
index 0000000..a7d4cb9
--- /dev/null
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
diff --git a/tests/nvmeof-mp/012 b/tests/nvmeof-mp/012
new file mode 100755 (executable)
index 0000000..e5a9cd2
--- /dev/null
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. tests/nvmeof-mp/rc
+
+DESCRIPTION="dm-mpath on top of multiple I/O schedulers"
+QUICK=1
+
+test_io_schedulers() {
+       local dev m
+
+       # Load all I/O scheduler kernel modules
+       for m in "/lib/modules/$(uname -r)/kernel/block/"*.ko; do
+               modprobe "$(basename "$m")" >&/dev/null
+       done
+       for mq in y n; do
+               use_blk_mq ${mq} || return $?
+               dev=$(get_bdev 0) || return $?
+               for sched in noop deadline bfq cfq; do
+                       set_scheduler "$(basename "$(readlink -f "${dev}")")" $sched \
+                                     >>"$FULL" 2>&1 || continue
+                       echo "I/O scheduler: $sched; use_blk_mq: $mq" >>"$FULL"
+                       run_fio --verify=md5 --rw=randwrite --bs=4K --size=64K \
+                               --ioengine=libaio --iodepth=64 \
+                               --iodepth_batch=32 --group_reporting --sync=1 \
+                               --direct=1 --filename="$dev" \
+                               --name=${sched} --thread --numjobs=1 \
+                               --output="${RESULTS_DIR}/nvmeof-mp/012-${sched}-${mq}.txt" \
+                               >>"$FULL" ||
+                               return $?
+               done
+       done
+       return 0
+}
+
+test() {
+       trap 'trap "" EXIT; teardown' EXIT
+       setup && test_io_schedulers && echo Passed
+}
diff --git a/tests/nvmeof-mp/012.out b/tests/nvmeof-mp/012.out
new file mode 100644 (file)
index 0000000..a7d4cb9
--- /dev/null
@@ -0,0 +1,2 @@
+Configured NVMe target driver
+Passed
diff --git a/tests/nvmeof-mp/multipath.conf b/tests/nvmeof-mp/multipath.conf
new file mode 100644 (file)
index 0000000..8aed22e
--- /dev/null
@@ -0,0 +1,27 @@
+defaults {
+       find_multipaths         no
+       user_friendly_names     yes
+       queue_without_daemon    no
+}
+devices {
+       device {
+               vendor          "NVME"
+               product         "Linux"
+               no_path_retry   "queue"
+               path_checker    "directio"
+       }
+}
+blacklist {
+       device {
+               vendor  "ATA|QEMU"
+       }
+       device {
+               vendor  "Linux"
+               product "scsi_debug"
+       }
+       devnode "^nullb.*"
+}
+blacklist_exceptions {
+       property        ".*"
+       devnode         "^nvme"
+}
diff --git a/tests/nvmeof-mp/rc b/tests/nvmeof-mp/rc
new file mode 100755 (executable)
index 0000000..9bfa3ad
--- /dev/null
@@ -0,0 +1,313 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Western Digital Corporation or its affiliates
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc.
+
+. common/rc
+. common/multipath-over-rdma
+
+namespace=(1)
+elevator=none
+nvme_subsysnqn="nvme-test"
+nvme_port=7777
+ini_timeout=1
+
+group_requires() {
+       local m name p required_modules
+
+       # Since the nvmeof-mp tests are based on the dm-mpath driver, these
+       # tests are incompatible with the NVME_MULTIPATH kernel configuration
+       # option.
+       if _have_kernel_option NVME_MULTIPATH; then
+               SKIP_REASON="CONFIG_NVME_MULTIPATH has been set in .config"
+               return 1
+       fi
+
+       _have_configfs || return $?
+       required_modules=(
+               dm_multipath
+               dm_queue_length
+               dm_service_time
+               null_blk
+               rdma_cm
+               ib_ipoib
+               ib_umad
+               nvme-rdma
+               nvmet-rdma
+               rdma_rxe
+               scsi_dh_alua
+               scsi_dh_emc
+               scsi_dh_rdac
+       )
+       for m in "${required_modules[@]}"; do
+               _have_module "$m" || return $?
+       done
+
+       for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof sg_reset; do
+               _have_program "$p" || return $?
+       done
+
+       _have_root || return $?
+
+       _have_kernel_option DM_UEVENT || return $?
+
+       # shellcheck disable=SC2043
+       for name in multipathd; do
+               if pidof "$name" >/dev/null; then
+                       SKIP_REASON="$name must be stopped before the nvmeof-mp tests are run"
+                       return 1
+               fi
+       done
+       if [ -e /etc/multipath.conf ] &&
+           ! diff -q /etc/multipath.conf tests/nvmeof-mp/multipath.conf >&/dev/null
+       then
+               SKIP_REASON="/etc/multipath.conf already exists"
+               return 1
+       fi
+}
+
+# Log out, set dm use_blk_mq parameter to $1 and log in.
+use_blk_mq() {
+       local dm_mode=$1
+
+       (
+               cd /sys/module/dm_mod/parameters || return $?
+               if [ -e use_blk_mq ]; then
+                       echo "$dm_mode" >use_blk_mq || return $?
+               fi
+       )
+
+       log_out &&
+               remove_mpath_devs &&
+               start_client &&
+               log_in &&
+               return 0
+
+       echo "use_blk_mq $* failed" >>"$FULL"
+       return 1
+}
+
+log_in() {
+       local i ipv4_addr
+
+       [ -c /dev/nvme-fabrics ] &&
+               for i in $(rdma_network_interfaces); do
+                       ipv4_addr=$(get_ipv4_addr "$i")
+                       if [ -n "${ipv4_addr}" ]; then
+                               { echo -n "transport=rdma,traddr=${ipv4_addr},trsvcid=${nvme_port},nqn=$nvme_subsysnqn" > /dev/nvme-fabrics; } 2>>"$FULL"
+                       fi
+               done
+}
+
+log_out() {
+       local c
+
+       for c in /sys/class/nvme-fabrics/ctl/*/delete_controller; do
+               [ -e "$c" ] && echo 1 > "$c" &
+       done
+       wait
+}
+
+# Simulate network failures for device $1 during $2 seconds.
+simulate_network_failure_loop() {
+       local d dev="$1" duration="$2" deadline i rc=0
+
+       [ -e "$dev" ] || return $?
+       [ -n "$duration" ] || return $?
+       deadline=$(($(uptime_s) + duration))
+       while [ $rc = 0 ]; do
+               sleep_until 5 ${deadline} || break
+               for d in $(held_by "$dev"); do
+                       echo 1 >"$d/device/reset_controller"
+               done
+       done
+
+       for ((i=0;i<5;i++)); do
+               log_in && break
+               sleep 1
+       done
+}
+
+remove_mpath_devs() {
+       local dm h
+
+       {
+               for h in /sys/class/block/nvme*/holders/*; do
+                       [ -e "$h" ] || continue
+                       d=$(basename "$(dirname "$(dirname "$h")")")
+                       dm=/dev/$(basename "$h")
+                       echo -n "NVME dev $d: removing $dm: "
+                       dmsetup remove "$(dev_to_mpath "$dm")" && echo "done"
+               done
+
+               remove_stale_mpath_devs
+       } &>> "$FULL"
+}
+
+# Arguments: module to unload ($1) and retry count ($2).
+unload_module() {
+       local i m=$1 rc=${2:-1}
+
+       [ ! -e "/sys/module/$m" ] && return 0
+       for ((i=rc;i>0;i--)); do
+               modprobe -r "$m"
+               [ ! -e "/sys/module/$m" ] && return 0
+               sleep .1
+       done
+       return 1
+}
+
+start_nvme_client() {
+       modprobe nvme-core dyndbg=+pmf &&
+               modprobe nvme dyndbg=+pmf &&
+               modprobe nvme-fabrics dyndbg=+pmf &&
+               modprobe nvme-rdma dyndbg=+pmf &&
+               mkdir -p "$(mountpoint 0)"
+       udevadm settle
+       if [ ! -c /dev/nvme-fabrics ]; then
+               echo "Error:  /dev/nvme-fabrics not available"
+       fi
+}
+
+stop_nvme_client() {
+       unload_module nvme-rdma || return $?
+       unload_module nvme-fabrics || return $?
+       # Ignore nvme and nvme-core unload errors - this test may be run on a
+       # system equipped with one or more NVMe SSDs.
+       unload_module nvme >&/dev/null
+       unload_module nvme-core >&/dev/null
+       return 0
+}
+
+# Load the initiator kernel driver with kernel module parameters $1..$n.
+start_client() {
+       start_nvme_client
+}
+
+stop_client() {
+       stop_nvme_client
+}
+
+# Get the name of the initiator device node that communicates with target
+# device $1. $1 is an index in the $namespace array.
+get_bdev_path() {
+       local i=$1 uuid
+
+       is_number "$i" || return $?
+       uuid=$(<"/sys/kernel/config/nvmet/subsystems/${nvme_subsysnqn}/namespaces/${namespace[$1]}/device_uuid") || return $?
+       echo "/dev/disk/by-id/dm-uuid-mpath-uuid.$uuid"
+}
+
+# Get a /dev/... path that points at dm device number $1. $1 is an index in
+# the $namespace array.
+get_bdev() {
+       get_bdev_n "$1" "$elevator" "$ini_timeout"
+}
+
+configure_nvmet_port() {
+       local p=$1 ipv4_addr=$2 i
+
+       echo "Configuring $p with address $ipv4_addr as an NVMeOF target port" \
+            >>"$FULL"
+       (
+               cd /sys/kernel/config/nvmet/ports &&
+                       for ((i=1;1;i++)); do [ -e "$i" ] || break; done &&
+                       mkdir "$i" &&
+                       cd "$i" &&
+                       echo ipv4            > addr_adrfam &&
+                       echo rdma            > addr_trtype &&
+                       echo -n "$ipv4_addr" > addr_traddr &&
+                       echo -n ${nvme_port} > addr_trsvcid
+       )
+}
+
+start_nvme_target() {
+       local d i ipv4_addr num_ports=0
+
+       echo "Configuring NVMe target driver ..." >>"$FULL"
+       modprobe nvmet dyndbg=+pmf &&
+               modprobe nvmet-rdma dyndbg=+pmf &&
+               sleep .1 &&
+               (
+                       cd /sys/kernel/config/nvmet/subsystems &&
+                               mkdir ${nvme_subsysnqn} &&
+                               cd ${nvme_subsysnqn} &&
+                               cd namespaces &&
+                               mkdir "${namespace[0]}" &&
+                               cd "${namespace[0]}" &&
+                               echo 00000000-0000-0000-0000-000000000000 >device_nguid &&
+                               echo -n /dev/nullb0 >device_path &&
+                               echo 1 >enable &&
+                               cd ../.. &&
+                               echo 1 >attr_allow_any_host
+               ) && for i in $(rdma_network_interfaces); do
+                       ipv4_addr=$(get_ipv4_addr "$i")
+                       if [ -n "${ipv4_addr}" ]; then
+                               configure_nvmet_port "$i" "${ipv4_addr}"
+                               ((num_ports++))
+                               true
+                       fi
+               done &&
+               if [ $num_ports = 0 ]; then
+                       echo "No NVMeOF target ports"
+                       false
+               fi && (
+                       cd /sys/kernel/config/nvmet/ports &&
+                               for i in *; do
+                                       [ -e "$i" ] && (
+                                               cd "$i/subsystems" &&
+                                                       ln -s "../../../subsystems/${nvme_subsysnqn}" .
+                                       )
+                               done
+               )
+       echo "Configured NVMe target driver"
+}
+
+stop_nvme_target() {
+       local d
+
+       (
+               cd /sys/kernel/config/nvmet 2>/dev/null &&
+                       rm -f -- ports/*/subsystems/* &&
+                       for d in {*/*/*/*,*/*}; do
+                               [ -e "$d" ] &&
+                                       [ "$(basename "$(dirname "$d")")" != ana_groups ] &&
+                                       rmdir "$d"
+                       done
+       )
+       unload_module nvmet_rdma &&
+               unload_module nvmet &&
+               unload_null_blk
+}
+
+start_target() {
+       start_nvme_target
+}
+
+stop_target() {
+       stop_nvme_target
+}
+
+shutdown_client() {
+       remove_mpath_devs &&
+               log_out &&
+               stop_client
+}
+
+# Set up test configuration
+setup() {
+       setup_test "$PWD/tests/nvmeof-mp/multipath.conf"
+}