- nvme_num_iter: 1000 (default)
The number of iterations a test should do.
-### Running nvme-rdma nvmeof-mp srp tests
+### Running nvme-rdma srp tests
Most of these tests will use the rdma_rxe (soft-RoCE) driver by default. The siw (soft-iWARP) driver is also supported.
```sh
To use the rdma_rxe driver:
nvme_trtype=rdma ./check nvme/
-./check nvmeof-mp/
./check srp/
To use the siw driver:
use_siw=1 nvme_trtype=rdma ./check nvme/
-use_siw=1 ./check nvmeof-mp/
use_siw=1 ./check srp/
```
# SPDX-License-Identifier: GPL-2.0+
# Copyright (c) 2017-2018 Western Digital Corporation or its affiliates.
#
-# Functions and global variables used by both the srp and nvmeof-mp tests.
+# Functions and global variables used by the srp tests.
. common/shellcheck
. common/null_blk
+++ /dev/null
-multipath.conf
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="Log in and log out"
-QUICK=1
-
-count_devices() {
- local d devs=0
-
- for d in /sys/class/nvme-fabrics/ctl/*/*/device; do
- [[ "$d" =~ hwmon[0-9]+|ng[0-9]+n[0-9]+ ]] && continue
- [ -d "$d" ] && ((devs++))
- done
- echo "$devs"
-}
-
-wait_for_devices() {
- local expected=0 i devices
-
- for i in $(rdma_network_interfaces); do
- ((expected++))
- done
-
- use_blk_mq y || return $?
- for ((i=0;i<100;i++)); do
- devices=$(count_devices)
- [ "$devices" -ge "$expected" ] && break
- sleep .1
- done
- echo "count_devices(): $devices <> $expected" >>"$FULL"
- [ "$devices" -ge "$expected" ]
-}
-
-test() {
- trap 'trap "" EXIT; teardown' EXIT
- setup && wait_for_devices && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="File I/O on top of multipath concurrently with logout and login (mq)"
-TIMED=1
-
-test_disconnect_repeatedly() {
- local dev fio_status m
-
- use_blk_mq y || return $?
- dev=$(get_bdev 0) || return $?
- m=$(mountpoint 0) || return $?
- create_filesystem "$dev" || return $?
- mount_and_check "$dev" "$m" || return $?
- # shellcheck disable=SC2064
- trap "unmount_and_check $m" RETURN
- simulate_network_failure_loop "$dev" "$TIMEOUT" &
- run_fio --verify=md5 --rw=randwrite --bs=4K --loops=$((10**6)) \
- --iodepth=64 --group_reporting --sync=1 --direct=1 \
- --ioengine=libaio --directory="$m" --runtime="${TIMEOUT}" \
- --name=data-integrity-test-mq --thread --numjobs=16 \
- --output="${RESULTS_DIR}/nvmeof-mp/fio-output-002.txt" \
- >>"$FULL"
- fio_status=$?
- wait
- return $fio_status
-}
-
-test() {
- : "${TIMEOUT:=30}"
- trap 'trap "" EXIT; teardown' EXIT
- setup && test_disconnect_repeatedly && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="File I/O on top of multipath concurrently with logout and login (sq-on-mq)"
-TIMED=1
-
-requires() {
- _have_legacy_dm
-}
-
-test_disconnect_repeatedly() {
- local dev fio_status m
-
- use_blk_mq n || return $?
- dev=$(get_bdev 0) || return $?
- m=$(mountpoint 0) || return $?
- create_filesystem "$dev" || return $?
- mount_and_check "$dev" "$m" || return $?
- # shellcheck disable=SC2064
- trap "unmount_and_check $m" RETURN
- simulate_network_failure_loop "$dev" "$TIMEOUT" &
- run_fio --verify=md5 --rw=randwrite --bs=4K --loops=$((10**6)) \
- --iodepth=64 --group_reporting --sync=1 --direct=1 \
- --ioengine=libaio --directory="$m" \
- --name=data-integrity-test-02-sq-on-mq --thread \
- --numjobs=16 --runtime="${TIMEOUT}" \
- --output="${RESULTS_DIR}/nvmeof-mp/fio-output-004.txt" >>"$FULL"
- fio_status=$?
- wait
- return $fio_status
-}
-
-test() {
- : "${TIMEOUT:=30}"
- trap 'trap "" EXIT; teardown' EXIT
- setup && test_disconnect_repeatedly && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="Direct I/O with large transfer sizes and bs=4M"
-QUICK=1
-
-test_large_transfer_size() {
- local dev m
-
- use_blk_mq y || return $?
- dev=$(get_bdev 0) || return $?
- run_fio --verify=md5 --rw=randwrite --bs=4M --loops=$((10**6)) \
- --iodepth=4 --group_reporting --sync=1 --direct=1 \
- --ioengine=libaio \
- --filename="$dev" --name=large-io-test --thread --numjobs=1 \
- --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-005.txt" \
- >>"$FULL"
-}
-
-test() {
- trap 'trap "" EXIT; teardown' EXIT
- setup && test_large_transfer_size && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="Direct I/O with large transfer sizes and bs=8M"
-QUICK=1
-
-test_large_transfer_size() {
- local dev m
-
- use_blk_mq y || return $?
- dev=$(get_bdev 0) || return $?
- run_fio --verify=md5 --rw=randwrite --bs=8M --loops=$((10**6)) \
- --iodepth=4 --group_reporting --sync=1 --direct=1 \
- --ioengine=libaio \
- --filename="$dev" --name=large-io-test --thread --numjobs=1 \
- --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-006.txt" \
- >>"$FULL"
-}
-
-test() {
- trap 'trap "" EXIT; teardown' EXIT
- setup && test_large_transfer_size && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="Buffered I/O with large transfer sizes and bs=4M"
-QUICK=1
-
-test_large_transfer_size() {
- local dev m
-
- use_blk_mq y || return $?
- dev=$(get_bdev 0) || return $?
- run_fio --verify=md5 --rw=randwrite --bs=4M --loops=$((10**6)) \
- --iodepth=4 --group_reporting --sync=1 --direct=0 \
- --ioengine=libaio \
- --filename="$dev" --name=large-io-test --thread --numjobs=1 \
- --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-009.txt" \
- >>"$FULL"
-}
-
-test() {
- trap 'trap "" EXIT; teardown' EXIT
- setup && test_large_transfer_size && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="Buffered I/O with large transfer sizes and bs=8M"
-QUICK=1
-
-test_large_transfer_size() {
- local dev m
-
- use_blk_mq y || return $?
- dev=$(get_bdev 0) || return $?
- run_fio --verify=md5 --rw=randwrite --bs=8M --loops=$((10**6)) \
- --iodepth=4 --group_reporting --sync=1 --direct=0 \
- --ioengine=libaio \
- --filename="$dev" --name=large-io-test --thread --numjobs=1 \
- --runtime=10 --output="${RESULTS_DIR}/nvmeof-mp/fio-output-010.txt" \
- >>"$FULL"
-}
-
-test() {
- trap 'trap "" EXIT; teardown' EXIT
- setup && test_large_transfer_size && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2016-2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="Block I/O on top of multipath concurrently with logout and login"
-TIMED=1
-
-test_disconnect_repeatedly() {
- local dev fio_status m
-
- use_blk_mq y || return $?
- dev=$(get_bdev 0) || return $?
- simulate_network_failure_loop "$dev" "$TIMEOUT" &
- run_fio --verify=md5 --rw=randwrite --bs=4K --loops=10000 \
- --ioengine=libaio --iodepth=64 --iodepth_batch=32 \
- --group_reporting --sync=1 --direct=1 --filename="$dev" \
- --name=data-integrity-test-06 --thread --numjobs=1 \
- --runtime="${TIMEOUT}" \
- --output="${RESULTS_DIR}/nvmeof-mp/fio-output-011.txt" \
- >>"$FULL"
- fio_status=$?
- wait
- return $fio_status
-}
-
-test() {
- : "${TIMEOUT:=30}"
- trap 'trap "" EXIT; teardown' EXIT
- setup && test_disconnect_repeatedly && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2018 Western Digital Corporation or its affiliates
-
-. tests/nvmeof-mp/rc
-
-DESCRIPTION="dm-mpath on top of multiple I/O schedulers"
-QUICK=1
-
-test_io_schedulers() {
- local dev m dm scheds
-
- # Load all I/O scheduler kernel modules
- for m in "/lib/modules/$(uname -r)/kernel/block/"*.ko; do
- modprobe "$(basename "$m")" >&/dev/null
- done
- for mq in y n; do
- use_blk_mq ${mq} || return $?
- dev=$(get_bdev 0) || return $?
- dm=$(basename "$(readlink -f "${dev}")") || return $?
- scheds="$(_io_schedulers "$dm")" || return $?
- for sched in $scheds; do
- set_scheduler "$dm" "$sched" \
- >>"$FULL" 2>&1 || continue
- echo "I/O scheduler: $sched; use_blk_mq: $mq" >>"$FULL"
- run_fio --verify=md5 --rw=randwrite --bs=4K --size=64K \
- --ioengine=libaio --iodepth=64 \
- --iodepth_batch=32 --group_reporting --sync=1 \
- --direct=1 --filename="$dev" \
- --name="${sched}" --thread --numjobs=1 \
- --output="${RESULTS_DIR}/nvmeof-mp/012-${sched}-${mq}.txt" \
- >>"$FULL" ||
- return $?
- done
- done
- return 0
-}
-
-test() {
- trap 'trap "" EXIT; teardown' EXIT
- setup && test_io_schedulers && echo Passed
-}
+++ /dev/null
-Configured NVMe target driver
-Passed
+++ /dev/null
-defaults {
- find_multipaths no
- user_friendly_names yes
- queue_without_daemon no
-}
-devices {
- device {
- vendor "NVME"
- product "Linux"
- no_path_retry "queue"
- path_checker "directio"
- }
-}
-blacklist {
- device {
- vendor "ATA|QEMU"
- }
- device {
- vendor "Linux"
- product "scsi_debug"
- }
- devnode "^nullb.*"
-}
-blacklist_exceptions {
- property ".*"
- devnode "^nvme"
-}
+++ /dev/null
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0+
-# Copyright (c) 2018 Western Digital Corporation or its affiliates
-
-. common/rc
-. common/multipath-over-rdma
-
-namespace=(1)
-elevator=none
-nvme_subsysnqn="nvme-test"
-nvme_port=7777
-ini_timeout=1
-
-group_requires() {
- local m name p
-
- _have_kernel_config_file || return
- # Since the nvmeof-mp tests are based on the dm-mpath driver, these
- # tests are incompatible with the NVME_MULTIPATH kernel configuration
- # option with multipathing enabled in the nvme_core kernel module.
- if _check_kernel_option NVME_MULTIPATH && \
- _have_module_param_value nvme_core multipath Y; then
- SKIP_REASONS+=("CONFIG_NVME_MULTIPATH has been set in .config and multipathing has been enabled in the nvme_core kernel module")
- return
- fi
-
- _have_configfs || return
-
- _have_module dm_multipath
- _have_module dm_queue_length
- _have_module dm_service_time
- _have_module null_blk
- _have_module rdma_cm
- _have_module ib_ipoib
- _have_module ib_umad
- _have_module nvme-rdma
- _have_module nvmet-rdma
- _have_module rdma_rxe
- _have_module scsi_dh_alua
- _have_module scsi_dh_emc
- _have_module scsi_dh_rdac
-
- for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof rdma fio; do
- _have_program "$p" || return
- done
-
- _multipathd_version_ge 0.7.0 || return
-
- _have_root || return
-
- _have_kernel_option DM_UEVENT || return
-
- # shellcheck disable=SC2043
- for name in multipathd; do
- if pidof "$name" >/dev/null; then
- SKIP_REASONS+=("$name must be stopped before the nvmeof-mp tests are run")
- return
- fi
- done
- if [ -e /etc/multipath.conf ] &&
- ! diff -q /etc/multipath.conf tests/nvmeof-mp/multipath.conf >&/dev/null
- then
- SKIP_REASONS+=("/etc/multipath.conf already exists")
- return
- fi
-}
-
-# Log out, set dm use_blk_mq parameter to $1 and log in.
-use_blk_mq() {
- local dm_mode=$1
-
- (
- cd /sys/module/dm_mod/parameters || return $?
- if [ -e use_blk_mq ]; then
- echo "$dm_mode" >use_blk_mq || return 1
- fi
- )
-
- log_out &&
- remove_mpath_devs &&
- start_client &&
- log_in &&
- return 0
-
- echo "use_blk_mq $* failed" >>"$FULL"
- return 1
-}
-
-log_in() {
- local i ipv4_addr loginparams
-
- [ -c /dev/nvme-fabrics ] &&
- for i in $(rdma_network_interfaces); do
- ipv4_addr=$(get_ipv4_addr "$i")
- if [ -n "${ipv4_addr}" ]; then
- loginparams="transport=rdma,traddr=${ipv4_addr},trsvcid=${nvme_port},nqn=$nvme_subsysnqn"
- echo "Login parameters: $loginparams" >>"$FULL"
- {
- echo -n "$loginparams" > /dev/nvme-fabrics
- } 2>>"$FULL"
- fi
- done
-}
-
-log_out() {
- local c
-
- for c in /sys/class/nvme-fabrics/ctl/*/delete_controller; do
- [ -e "$c" ] && echo 1 > "$c" &
- done
- wait
-}
-
-# Simulate network failures for device $1 during $2 seconds.
-simulate_network_failure_loop() {
- local d dev="$1" duration="$2" deadline i rc=0 sf
-
- [ -e "$dev" ] || return 1
- [ -n "$duration" ] || return 1
- deadline=$(($(_uptime_s) + duration))
- while [ $rc = 0 ]; do
- sleep_until 5 ${deadline} || break
- for d in $(held_by "$dev"); do
- for sf in /sys/class/nvme/*/device/*/"${d#/dev/}/reset_controller"; do
- [ -e "$sf" ] && echo 1 > "$sf"
- done
- done
- done 2>>"$FULL"
-
- for ((i=0;i<5;i++)); do
- log_in 2>/dev/null && break
- sleep 1
- done
-}
-
-remove_mpath_devs() {
- local dm h
-
- {
- for h in /sys/class/block/nvme*/holders/*; do
- [ -e "$h" ] || continue
- d=$(basename "$(dirname "$(dirname "$h")")")
- dm=/dev/$(basename "$h")
- echo -n "NVME dev $d: removing $dm: "
- dmsetup remove "$(dev_to_mpath "$dm")" && echo "done"
- done
-
- remove_stale_mpath_devs
- } &>> "$FULL"
-}
-
-start_nvme_client() {
- modprobe nvme-core dyndbg=+pmf &&
- modprobe nvme dyndbg=+pmf &&
- modprobe nvme-fabrics dyndbg=+pmf &&
- modprobe nvme-rdma dyndbg=+pmf &&
- mkdir -p "$(mountpoint 0)"
- udevadm settle
- if [ ! -c /dev/nvme-fabrics ]; then
- echo "Error: /dev/nvme-fabrics not available"
- fi
-}
-
-stop_nvme_client() {
- _unload_module nvme-rdma || return $?
- _unload_module nvme-fabrics || return $?
- # Ignore nvme and nvme-core unload errors - this test may be run on a
- # system equipped with one or more NVMe SSDs.
- _unload_module nvme >&/dev/null
- _unload_module nvme-core >&/dev/null
- return 0
-}
-
-# Load the initiator kernel driver with kernel module parameters $1..$n.
-start_client() {
- start_nvme_client
-}
-
-stop_client() {
- stop_nvme_client
-}
-
-# Get the name of the initiator device node that communicates with target
-# device $1. $1 is an index in the $namespace array.
-get_bdev_path() {
- local i=$1 uuid
-
- is_number "$i" || return $?
- uuid=$(<"/sys/kernel/config/nvmet/subsystems/${nvme_subsysnqn}/namespaces/${namespace[$1]}/device_uuid") || return $?
- echo "/dev/disk/by-id/dm-uuid-mpath-uuid.$uuid"
-}
-
-# Get a /dev/... path that points at dm device number $1. $1 is an index in
-# the $namespace array.
-get_bdev() {
- get_bdev_n "$1" "$elevator" "$ini_timeout"
-}
-
-configure_nvmet_port() {
- local p=$1 ipv4_addr=$2 i
-
- echo "Configuring $p with address $ipv4_addr as an NVMeOF target port" \
- >>"$FULL"
- (
- cd /sys/kernel/config/nvmet/ports &&
- for ((i=1;1;i++)); do [ -e "$i" ] || break; done &&
- mkdir "$i" &&
- cd "$i" &&
- echo ipv4 > addr_adrfam &&
- echo rdma > addr_trtype &&
- echo -n "$ipv4_addr" > addr_traddr &&
- echo -n ${nvme_port} > addr_trsvcid
- )
-}
-
-start_nvme_target() {
- local d i ipv4_addr num_ports=0
-
- echo "Configuring NVMe target driver ..." >>"$FULL"
- modprobe nvmet dyndbg=+pmf &&
- modprobe nvmet-rdma dyndbg=+pmf &&
- sleep .1 &&
- (
- cd /sys/kernel/config/nvmet/subsystems &&
- mkdir ${nvme_subsysnqn} &&
- cd ${nvme_subsysnqn} &&
- cd namespaces &&
- mkdir "${namespace[0]}" &&
- cd "${namespace[0]}" &&
- echo 00000000-0000-0000-0000-000000000000 >device_nguid &&
- echo -n /dev/nullb0 >device_path &&
- echo 1 >enable &&
- cd ../.. &&
- echo 1 >attr_allow_any_host
- ) && for i in $(rdma_network_interfaces); do
- ipv4_addr=$(get_ipv4_addr "$i")
- if [ -n "${ipv4_addr}" ]; then
- configure_nvmet_port "$i" "${ipv4_addr}"
- ((num_ports++))
- true
- fi
- done &&
- if [[ $num_ports = 0 ]]; then
- echo "No NVMeOF target ports"
- false
- fi && (
- cd /sys/kernel/config/nvmet/ports &&
- for i in *; do
- [ -e "$i" ] && (
- cd "$i/subsystems" &&
- ln -s "../../../subsystems/${nvme_subsysnqn}" .
- )
- done
- )
- echo "Configured NVMe target driver"
-}
-
-stop_nvme_target() {
- local d
-
- (
- cd /sys/kernel/config/nvmet 2>/dev/null &&
- rm -f -- ports/*/subsystems/* &&
- for d in {*/*/*/*,*/*}; do
- [ -e "$d" ] &&
- [[ ! "$(basename "$(dirname "$d")")" =~ ana_groups|passthru ]] &&
- rmdir "$d"
- done
- )
- _unload_module nvmet_rdma &&
- _unload_module nvmet &&
- _exit_null_blk
-}
-
-start_target() {
- start_nvme_target
-}
-
-stop_target() {
- stop_nvme_target
-}
-
-shutdown_client() {
- remove_mpath_devs &&
- log_out &&
- stop_client
-}
-
-# Set up test configuration
-setup() {
- setup_test "$PWD/tests/nvmeof-mp/multipath.conf"
-}
projects / users / sagi / blktests.git / commitdiff