check: support test case repeat by different conditions

It is often required to run the same test with slightly different test
conditions. If we create each test case for each test condition, those
test cases are almost same and have code duplication. Such duplication
is seen in many of the nvme test cases that set up nvme transport.

To avoid the code duplication, introduce a new feature to support test
case repetition with different conditions. When a test case implements
the function set_conditions(), blktests repeat the test case. When
set_conditions() is called without an argument, it returns how many
times the test case is to be repeated. Before each test case run,
blktests calls set_conditions() with an argument number from 0 to the
number of repetitions minus 1. set_conditions() sets up the condition
for each test run referring to the argument as the index of the
condition to set up. set_conditions() also sets up a short string in
the COND_DESC variable. This string is printed to stdout to identify the
condition of each run. It is also used as the directory path name to
hold result files.

Document the usage of set_conditions() in the new script. Separate out
shellcheck command line for the new script to avoid a false-positive
warning unique to the file.

Reviewed-by: Daniel Wagner <dwagner@suse.de>
Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com>
Acked-by: Nitesh Shetty <nj.shetty@samsung.com>
Signed-off-by: Shin'ichiro Kawasaki <shinichiro.kawasaki@wdc.com>

diff --git a/Makefile b/Makefile
index 43f2ab0b4f370ec8d1c6ce9d752154953dbfd8a5..1c685fe0d599cec7c8a74578e7e858d8113125c2 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -18,8 +18,9 @@ install:
 SHELLCHECK_EXCLUDE := SC2119
 
 check:
-       shellcheck -x -e $(SHELLCHECK_EXCLUDE) -f gcc check new common/* \
+       shellcheck -x -e $(SHELLCHECK_EXCLUDE) -f gcc check common/* \
                tests/*/rc tests/*/[0-9]*[0-9] src/*.sh
+       shellcheck --exclude=$(SHELLCHECK_EXCLUDE),SC2154 --format=gcc new
        ! grep TODO tests/*/rc tests/*/[0-9]*[0-9]
        ! find -name '*.out' -perm /u=x+g=x+o=x -printf '%p is executable\n' | grep .
 

diff --git a/check b/check
index 7d09ec0140300c50ada4fc4706a0cfe14d461c45..edc421d6cc374c0447583542948f388dc02e73a6 100755 (executable)
--- a/check
+++ b/check
@@ -17,7 +17,9 @@ _found_test() {
        local test_name="$1"
        local explicit="$2"
 
-       unset CAN_BE_ZONED DESCRIPTION QUICK TIMED requires device_requires test test_device fallback_device cleanup_fallback_device
+       unset CAN_BE_ZONED DESCRIPTION QUICK TIMED requires device_requires \
+             test test_device fallback_device cleanup_fallback_device \
+             set_conditions
 
        # shellcheck disable=SC1090
        if ! . "tests/${test_name}"; then
@@ -190,15 +192,12 @@ _write_test_run() {
 _output_status() {
        local test="$1"
        local status="$2"
-       local zoned=" "
+       local str="${test} "
 
-       if (( RUN_FOR_ZONED )); then zoned=" (zoned) "; fi
-
-       if [[ "${DESCRIPTION:-}" ]]; then
-               printf '%-60s' "${test}${zoned}($DESCRIPTION)"
-       else
-               printf '%-60s' "${test}${zoned}"
-       fi
+       (( RUN_FOR_ZONED )) && str="$str(zoned) "
+       [[ ${COND_DESC:-} ]] && str="$str(${COND_DESC}) "
+       [[ ${DESCRIPTION:-} ]] && str="$str(${DESCRIPTION})"
+       printf '%-60s' "${str}"
        if [[ -z $status ]]; then
                echo
                return
@@ -464,17 +463,19 @@ _unload_modules() {
 }
 
 _check_and_call_test() {
+       local postfix
        local ret
 
        if declare -fF requires >/dev/null; then
                requires
        fi
 
-       RESULTS_DIR="$OUTPUT/nodev"
+       [[ -n $COND_DESC ]] && postfix=_${COND_DESC//[ =]/_}
+       RESULTS_DIR="$OUTPUT/nodev${postfix}"
        _call_test test
        ret=$?
        if (( RUN_ZONED_TESTS && CAN_BE_ZONED )); then
-               RESULTS_DIR="$OUTPUT/nodev_zoned"
+               RESULTS_DIR="$OUTPUT/nodev_zoned${postfix}"
                RUN_FOR_ZONED=1
                _call_test test
                ret=$(( ret || $? ))
@@ -484,6 +485,7 @@ _check_and_call_test() {
 }
 
 _check_and_call_test_device() {
+       local postfix
        local unset_skip_reason
        local ret
 
@@ -491,6 +493,7 @@ _check_and_call_test_device() {
                requires
        fi
 
+       [[ -n $COND_DESC ]] && postfix=_${COND_DESC//[ =]/_}
        for TEST_DEV in "${TEST_DEVS[@]}"; do
                TEST_DEV_SYSFS="${TEST_DEV_SYSFS_DIRS["$TEST_DEV"]}"
                TEST_DEV_PART_SYSFS="${TEST_DEV_PART_SYSFS_DIRS["$TEST_DEV"]}"
@@ -504,7 +507,7 @@ _check_and_call_test_device() {
                                device_requires
                        fi
                fi
-               RESULTS_DIR="$OUTPUT/$(basename "$TEST_DEV")"
+               RESULTS_DIR="$OUTPUT/$(basename "$TEST_DEV")""$postfix"
                if ! _call_test test_device; then
                        ret=1
                fi
@@ -522,9 +525,11 @@ _run_test() {
        CHECK_DMESG=1
        DMESG_FILTER="cat"
        RUN_FOR_ZONED=0
+       COND_DESC=""
        FALLBACK_DEVICE=0
        MODULES_TO_UNLOAD=()
 
+       local nr_conds cond_i
        local ret=0
 
        # Ensure job control monitor mode is off in the sub-shell for test case
@@ -535,8 +540,18 @@ _run_test() {
        . "tests/${TEST_NAME}"
 
        if declare -fF test >/dev/null; then
-               _check_and_call_test
-               ret=$?
+               if declare -fF set_conditions >/dev/null; then
+                       nr_conds=$(set_conditions)
+                       for ((cond_i = 0; cond_i < nr_conds; cond_i++)); do
+                               set_conditions $cond_i
+                               _check_and_call_test
+                               ret=$(( ret || $? ))
+                               unset SKIP_REASONS
+                       done
+               else
+                       _check_and_call_test
+                       ret=$?
+               fi
        else
                if [[ ${#TEST_DEVS[@]} -eq 0 ]] && \
                        declare -fF fallback_device >/dev/null; then
@@ -558,8 +573,18 @@ _run_test() {
                        return 0
                fi
 
-               _check_and_call_test_device
-               ret=$?
+               if declare -fF set_conditions >/dev/null; then
+                       nr_conds=$(set_conditions)
+                       for ((cond_i = 0; cond_i < nr_conds; cond_i++)); do
+                               set_conditions $cond_i
+                               _check_and_call_test_device
+                               ret=$(( ret || $? ))
+                               unset SKIP_REASONS
+                       done
+               else
+                       _check_and_call_test_device
+                       ret=$?
+               fi
 
                if (( FALLBACK_DEVICE )); then
                        cleanup_fallback_device

diff --git a/common/shellcheck b/common/shellcheck
index 8c324bd3113a1797bc9b2c38cd14698ed08052a4..ac0a51e5472b01c2f5689fef4febdaea3bcdcbe3 100644 (file)
--- a/common/shellcheck
+++ b/common/shellcheck
@@ -6,5 +6,5 @@
 
 # Suppress unused global variable warnings.
 _silence_sc2034() {
-       echo "$CAN_BE_ZONED $CGROUP2_DIR $CHECK_DMESG $DESCRIPTION $DMESG_FILTER $FIO_PERF_FIELDS $FIO_PERF_PREFIX $QUICK $SKIP_REASONS ${TEST_RUN[*]} $TIMED" > /dev/null
+       echo "$CAN_BE_ZONED $CGROUP2_DIR $CHECK_DMESG $COND_DESC $DESCRIPTION $DMESG_FILTER $FIO_PERF_FIELDS $FIO_PERF_PREFIX $QUICK $SKIP_REASONS ${TEST_RUN[*]} $TIMED" > /dev/null
 }

diff --git a/new b/new
index 574d8b43c055c64cdbfa6de7dff8731dc323a6ec..d84f01d62e302d3b6dc6496b6ad6d7db4207d2ee 100755 (executable)
--- a/new
+++ b/new
@@ -180,6 +180,27 @@ DESCRIPTION=""
 #      _require_test_dev_is_foo && _require_test_dev_supports_bar
 # }
 
+# TODO: if the test case can run the same test for different conditions, define
+# the helper function "set_conditions". When no argument is specified, return
+# the number of condition variations. Blktests repeats the test case as many
+# times as the returned number. When its argument is specified, refer to it as
+# the condition variation index and set up the conditions for it. Also set the
+# global variable COND_DESC which is printed at the test case run and used for
+# the result directory name. Blktests calls set_conditions() before each run of
+# the test case incrementing the argument index from 0.
+# set_conditions() {
+#      local index=\$1
+#
+#      if [[ -z \$index ]]; then
+#              echo 2  # return number of condition variations
+#              return
+#      fi
+#
+#      # Set test conditions based on the $index
+#      ...
+#      COND_DESC="Describe the conditions shortly"
+# }
+
 # TODO: define the test. The output of this function (stdout and stderr) will
 # be compared to tests/\${TEST_NAME}.out. If it does not match, the test is
 # considered a failure. If the test runs a command which has unnecessary