xfs/{422,517}: kill background jobs on test termination

Those tests failed to cleanup background jobs properly after test
is interrupted and even sometimes when it completed succefully.

xfs/517 would sometime fails randomally with this false positive error:
     QA output created by 517
     Format and populate
     Concurrent fsmap and freeze
    +Terminated
     Test done

The tests have several background sub-shells that spawn short lived
programs in a loop.  By killing the spawned programs using killall,
killall could find no process to kill and the sub-shell loop could still
spawn another process that is not going to be killed and in the worst
case, the freeze_loop() could spawn the xfs_io "freeze" command after
test has thawn the fs before exit, which leaves the fs frozen after the
test.

The "Terminated" output is emitted by the sub-shell when killing the
programs that it has spawned when the loop did not finish before test
timeout.  By killing the sub-shell and not the spawned programs, we
avoid the false positive "Terminated" error.

Use a helper to perform this cleanup dance:
First kill and wait the freeze_loop so it won't try to freeze fs again
Then make sure fs is not frozen.
Then kill and wait for the rest of the sub-shells, because
if fs is frozen a killed writer process will never exit.

Signed-off-by: Amir Goldstein <amir73il@gmail.com>
Signed-off-by: Zorro Lang <zlang@kernel.org>

diff --git a/tests/xfs/422 b/tests/xfs/422
index a83a66df80d189006df8a952c3700c4e6be10026..fdbb8bf14c899ffae4ec6e8a43a32a54d102510b 100755 (executable)
--- a/tests/xfs/422
+++ b/tests/xfs/422
@@ -13,6 +13,32 @@ _begin_fstest dangerous_scrub dangerous_online_repair freeze
 
 _register_cleanup "_cleanup" BUS
 
+# First kill and wait the freeze loop so it won't try to freeze fs again
+# Then make sure fs is not frozen
+# Then kill and wait for the rest of the workers
+# Because if fs is frozen a killed writer will never exit
+kill_loops() {
+       local sig=$1
+
+       [ -n "$freeze_pid" ] && kill $sig $freeze_pid
+       wait $freeze_pid
+       unset freeze_pid
+       $XFS_IO_PROG -x -c 'thaw' $SCRATCH_MNT
+       [ -n "$stress_pid" ] && kill $sig $stress_pid
+       [ -n "$repair_pid" ] && kill $sig $repair_pid
+       wait
+       unset stress_pid
+       unset repair_pid
+}
+
+# Override the default cleanup function.
+_cleanup()
+{
+       kill_loops -9 > /dev/null 2>&1
+       cd /
+       rm -rf $tmp.*
+}
+
 # Import common functions.
 . ./common/filter
 . ./common/fuzzy
@@ -78,8 +104,11 @@ end=$((start + (30 * TIME_FACTOR) ))
 
 echo "Loop started at $(date --date="@${start}"), ending at $(date --date="@${end}")" >> $seqres.full
 stress_loop $end &
+stress_pid=$!
 freeze_loop $end &
+freeze_pid=$!
 repair_loop $end &
+repair_pid=$!
 
 # Wait until 2 seconds after the loops should have finished...
 while [ "$(date +%s)" -lt $((end + 2)) ]; do
@@ -87,8 +116,7 @@ while [ "$(date +%s)" -lt $((end + 2)) ]; do
 done
 
 # ...and clean up after the loops in case they didn't do it themselves.
-$KILLALL_PROG -TERM xfs_io fsstress >> $seqres.full 2>&1
-$XFS_IO_PROG -x -c 'thaw' $SCRATCH_MNT >> $seqres.full 2>&1
+kill_loops >> $seqres.full 2>&1
 
 echo "Loop finished at $(date)" >> $seqres.full
 echo "Test done"

diff --git a/tests/xfs/517 b/tests/xfs/517
index f7f9a8a276d1830754ddd4e4d6ef77071f5582b2..6877af13bb4e238840e7ee4f6a99b27d74545c35 100755 (executable)
--- a/tests/xfs/517
+++ b/tests/xfs/517
@@ -11,11 +11,29 @@ _begin_fstest auto quick fsmap freeze
 
 _register_cleanup "_cleanup" BUS
 
+# First kill and wait the freeze loop so it won't try to freeze fs again
+# Then make sure fs is not frozen
+# Then kill and wait for the rest of the workers
+# Because if fs is frozen a killed writer will never exit
+kill_loops() {
+       local sig=$1
+
+       [ -n "$freeze_pid" ] && kill $sig $freeze_pid
+       wait $freeze_pid
+       unset freeze_pid
+       $XFS_IO_PROG -x -c 'thaw' $SCRATCH_MNT
+       [ -n "$stress_pid" ] && kill $sig $stress_pid
+       [ -n "$fsmap_pid" ] && kill $sig $fsmap_pid
+       wait
+       unset stress_pid
+       unset fsmap_pid
+}
+
 # Override the default cleanup function.
 _cleanup()
 {
+       kill_loops -9 > /dev/null 2>&1
        cd /
-       $XFS_IO_PROG -x -c 'thaw' $SCRATCH_MNT > /dev/null 2>&1
        rm -rf $tmp.*
 }
 
@@ -83,8 +101,11 @@ end=$((start + (30 * TIME_FACTOR) ))
 
 echo "Loop started at $(date --date="@${start}"), ending at $(date --date="@${end}")" >> $seqres.full
 stress_loop $end &
+stress_pid=$!
 freeze_loop $end &
+freeze_pid=$!
 fsmap_loop $end &
+fsmap_pid=$!
 
 # Wait until 2 seconds after the loops should have finished...
 while [ "$(date +%s)" -lt $((end + 2)) ]; do
@@ -92,8 +113,7 @@ while [ "$(date +%s)" -lt $((end + 2)) ]; do
 done
 
 # ...and clean up after the loops in case they didn't do it themselves.
-$KILLALL_PROG -TERM xfs_io fsstress >> $seqres.full 2>&1
-$XFS_IO_PROG -x -c 'thaw' $SCRATCH_MNT >> $seqres.full 2>&1
+kill_loops >> $seqres.full 2>&1
 
 echo "Loop finished at $(date)" >> $seqres.full
 echo "Test done"