From c4d674bcd342cd8ccc9b5870031a025a633e3da4 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 Mar 2024 17:11:44 +0000 Subject: [PATCH] btrfs/06[0-9]..07[0-4]: kill all background tasks when test is killed/interrupted Test cases btrfs/06[0-9] and btrfs/07[0-4] exercise multiple concurrent operations while fsstress is running in parallel, and all these are left as child processes running in the background, which are correctly stopped if the tests are not interrupted/killed. However if any of these tests is interrupted/killed, it often leaves child processes still running in the background, which prevent further running fstests again. For example: $ /check -g auto (...) btrfs/060 394s ... 264s btrfs/061 83s ... 69s btrfs/062 109s ... 105s btrfs/063 52s ... 67s btrfs/064 53s ... 51s btrfs/065 88s ... 271s btrfs/066 127s ... 241s btrfs/067 435s ... 248s btrfs/068 161s ... ^C^C ^C $ ./check btrfs/068 FSTYP -- btrfs PLATFORM -- Linux/x86_64 debian0 6.8.0-rc7-btrfs-next-153+ #1 SMP PREEMPT_DYNAMIC Mon Mar 4 17:19:19 WET 2024 MKFS_OPTIONS -- /dev/sdb MOUNT_OPTIONS -- /dev/sdb /home/fdmanana/btrfs-tests/scratch_1 our local _scratch_mkfs routine ... btrfs-progs v6.6.2 See https://btrfs.readthedocs.io for more information. ERROR: unable to open /dev/sdb: Device or resource busy check: failed to mkfs $SCRATCH_DEV using specified options Interrupted! Passed all 0 tests In this case there was still a process running _btrfs_stress_subvolume() from common/btrfs. This is a bit annoying because it requires manually finding out which process is preventing unmounting the scratch device and then properly stop/kill it. So fix this by adding a _cleanup() function to all these tests and then making sure it stops all the child processes it spawned and are running in the background. All these tests have the same structure as they were part of the same patchset and from the same author. Reviewed-by: Anand Jain Signed-off-by: Filipe Manana Signed-off-by: Anand Jain --- tests/btrfs/060 | 22 +++++++++++++++++++++- tests/btrfs/061 | 19 +++++++++++++++++++ tests/btrfs/062 | 19 +++++++++++++++++++ tests/btrfs/063 | 19 +++++++++++++++++++ tests/btrfs/064 | 19 +++++++++++++++++++ tests/btrfs/065 | 22 +++++++++++++++++++++- tests/btrfs/066 | 22 +++++++++++++++++++++- tests/btrfs/067 | 22 +++++++++++++++++++++- tests/btrfs/068 | 22 +++++++++++++++++++++- tests/btrfs/069 | 19 +++++++++++++++++++ tests/btrfs/070 | 19 +++++++++++++++++++ tests/btrfs/071 | 19 +++++++++++++++++++ tests/btrfs/072 | 19 +++++++++++++++++++ tests/btrfs/073 | 19 +++++++++++++++++++ tests/btrfs/074 | 19 +++++++++++++++++++ 15 files changed, 295 insertions(+), 5 deletions(-) diff --git a/tests/btrfs/060 b/tests/btrfs/060 index 53cbd3a0d..f74d95934 100755 --- a/tests/btrfs/060 +++ b/tests/btrfs/060 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto balance subvol scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$stop_file" ] && [ ! -z "$subvol_pid" ]; then + _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + fi + if [ ! -z "$balance_pid" ]; then + _btrfs_kill_stress_balance_pid $balance_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -20,11 +36,12 @@ _require_scratch_nocheck _require_scratch_dev_pool 4 _btrfs_get_profile_configs +stop_file=$TEST_DIR/$seq.stop.$$ + run_test() { local mkfs_opts=$1 local subvol_mnt=$TEST_DIR/$seq.mnt - local stop_file=$TEST_DIR/$seq.stop.$$ echo "Test $mkfs_opts" >>$seqres.full @@ -53,9 +70,12 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + unset subvol_pid _btrfs_kill_stress_balance_pid $balance_pid + unset balance_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/061 b/tests/btrfs/061 index b8b2706c6..fec908821 100755 --- a/tests/btrfs/061 +++ b/tests/btrfs/061 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto balance scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$balance_pid" ]; then + _btrfs_kill_stress_balance_pid $balance_pid + fi + if [ ! -z "$scrub_pid" ]; then + _btrfs_kill_stress_scrub_pid $scrub_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -51,8 +67,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_balance_pid $balance_pid + unset balance_pid _btrfs_kill_stress_scrub_pid $scrub_pid + unset scrub_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/062 b/tests/btrfs/062 index 59d581bea..0b57681f9 100755 --- a/tests/btrfs/062 +++ b/tests/btrfs/062 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto balance defrag compress scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$balance_pid" ]; then + _btrfs_kill_stress_balance_pid $balance_pid + fi + if [ ! -z "$defrag_pid" ]; then + _btrfs_kill_stress_defrag_pid $defrag_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -52,8 +68,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_balance_pid $balance_pid + unset balance_pid _btrfs_kill_stress_defrag_pid $defrag_pid + unset defrag_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/063 b/tests/btrfs/063 index 5ee2837f8..99d9d2c1e 100755 --- a/tests/btrfs/063 +++ b/tests/btrfs/063 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto balance remount compress scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$balance_pid" ]; then + _btrfs_kill_stress_balance_pid $balance_pid + fi + if [ ! -z "$remount_pid" ]; then + _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -51,8 +67,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_balance_pid $balance_pid + unset balance_pid _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + unset remount_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/064 b/tests/btrfs/064 index 9e0b3b302..663442c69 100755 --- a/tests/btrfs/064 +++ b/tests/btrfs/064 @@ -12,6 +12,22 @@ . ./common/preamble _begin_fstest auto balance replace volume scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$balance_pid" ]; then + _btrfs_kill_stress_balance_pid $balance_pid + fi + if [ ! -z "$replace_pid" ]; then + _btrfs_kill_stress_replace_pid $replace_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -63,8 +79,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_balance_pid $balance_pid + unset balance_pid _btrfs_kill_stress_replace_pid $replace_pid + unset replace_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/065 b/tests/btrfs/065 index f9e43cdc0..b1e54fc8a 100755 --- a/tests/btrfs/065 +++ b/tests/btrfs/065 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto subvol replace volume scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$stop_file" ] && [ ! -z "$subvol_pid" ]; then + _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + fi + if [ ! -z "$replace_pid" ]; then + _btrfs_kill_stress_replace_pid $replace_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -21,12 +37,13 @@ _require_scratch_dev_pool 5 _require_scratch_dev_pool_equal_size _btrfs_get_profile_configs replace +stop_file=$TEST_DIR/$seq.stop.$$ + run_test() { local mkfs_opts=$1 local saved_scratch_dev_pool=$SCRATCH_DEV_POOL local subvol_mnt=$TEST_DIR/$seq.mnt - local stop_file=$TEST_DIR/$seq.stop.$$ echo "Test $mkfs_opts" >>$seqres.full @@ -61,9 +78,12 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + unset subvol_pid _btrfs_kill_stress_replace_pid $replace_pid + unset replace_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/066 b/tests/btrfs/066 index b6f904ac5..feb6062e3 100755 --- a/tests/btrfs/066 +++ b/tests/btrfs/066 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto subvol scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$stop_file" ] && [ ! -z "$subvol_pid" ]; then + _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + fi + if [ ! -z "$scrub_pid" ]; then + _btrfs_kill_stress_scrub_pid $scrub_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -20,11 +36,12 @@ _require_scratch_nocheck _require_scratch_dev_pool 4 _btrfs_get_profile_configs +stop_file=$TEST_DIR/$seq.stop.$$ + run_test() { local mkfs_opts=$1 local subvol_mnt=$TEST_DIR/$seq.mnt - local stop_file=$TEST_DIR/$seq.stop.$$ echo "Test $mkfs_opts" >>$seqres.full @@ -53,9 +70,12 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + unset subvol_pid _btrfs_kill_stress_scrub_pid $scrub_pid + unset scrub_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/067 b/tests/btrfs/067 index 7be09d52f..0bbfe83f0 100755 --- a/tests/btrfs/067 +++ b/tests/btrfs/067 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto subvol defrag compress scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$stop_file" ] && [ ! -z "$subvol_pid" ]; then + _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + fi + if [ ! -z "$defrag_pid" ]; then + _btrfs_kill_stress_defrag_pid $defrag_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -20,12 +36,13 @@ _require_scratch_nocheck _require_scratch_dev_pool 4 _btrfs_get_profile_configs +stop_file=$TEST_DIR/$seq.stop.$$ + run_test() { local mkfs_opts=$1 local with_compress=$2 local subvol_mnt=$TEST_DIR/$seq.mnt - local stop_file=$TEST_DIR/$seq.stop.$$ echo "Test $mkfs_opts with $with_compress" >>$seqres.full @@ -54,9 +71,12 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + unset subvol_pid _btrfs_kill_stress_defrag_pid $defrag_pid + unset defrag_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/068 b/tests/btrfs/068 index 19e370102..7ab6fecac 100755 --- a/tests/btrfs/068 +++ b/tests/btrfs/068 @@ -11,6 +11,22 @@ . ./common/preamble _begin_fstest auto subvol remount compress scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$stop_file" ] && [ ! -z "$subvol_pid" ]; then + _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + fi + if [ ! -z "$remount_pid" ]; then + _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -21,11 +37,12 @@ _require_scratch_nocheck _require_scratch_dev_pool 4 _btrfs_get_profile_configs +stop_file=$TEST_DIR/$seq.stop.$$ + run_test() { local mkfs_opts=$1 local subvol_mnt=$TEST_DIR/$seq.mnt - local stop_file=$TEST_DIR/$seq.stop.$$ echo "Test $mkfs_opts with $with_compress" >>$seqres.full @@ -54,9 +71,12 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_subvolume_pid $subvol_pid $stop_file + unset subvol_pid _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + unset remount_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/069 b/tests/btrfs/069 index ad1609d40..3fbfecdbb 100755 --- a/tests/btrfs/069 +++ b/tests/btrfs/069 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto replace scrub volume +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$replace_pid" ]; then + _btrfs_kill_stress_replace_pid $replace_pid + fi + if [ ! -z "$scrub_pid" ]; then + _btrfs_kill_stress_scrub_pid $scrub_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -59,8 +75,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_scrub_pid $scrub_pid + unset scrub_pid _btrfs_kill_stress_replace_pid $replace_pid + unset replace_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/070 b/tests/btrfs/070 index 3054c270b..11fddc862 100755 --- a/tests/btrfs/070 +++ b/tests/btrfs/070 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto replace defrag compress volume scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$replace_pid" ]; then + _btrfs_kill_stress_replace_pid $replace_pid + fi + if [ ! -z "$defrag_pid" ]; then + _btrfs_kill_stress_defrag_pid $defrag_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -60,8 +76,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_replace_pid $replace_pid + unset replace_pid _btrfs_kill_stress_defrag_pid $defrag_pid + unset defrag_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/071 b/tests/btrfs/071 index 36b393412..1a91ec450 100755 --- a/tests/btrfs/071 +++ b/tests/btrfs/071 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto replace remount compress volume scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$replace_pid" ]; then + _btrfs_kill_stress_replace_pid $replace_pid + fi + if [ ! -z "$remount_pid" ]; then + _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -59,8 +75,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_replace_pid $replace_pid + unset replace_pid _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + unset remount_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/072 b/tests/btrfs/072 index 505d0b578..e66e49c9c 100755 --- a/tests/btrfs/072 +++ b/tests/btrfs/072 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto scrub defrag compress +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$defrag_pid" ]; then + _btrfs_kill_stress_defrag_pid $defrag_pid + fi + if [ ! -z "$scrub_pid" ]; then + _btrfs_kill_stress_scrub_pid $scrub_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -52,9 +68,12 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_defrag_pid $defrag_pid + unset defrag_pid _btrfs_kill_stress_scrub_pid $scrub_pid + unset scrub_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/073 b/tests/btrfs/073 index 503582862..e6cfd92a8 100755 --- a/tests/btrfs/073 +++ b/tests/btrfs/073 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto scrub remount compress +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$remount_pid" ]; then + _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + fi + if [ ! -z "$scrub_pid" ]; then + _btrfs_kill_stress_scrub_pid $scrub_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -51,8 +67,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + unset remount_pid _btrfs_kill_stress_scrub_pid $scrub_pid + unset scrub_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 diff --git a/tests/btrfs/074 b/tests/btrfs/074 index 6e93b36a9..1dd88bcd6 100755 --- a/tests/btrfs/074 +++ b/tests/btrfs/074 @@ -10,6 +10,22 @@ . ./common/preamble _begin_fstest auto defrag remount compress scrub +_cleanup() +{ + cd / + rm -rf $tmp.* + if [ ! -z "$remount_pid" ]; then + _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + fi + if [ ! -z "$defrag_pid" ]; then + _btrfs_kill_stress_defrag_pid $defrag_pid + fi + if [ ! -z "$fsstress_pid" ]; then + kill $fsstress_pid &> /dev/null + wait $fsstress_pid &> /dev/null + fi +} + # Import common functions. . ./common/filter @@ -52,8 +68,11 @@ run_test() echo "Wait for fsstress to exit and kill all background workers" >>$seqres.full wait $fsstress_pid + unset fsstress_pid _btrfs_kill_stress_remount_compress_pid $remount_pid $SCRATCH_MNT + unset remount_pid _btrfs_kill_stress_defrag_pid $defrag_pid + unset defrag_pid echo "Scrub the filesystem" >>$seqres.full $BTRFS_UTIL_PROG scrub start -B $SCRATCH_MNT >>$seqres.full 2>&1 -- 2.50.1