From 040e74c891922424519060095d1d322977cde50a Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 3 Feb 2025 14:00:30 -0800 Subject: [PATCH] fuzzy: don't use readarray for xfsfind output Some of the scrub stress tests (e.g. xfs/796) walk the directory tree to find filepaths to scrub, and load the entire list of paths into a bash array. On a large filesystem or a long-running test this is hugely wasteful of memory because we use each path exactly once. Fix __stress_one_scrub_loop to avoid this by reading lines directly from the output of the xfsfind utility. However, we play some games with fd 77 so that the processes in the loop body will use the same stdin as the test and /not/ the piped stdout of xfsfind. To avoid read(1) becoming confused by newlines in the file paths, adapt xfsfind to print nulls between pathnames, and the bash code to recognize them. This was a debugging patch while I was trying to figure out why xfs/286 and other scrub soak tests started OOMing after the v2024.12.08 changes, though in the end the OOMs were the result of memory leaks in fsstress. Signed-off-by: "Darrick J. Wong" Signed-off-by: Zorro Lang --- common/fuzzy | 22 +++++++++++++--------- src/xfsfind.c | 14 +++++++++++--- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/common/fuzzy b/common/fuzzy index 16ec71729..486dcb023 100644 --- a/common/fuzzy +++ b/common/fuzzy @@ -861,14 +861,14 @@ __stress_one_scrub_loop() { ;; esac - local target_cmd=(echo "$scrub_tgt") + local target_cmd=(echo -en "$scrub_tgt\0") case "$scrub_tgt" in - "%file%") target_cmd=($here/src/xfsfind -q "$SCRATCH_MNT");; - "%attrfile%") target_cmd=($here/src/xfsfind -qa "$SCRATCH_MNT");; - "%datafile%") target_cmd=($here/src/xfsfind -qb "$SCRATCH_MNT");; - "%dir%") target_cmd=($here/src/xfsfind -qd "$SCRATCH_MNT");; - "%regfile%") target_cmd=($here/src/xfsfind -qr "$SCRATCH_MNT");; - "%cowfile%") target_cmd=($here/src/xfsfind -qs "$SCRATCH_MNT");; + "%file%") target_cmd=($here/src/xfsfind -0q "$SCRATCH_MNT");; + "%attrfile%") target_cmd=($here/src/xfsfind -0qa "$SCRATCH_MNT");; + "%datafile%") target_cmd=($here/src/xfsfind -0qb "$SCRATCH_MNT");; + "%dir%") target_cmd=($here/src/xfsfind -0qd "$SCRATCH_MNT");; + "%regfile%") target_cmd=($here/src/xfsfind -0qr "$SCRATCH_MNT");; + "%cowfile%") target_cmd=($here/src/xfsfind -0qs "$SCRATCH_MNT");; esac while __stress_scrub_running "$scrub_startat" "$runningfile"; do @@ -876,12 +876,16 @@ __stress_one_scrub_loop() { done while __stress_scrub_running "$end" "$runningfile"; do - readarray -t fnames < <("${target_cmd[@]}" 2>> $seqres.full) - for fname in "${fnames[@]}"; do + # Attach the stdout of xfsfind to fd 77 so that we can read + # pathnames from that file descriptor without passing the pipe + # to the loop body as stdin. + exec 77< <("${target_cmd[@]}" 2>> $seqres.full) + while read -u 77 -d '' fname; do $XFS_IO_PROG -x "${xfs_io_args[@]}" "$fname" 2>&1 | \ __stress_scrub_filter_output "${extra_filters[@]}" __stress_scrub_running "$end" "$runningfile" || break done + exec 77<&- done } diff --git a/src/xfsfind.c b/src/xfsfind.c index c81deaf64..2043d01de 100644 --- a/src/xfsfind.c +++ b/src/xfsfind.c @@ -20,6 +20,7 @@ static int want_dir; static int want_regfile; static int want_sharedfile; static int report_errors = 1; +static int print0; static int check_datafile( @@ -115,6 +116,7 @@ print_help( printf("\n"); printf("Print all file paths matching any of the given predicates.\n"); printf("\n"); + printf("-0 Print nulls between paths instead of newlines.\n"); printf("-a Match files with xattrs.\n"); printf("-b Match files with data blocks.\n"); printf("-d Match directories.\n"); @@ -208,8 +210,13 @@ visit( out_fd: close(fd); out: - if (printme) - printf("%s\n", path); + if (printme) { + if (print0) + printf("%s%c", path, 0); + else + printf("%s\n", path); + fflush(stdout); + } return retval; } @@ -236,8 +243,9 @@ main( int c; int ret; - while ((c = getopt(argc, argv, "abdqrs")) >= 0) { + while ((c = getopt(argc, argv, "0abdqrs")) >= 0) { switch (c) { + case '0': print0 = 1; break; case 'a': want_attrfile = 1; break; case 'b': want_datafile = 1; break; case 'd': want_dir = 1; break; -- 2.49.0