From: Nathan Scott <nathans@sgi.com>
Date: Wed, 18 Sep 2002 03:28:20 +0000 (+0000)
Subject: First cut at a framework for running benchmarks along with all of the
X-Git-Tag: v1.1.0~1047
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=d4d5d06c1ee7d0d8506c7c47a5737d25a1f23bbf;p=users%2Fhch%2Fxfstests-dev.git

First cut at a framework for running benchmarks along with all of the
functional tests we have.  Aim is to get wind of performance regressions
as changes are checked in, rather than at some arbitrary point down the
track.  Still a bit of work to do wrt this framework though.
---

diff --git a/bench b/bench
new file mode 100755
index 000000000..71ebca83a
--- /dev/null
+++ b/bench
@@ -0,0 +1,156 @@
+#!/bin/sh
+# 
+# Wrapper for automating benchmarking runs.
+# Usage:   bench [passes] [uid] [gid] [script]
+# 
+# ..where passes is the number of times to run each script; uid/gid
+# gives credentials to use when running the script; and script is a
+# simple wrapper around each actual benchmark tool (eg. see run.*),
+# if this is ommited, all run.* scripts are used in turn.
+# 
+# Each run.foo script should report a comma-separated-value list of
+# benchmark results on stdout or fail with a non-zero exit code;
+# unless the -i option is supplied in which case it should instead
+# report a comma-separated-value list of column headers (for report
+# generation purposes).
+# 
+#-----------------------------------------------------------------------
+# Copyright (c) 2002 Silicon Graphics, Inc.  All Rights Reserved.
+# 
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+# 
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# 
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston MA 02111-1307, USA.
+# 
+# Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
+# Mountain View, CA  94043, or:
+# 
+# http://www.sgi.com 
+# 
+# For further information regarding this notice, see: 
+# 
+# http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
+#-----------------------------------------------------------------------
+#
+# creator
+owner=nathans@sgi.com
+
+tmp=/tmp/$$
+seq=bench
+here=`pwd`
+status=1	# failure is the default!
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_cleanup()
+{
+    echo "        *** umount"
+    umount $SCRATCH_DEV >/dev/null 2>&1
+}
+
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+ROOT="."
+LOG="$ROOT/soak.log"
+FULL="$ROOT/soak.full"
+
+_log()
+{
+    echo "$*" 1>&2
+    echo "$*" >>$LOG
+    echo "$*" >>$FULL
+    sync
+}
+
+_logp()
+{
+    tee -a $FULL
+}
+
+_fail()
+{
+    _log "$*"
+    status=1
+    exit 1
+}
+
+_run_benchmark()
+{
+    pass=1
+    
+    while [ $pass -le $passes -o $passes -lt 0 ]
+    do
+        _log "        *** clean scratch device [starting pass $pass]"
+        mkfs_xfs -f $SCRATCH_DEV 2>&1 |  _fix_malloc >>$FULL \
+                        || _fail "            !!! failed to mkfs SCRATCH_DEV"
+
+        _log "        *** mounting scratch device"
+        mount -t xfs $SCRATCH_DEV $SCRATCH_MNT \
+                            || _fail "            !!! failed to mount"
+        
+        _log "        *** mkdir"
+        mkdir $SCRATCH_MNT/bench \
+                            || _fail "            !!! couldn't mkdir bench dir"
+        chown -R $user.$group $SCRATCH_MNT/bench \
+                            || _fail "            !!! couldn't chown bench dir"
+        cd $SCRATCH_MNT/bench
+
+        _log "        *** bench"
+        $here/src/runas -u $user -g $group $here/run.$bench > $FULL
+        [ $? -eq 0 ]        || _fail "            !!! non-zero $bench exit code"
+        cd $here
+
+        _log "        *** unmounting scratch device"
+        umount $SCRATCH_DEV 2>&1 | _logp \
+                            || _fail "            !!! failed to umount"
+
+        _log "        *** post-umount filesystem check"
+        _check_fs $SCRATCH_DEV
+        
+        let "pass = pass + 1"
+    done
+}
+
+# real QA test starts here
+
+_require_scratch
+
+passes=-1
+user=root
+group=root
+benches=`echo run.*`
+
+[ $# -gt 0 ] && passes=$1
+[ $# -gt 1 ] && user=$3
+[ $# -gt 2 ] && group=$4
+[ $# -gt 3 ] && benches=$2
+
+for bench in "$benches"
+do 
+    echo "" >$FULL
+    echo "" >$LOG
+    _log "*** benchmark started (passes=$passes, benchmark=$bench)"
+    _log "***     (`date`)"
+    _log "        *** unmounting scratch device"
+    umount $SCRATCH_DEV 2>&1 |  _fix_malloc >>$FULL
+    
+    _run_benchmark  # $passes $bench $user $group
+
+    _log "*** done $bench"
+done
diff --git a/common.config b/common.config
index 934f31681..269e6b0b0 100644
--- a/common.config
+++ b/common.config
@@ -53,35 +53,32 @@
 
 #
 # - This script is shared by the stress test system and the auto-qa
-#   system
+#   system (includes both regression test and benchmark components).
 # - TEST_DEV & TEST_DIR must be assigned.
 # - this script shouldn't make any assertions about filesystem
 #   validity or mountedness.
 #
 
-_readlink()
-{
-    if [ $# -ne 1 ]
-    then
-	echo "Usage: _readlink filename" 1>&2
-	exit 1
-    fi
+HOST=`hostname -s`
+MODULAR=0		# using XFS as a module or not
+BOOT="/boot"		# install target for kernels
+export EXTRA=${EXTRA:=-xfs-qa}
 
-    perl -e "\$in=\"$1\";" -e '
-	$lnk = readlink($in);
-	if ($lnk =~ m!^/.*!) {
-	  print "$lnk\n";
-	}
-	else {
-	  chomp($dir = `dirname $in`);
-	  print "$dir/$lnk\n";
-	}'
-}
+# general parameters (mainly for auto-qa)
+SOAK_PROC=3		# -p option to fsstress
+SOAK_STRESS=10000	# -n option to fsstress
+SOAK_PASSES=-1		# count of repetitions of fsstress (while soaking)
+BENCH_PASSES=5		# count of repetitions of benchmarks (want averages)
+EMAIL=root@localhost	# where auto-qa will send its status messages
+export MKFS_OPTIONS=${MKFS_OPTIONS:=-bsize=4096}
+export MOUNT_OPTIONS=${MOUNT_OPTIONS:=-ologbufs=2}
+export MALLOCLIB=${MALLOCLIB:=/usr/lib/libefence.a}
 
-
-case `hostname -s`
+case "$HOST"
 in
     bruce)
+	MODULAR=0
+	EMAIL="nathans@larry"
         TEST_DEV=/dev/sda10
 	TEST_DIR=/mnt/xfs1
 	SCRATCH_DEV=/dev/sda9
@@ -99,6 +96,8 @@ in
 	SCRATCH_MNT=/mnt/xfs0
         ;;
     sagan)
+	MODULAR=1
+	EMAIL="tes@larry"
         TEST_DEV=/dev/sda6
 	TEST_DIR=/mnt/xfs0
 	SCRATCH_DEV=/dev/sda7
@@ -109,12 +108,16 @@ in
 	RMT_TAPE_USER=guest
 	;;
     frodo)
+	MODULAR=0
+	EMAIL="nathans@larry"
         TEST_DEV=/dev/hda6
 	TEST_DIR=/mnt/test
         SCRATCH_DEV=/dev/hda7
 	SCRATCH_MNT=/mnt/scratch
 	;;
     troppo)
+	MODULAR=0
+	EMAIL="kaos@larry"
 	TEST_DEV=/dev/hdb13
 	TEST_DIR=/mnt/test
 	SCRATCH_DEV=/dev/hdb14
@@ -186,7 +189,7 @@ in
 	SCRATCH_MNT=/mnt/sda2
 	SCRATCH_LOGDEV=/dev/sda3
 	;;
-   dmfnt2)
+    dmfnt2)
         TEST_DEV=/dev/sda6
 	TEST_DIR=/mnt/test
 	SCRATCH_DEV=/dev/sdc6
@@ -197,7 +200,7 @@ in
 	RMT_TAPE_USER=guest
 	;;
     *)
-	echo "common.config: Error: need to define parameters for host `hostname -s`"
+	echo "common.config: Error: need to define parameters for host $HOST"
 	exit 1
 	;;
 esac
@@ -226,6 +229,25 @@ then
     exit 1
 fi
 
+_readlink()
+{
+    if [ $# -ne 1 ]
+    then
+	echo "Usage: _readlink filename" 1>&2
+	exit 1
+    fi
+
+    perl -e "\$in=\"$1\";" -e '
+	$lnk = readlink($in);
+	if ($lnk =~ m!^/.*!) {
+	  print "$lnk\n";
+	}
+	else {
+	  chomp($dir = `dirname $in`);
+	  print "$dir/$lnk\n";
+	}'
+}
+
 # if devfs is running expand the full /dev/.. pathname - this is what will be
 # returned by utilities such as mount
 [ -L "$TEST_DEV" ] && TEST_DEV=`_readlink $TEST_DEV`
diff --git a/run.dbench b/run.dbench
new file mode 100755
index 000000000..c0f2dd3ca
--- /dev/null
+++ b/run.dbench
@@ -0,0 +1,24 @@
+#!/bin/sh
+# 
+# Does a dbench run (10 clients if $DBENCH_CLIENTS is not set),
+# then massages the output into CSV format.
+# 
+DBENCH_CLIENTS=${DBENCH_CLIENTS:=10}
+
+run_dbench()
+{
+	mkdir ./dbench || exit 1
+	dbench $DBENCH_CLIENTS || exit 1
+	rm -fr ./dbench || exit 1
+}
+
+# dbench gives:
+# "Throughput 40.6701 MB/sec (NB=50.8376 MB/sec  406.701 MBit/sec)"
+# 
+if [ $# -gt 0 ]; then
+	echo "clients,MB/sec"
+	exit 0
+fi
+run_dbench | perl -ne \
+	'if (m/^Throughput (\S+) /) { print '$DBENCH_CLIENTS',",",$1,"\n"; }'
+
diff --git a/run.tar b/run.tar
new file mode 100755
index 000000000..c22676fe3
--- /dev/null
+++ b/run.tar
@@ -0,0 +1,35 @@
+#!/bin/sh -x
+# 
+# Produces a .tar file (if one doesn't exist as $TARFILE), then
+# times how long it takes to untar it onto the current directory.
+# 
+TARFILE=${TARFILE:=/var/tmp/bench.tar}
+
+barf()
+{
+	echo $@ >2
+	exit 1
+}
+
+new_tar()
+{
+	source="/bin /sbin /lib /dev"
+	if [ ! -f $1 ]; then
+		tar cf $1 $source || barf "tar c failed"
+	fi
+}
+
+run_tar()
+{
+	# %U=user %S=system %E=elapsed
+	mkdir ./tar || exit 1
+	/usr/bin/time -f '%U,%S,%E' tar xf $1 ./tar || exit 1
+	rm -fr ./tar || exit 1
+}
+
+if [ $# -gt 0 ]; then
+	echo "user,system,elapsed"
+	exit 0
+fi
+new_tar $TARFILE
+run_tar $TARFILE
diff --git a/tools/auto-qa b/tools/auto-qa
index 3be484961..92ca38f24 100755
--- a/tools/auto-qa
+++ b/tools/auto-qa
@@ -35,19 +35,11 @@
 
 # configuration (you could tune this)
 
-BOOT="/boot"
-SOAK_PASSES="-1"
-SOAK_STRESS="10000"
-SOAK_PROC="3"
-export EXTRA=${EXTRA:=-xfs-qa}
-export MKFS_OPTIONS=${MKFS_OPTIONS:=-bsize=4096}
-export MOUNT_OPTIONS=${MOUNT_OPTIONS:=-ologbufs=2}
-export MALLOCLIB=${MALLOCLIB:=/usr/lib/libefence.a}
-
 _log()
 {
     echo "$*" >&2
     echo "$*" >> $LOG
+    sync
 }
 
 _fail()
@@ -64,7 +56,7 @@ _fail()
     case $state
     in
 	cron*)
-	    mail -s "XFS QA status report" $ADMINEMAIL \
+	    mail -s "XFS QA status report" $EMAIL \
 		< $LOG 2>&1
 	;;
     esac
@@ -113,35 +105,6 @@ COMMON_CONFIG="$WORKAREA/cmd/xfstests/common.config"
 SH="/bin/sh"
 LOG="$ROOT/qa.log"
 
-# need to add auto-qa hosts here
-
-case $HOST
-in
-    bruce)
-	EMAIL="nathans@larry"
-	ADMINEMAIL="nathans@larry"
-	MODULAR=0
-	;;
-    sagan)
-	EMAIL="tes@larry"
-	ADMINEMAIL="tes@larry"
-	MODULAR=1
-	;;
-    troppo)
-	EMAIL="kaos@larry"
-	ADMINEMAIL="kaos@larry"
-	MODULAR=0
-	;;
-    frodo)
-	EMAIL="nathans@larry"
-	ADMINEMAIL="nathans@larry"
-	MODULAR=0
-	;;
-    *)
-	_fail "auto-qa: no configuration information for host '$HOST'"
-	;;
-esac
-
 # do some cleanup on exit
 
 _cleanup()
@@ -427,7 +390,7 @@ do
 	    
 	*restart)
 	    _log "	    *** select qa kernel"
-	    _sudo /sbin/lilo -R linux-xfs-qa 2>&1 \
+	    _sudo /sbin/lilo -R linux-xfs-qa $KERNEL_OPTIONS 2>&1 \
 			|| _fail "	    !!! lilo failed"
 	    
 	    _log "	    *** prepare to restart"
@@ -510,12 +473,21 @@ do
 	    new_state="run"
 	    ;;
 	    
-	    
 	soak-run)
 	    cd $QADIR
 	    
 	    _log "	    *** run soak test"
-	    _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC\
+	    _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC \
+			|| _fail "	    !!! failed to run soak test"
+
+	    new_state="done"
+	    ;;
+	    
+	bench-run)
+	    cd $QADIR
+	    
+	    _log "	    *** run benchmarks"
+	    _sudo ./bench $BENCH_PASSES `id -u && id -g` \
 			|| _fail "	    !!! failed to run soak test"
 
 	    new_state="done"