====== xymon-fstest ======
^ Author | [[ jlaidman+xymon-fstest@rebel-it.com.au | Jeremy Laidman ]] |
^ Compatibility | Xymon 4.2+ |
^ Requirements | Bourne or Bash shell (or compatible), Xymon server |
^ Download | None |
^ Last Update | 2013-02-28 |
===== Description =====
This script checks filesystems and reports them in their own separate
tests (''fs_1'', ''fs_2'', ... or ''fs_root'', ''fs_var'', ...), by making use of the combined "''disk''" test.
Initially, only disks with non-green status will have the new "''fs_*''" pages created.
Once a page exists, it will be persistent, going green when the "disk" test is green.
If the "''disk''" test goes purple or blue, the "''fs_*''" tests will change to clear.
Filesystem mount points are used in the test name, where possible (eg "''/''" becomes "''fs_root''", "''/usr''" becomes "''fs_usr''") but names longer than four characters (excluding leading slash) are converted to an index number (eg "''/var/log''" becomes "''fs_2''" for the second filesystem).
Screenshot:
{{ :monitors:fs-test-screenshot.png? |}}
===== Installation =====
=== Client side ===
Ensure "disk" test is working. That is all.
=== Server side ===
Copy to server/ext/ directory, create entry in tasks.cfg or new file in tasks.d/ directory.
For troubleshooting, run with DEBUG and/or DRYRUN set to a positive integer.
===== Source =====
==== fs-test.sh ====
Copy this to the "server/ext" directory on your Xymon server.
#!/bin/sh
# Checks filesystems and reports them in their own separate
# tests (fs_1, fs_2, ...) as well as the one combined disk test.
# Todo:
# - allow specifying filesystems to include/exclude in hosts.cfg
# - allow specifying per-filesystem thresholds
die() { echo "$@" >&2; exit 1; }
[ "$XYMON" -a "$XYMSRV" ] || die "Run under Xymon environment"
do_usage() {
echo "Usage: $0 [-h|-d|-y|-m match]"
echo "-m match : match hostname"
echo "-h : help"
echo "-d : increase debug level by one (repeat for more)"
echo "-y : dry run"
}
map_fsname() {
# maps filesystem name to test name
# must try to keep it short (no more than 4 chars)
# TODO: allow mapping in hosts.cfg
NEWNAME=""
case $1 in
/) NEWNAME="root";;
/?) NEWNAME=`IFS=/; set - $1""; echo $2`;;
/??) NEWNAME=`IFS=/; set - $1""; echo $2`;;
/???) NEWNAME=`IFS=/; set - $1""; echo $2`;;
/????) NEWNAME=`IFS=/; set - $1""; echo $2`;;
esac
echo "$NEWNAME"
[ 0$DEBUG -ge 2 ] && echo "Changed '$1' to '$NEWNAME'" >&2
}
while [ "$1" ]; do
case $1 in
-h|--h*) do_usage; exit;;
-d|--debug) DEBUG=`expr 0$DEBUG + 1`;;
-y|--dryrun) DRYRUN=1;;
-m|--match) shift; MATCHHOST="$1";;
--) shift; break;;
-*) die "Invalid option: $1"; exit;;
*) break;
esac
shift
done
[ 0$DEBUG -gt 0 ] && echo "Debug level: $DEBUG"
HOSTLIST=`$XYMON $XYMSRV "xymondboard test=\Adisk\Z color=red,yellow,blue,purple,green fields=hostname"`
for HOST in $HOSTLIST; do
if [ "$MATCHHOST" ]; then
if [ "$HOST" = "$MATCHHOST" ]; then
[ 0$DEBUG -gt 1 ] && echo "Host $HOST matched $MATCHHOST"
else
[ 0$DEBUG -gt 3 ] && echo "Host $HOST did not match $MATCHHOST"
continue
fi
fi
#REPORT=`$XYMON $XYMSRV "xymondlog $HOST.disk"`
REPORT=`$XYMON $XYMSRV "xymondboard host=$HOST test=disk fields=msg"`
REPORT=`echo "$REPORT" | sed 's/\\\r\\\n/\n/g;s/\\\n/\\n/g;s/\\\r//g'`
[ "$REPORT" ] || echo "WARNING: no report for $HOST.disk"
COLOR=`$XYMON $XYMSRV "xymondboard host=$HOST test=disk fields=color"`
#COLOR=`echo "$REPORT" | head -1 | cut -d"|" -f3`
[ 0$DEBUG -gt 0 ] && echo "Checking host $HOST with disk=$COLOR:"
[ 0$DEBUG -gt 2 ] && { echo "Report:"; echo "$REPORT"; echo; }
REPORT=`echo "$REPORT" | sed '1d'`
case $COLOR in
purple)
# no recent disk status? don't report our own
;;
red|yellow|blue)
FSLIST=`echo "$REPORT" | sed -n '/^Filesystem/,/^$/p' | grep -v Filesystem | sed 's/^.*% *//;s/ .*$//'|tr -d '\r'`
FSLIST=`echo $FSLIST`
POS=0
REPLIST=""
for FS in $FSLIST; do
POS=`expr $POS + 1`
FSCOL=`echo "$REPORT" | grep "^&.* $FS " | sed 's/ .*$//;s/^&//'`
[ "$FSCOL" = "" -a 0$DEBUG -ge 1 ] && echo "Color not found for $FS, must be green"
[ "$FSCOL" ] || FSCOL="green" # not sure, assume green
FSNAME=`map_fsname $FS`
[ "$FSNAME" ] || FSNAME="$POS"
#FSNAME=`echo "$FS" | sed 's,^/,,;s/[^-_0-9A-Za-z]/_/g'`
TESTNAME="fs_$FSNAME"
[ 0$DEBUG -gt 1 ] && echo "[$TESTNAME] $FS ($FSNAME) is $FSCOL"
MSG="Filesystem $FS is $FSCOL, part of $COLOR disk report:
$REPORT"
if [ 0$DRYRUN -gt 0 ]; then
[ 0$DEBUG -ge 1 ] && echo $XYMON $XYMSRV "status $HOST.$TESTNAME $FSCOL `date` $MSG"
else
[ 0$DEBUG -ge 2 ] && echo $XYMON $XYMSRV "status $HOST.$TESTNAME $FSCOL `date` $MSG"
$XYMON $XYMSRV "status $HOST.$TESTNAME $FSCOL `date` $MSG"
fi
REPLIST="$REPLIST $TESTNAME"
done
if [ 0$DEBUG -gt 0 ]; then
[ "$REPLIST" ] && echo "Reported for: $REPLIST" || echo "Nothing to report"
fi
# clean up any old fs_* names not in $REPLIST
TESTNAMES=`$XYMON $XYMSRV "xymondboard host=$HOST test=^fs_ fields=testname"`
for TESTNAME in $TESTNAMES; do
FOUND=""
for REP in $REPLIST; do
[ "$REP" = "$TESTNAME" ] && FOUND=1
done
if [ "$FOUND" = "" ]; then
[ 0$DEBUG -ge 2 ] && echo "Remove $TESTNAME"
if [ 0$DRYRUN -gt 0 ]; then
[ 0$DEBUG -ge 1 ] && echo $XYMON $XYMSRV "drop $HOST $TESTNAME"
else
$XYMON $XYMSRV "drop $HOST $TESTNAME"
fi
fi
done
;;
green)
[ 0$DEBUG -ge 1 ] && echo "All green, nothing to do"
# TODO: remove any entries we have added previously
TESTNAMES=`$XYMON $XYMSRV "xymondboard host=$HOST test=^fs_ fields=testname"`
for TESTNAME in $TESTNAMES; do
[ 0$DEBUG -ge 2 ] && echo "Remove $TESTNAME"
if [ 0$DRYRUN -gt 0 ]; then
[ 0$DEBUG -ge 1 ] && echo $XYMON $XYMSRV "drop $HOST $TESTNAME"
else
$XYMON $XYMSRV "drop $HOST $TESTNAME"
fi
done
;;
clear)
# we make all of the known filesystems clear
REPORT2=`$XYMON $XYMSRV "xymondboard host=$HOST test=\Afs_\d+\Z fields=testname,color"`
if [ "$REPORT2" ]; then
REPLIST=""
for REPLINE in $REPORT2; do
TESTNAME=`echo "$REPLINE" | head -1 | cut -d"|" -f1` # eg fs_2
FSCOL=`echo "$REPLINE" | head -1 | cut -d"|" -f2`
if [ "$FSCOL" = "clear" ]; then
[ 0$DEBUG -gt 0 ] && echo "Not going to clearify $TESTNAME, already clear"
else
[ 0$DEBUG -gt 0 ] && echo "Clearifying $TESTNAME ($FSCOL)"
MSG="Filesystem $FS is $FSCOL, part of $COLOR disk test report:
$REPORT"
if [ 0$DRYRUN -gt 0 ]; then
echo $XYMON $XYMSRV "status $HOST.$TESTNAME clear `date` $MSG"
else
[ 0$DEBUG -ge 1 ] && $XYMON $XYMSRV "status $HOST.$TESTNAME clear `date` $MSG"
fi
REPLIST="$REPLIST $TESTNAME"
fi
done
if [ 0$DEBUG -gt 0 ]; then
[ "$REPLIST" ] && echo "Reported for: $REPLIST" || echo "Nothing to report"
fi
else
[ 0$DEBUG -gt 0 ] && echo "No fs reports for $HOST"
fi
;;
esac
[ 0$DEBUG -gt 0 ] && echo
done
==== tasks.d/xymon-fs-test.cfg ====
Copy this to the "tasks.d" directory on your Xymon server.
[fs-test]
ENVFILE $XYMONHOME/etc/xymonclient.cfg
CMD $XYMONHOME/ext/fs-test.sh
LOGFILE $XYMONSERVERLOGS/xymon-fstest.log
INTERVAL 5m
===== Known Bugs and Issues =====
===== To Do =====
* allow specifying filesystems to include/exclude in hosts.cfg
* allow specifying per-filesystem thresholds
===== Credits =====
===== Changelog =====
* **2013-02-19**
* Initial release
* **2013-02-28**
* auto-cleanup temporary test names