monitors:fs-test

no way to compare when less than two revisions

Differences

This shows you the differences between two versions of the page.


—monitors:fs-test [2013/02/28 06:16] (current) – created - external edit 127.0.0.1
Line 1: Line 1:
 +====== xymon-fstest ======
  
 +^ Author | [[ jlaidman+xymon-fstest@rebel-it.com.au | Jeremy Laidman ]] |
 +^ Compatibility | Xymon 4.2+ |
 +^ Requirements | Bourne or Bash shell (or compatible), Xymon server |
 +^ Download | None |
 +^ Last Update | 2013-02-28 |
 +
 +===== Description =====
 +This script checks filesystems and reports them in their own separate
 +tests (''fs_1'', ''fs_2'', ... or ''fs_root'', ''fs_var'', ...), by making use of the combined "''disk''" test.
 +
 +Initially, only disks with non-green status will have the new "''fs_*''" pages created.
 +Once a page exists, it will be persistent, going green when the "disk" test is green.
 +
 +If the "''disk''" test goes purple or blue, the "''fs_*''" tests will change to clear.
 +
 +Filesystem mount points are used in the test name, where possible (eg "''/''" becomes "''fs_root''", "''/usr''" becomes "''fs_usr''") but names longer than four characters (excluding leading slash) are converted to an index number (eg "''/var/log''" becomes "''fs_2''" for the second filesystem).
 +
 +Screenshot:
 +{{ :monitors:fs-test-screenshot.png? |}}
 +===== Installation =====
 +=== Client side ===
 +Ensure "disk" test is working.  That is all.
 +
 +=== Server side ===
 +Copy to server/ext/ directory, create entry in tasks.cfg or new file in tasks.d/ directory.
 +
 +For troubleshooting, run with DEBUG and/or DRYRUN set to a positive integer.
 +===== Source =====
 +==== fs-test.sh ====
 +
 +Copy this to the "server/ext" directory on your Xymon server.
 +
 +<hidden onHidden="Show Code ⇲" onVisible="Hide Code ⇱">
 +<code>
 +
 +#!/bin/sh
 +
 +# Checks filesystems and reports them in their own separate
 +# tests (fs_1, fs_2, ...) as well as the one combined disk test.
 +# Todo:
 +#  - allow specifying filesystems to include/exclude in hosts.cfg
 +#  - allow specifying per-filesystem thresholds
 +
 +die() { echo "$@" >&2; exit 1; }
 +
 +[ "$XYMON" -a "$XYMSRV" ] || die "Run under Xymon environment"
 +
 +do_usage() {
 +        echo "Usage: $0 [-h|-d|-y|-m match]"
 +        echo "-m match  : match hostname"
 +        echo "-h        : help"
 +        echo "-d        : increase debug level by one (repeat for more)"
 +        echo "-y        : dry run"
 +}
 +
 +map_fsname() {
 +        # maps filesystem name to test name
 +        # must try to keep it short (no more than 4 chars)
 +        # TODO: allow mapping in hosts.cfg
 +        NEWNAME=""
 +        case $1 in
 +                /)      NEWNAME="root";;
 +                /?)     NEWNAME=`IFS=/; set - $1""; echo $2`;;
 +                /??)    NEWNAME=`IFS=/; set - $1""; echo $2`;;
 +                /???)   NEWNAME=`IFS=/; set - $1""; echo $2`;;
 +                /????)  NEWNAME=`IFS=/; set - $1""; echo $2`;;
 +        esac
 +        echo "$NEWNAME"
 +        [ 0$DEBUG -ge 2 ] && echo "Changed '$1' to '$NEWNAME'" >&2
 +}
 +
 +while [ "$1" ]; do
 +        case $1 in
 +                -h|--h*)        do_usage; exit;;
 +                -d|--debug)     DEBUG=`expr 0$DEBUG + 1`;;
 +                -y|--dryrun)    DRYRUN=1;;
 +                -m|--match)     shift; MATCHHOST="$1";;
 +                --)             shift; break;;
 +                -*)             die "Invalid option: $1"; exit;;
 +                *)              break;
 +        esac
 +        shift
 +done
 +
 +[ 0$DEBUG -gt 0 ] && echo "Debug level: $DEBUG"
 +
 +HOSTLIST=`$XYMON $XYMSRV "xymondboard test=\Adisk\Z color=red,yellow,blue,purple,green fields=hostname"`
 +for HOST in $HOSTLIST; do
 +        if [ "$MATCHHOST" ]; then
 +                if [ "$HOST" = "$MATCHHOST" ]; then
 +                        [ 0$DEBUG -gt 1 ] && echo "Host $HOST matched $MATCHHOST"
 +                else
 +                        [ 0$DEBUG -gt 3 ] && echo "Host $HOST did not match $MATCHHOST"
 +                        continue
 +                fi
 +        fi
 +        #REPORT=`$XYMON $XYMSRV "xymondlog $HOST.disk"`
 +        REPORT=`$XYMON $XYMSRV "xymondboard host=$HOST test=disk fields=msg"`
 +        REPORT=`echo "$REPORT" | sed 's/\\\r\\\n/\n/g;s/\\\n/\\n/g;s/\\\r//g'`
 +        [ "$REPORT" ] || echo "WARNING: no report for $HOST.disk"
 +        COLOR=`$XYMON $XYMSRV "xymondboard host=$HOST test=disk fields=color"`
 +        #COLOR=`echo "$REPORT" | head -1 | cut -d"|" -f3`
 +        [ 0$DEBUG -gt 0 ] && echo "Checking host $HOST with disk=$COLOR:"
 +        [ 0$DEBUG -gt 2 ] && { echo "Report:"; echo "$REPORT"; echo; }
 +        REPORT=`echo "$REPORT" | sed '1d'`
 +        case $COLOR in
 +                purple)
 +                        # no recent disk status? don't report our own
 +                        ;;
 +                red|yellow|blue)
 +                        FSLIST=`echo "$REPORT" | sed -n '/^Filesystem/,/^$/p' | grep -v Filesystem | sed 's/^.*% *//;s/ .*$//'|tr -d '\r'`
 +                        FSLIST=`echo $FSLIST`
 +                        POS=0
 +                        REPLIST=""
 +                        for FS in $FSLIST; do
 +                                POS=`expr $POS + 1`
 +                                FSCOL=`echo "$REPORT" | grep "^&.* $FS " | sed 's/ .*$//;s/^&//'`
 +                                [ "$FSCOL" = "" -a 0$DEBUG -ge 1 ] && echo "Color not found for $FS, must be green"
 +                                [ "$FSCOL" ] || FSCOL="green"   # not sure, assume green
 +
 +                                FSNAME=`map_fsname $FS`
 +                                [ "$FSNAME" ] || FSNAME="$POS"
 +                                #FSNAME=`echo "$FS" | sed 's,^/,,;s/[^-_0-9A-Za-z]/_/g'`
 +
 +                                TESTNAME="fs_$FSNAME"
 +                                [ 0$DEBUG -gt 1 ] && echo "[$TESTNAME] $FS ($FSNAME) is $FSCOL"
 +                                MSG="Filesystem $FS is $FSCOL, part of $COLOR disk report:
 +$REPORT"
 +                                if [ 0$DRYRUN -gt 0 ]; then
 +                                        [ 0$DEBUG -ge 1 ] && echo $XYMON $XYMSRV "status $HOST.$TESTNAME $FSCOL `date` $MSG"
 +                                else
 +                                        [ 0$DEBUG -ge 2 ] && echo $XYMON $XYMSRV "status $HOST.$TESTNAME $FSCOL `date` $MSG"
 +                                        $XYMON $XYMSRV "status $HOST.$TESTNAME $FSCOL `date` $MSG"
 +                                fi
 +                                REPLIST="$REPLIST $TESTNAME"
 +                        done
 +
 +                        if [ 0$DEBUG -gt 0 ]; then
 +                                [ "$REPLIST" ] && echo "Reported for: $REPLIST" || echo "Nothing to report"
 +                        fi
 +
 +                        # clean up any old fs_* names not in $REPLIST
 +                        TESTNAMES=`$XYMON $XYMSRV "xymondboard host=$HOST test=^fs_ fields=testname"`
 +                        for TESTNAME in $TESTNAMES; do
 +                                FOUND=""
 +                                for REP in $REPLIST; do
 +                                        [ "$REP" = "$TESTNAME" ] && FOUND=1
 +                                done
 +                                if [ "$FOUND" = "" ]; then
 +                                        [ 0$DEBUG -ge 2 ] && echo "Remove $TESTNAME"
 +                                        if [ 0$DRYRUN -gt 0 ]; then
 +                                                [ 0$DEBUG -ge 1 ] && echo $XYMON $XYMSRV "drop $HOST $TESTNAME"
 +                                        else
 +                                                $XYMON $XYMSRV "drop $HOST $TESTNAME"
 +                                        fi
 +                                fi
 +                        done
 +                        ;;
 +                green)
 +                        [ 0$DEBUG -ge 1 ] && echo "All green, nothing to do"
 +                        # TODO: remove any entries we have added previously
 +                        TESTNAMES=`$XYMON $XYMSRV "xymondboard host=$HOST test=^fs_ fields=testname"`
 +                        for TESTNAME in $TESTNAMES; do
 +                                [ 0$DEBUG -ge 2 ] && echo "Remove $TESTNAME"
 +                                if [ 0$DRYRUN -gt 0 ]; then
 +                                        [ 0$DEBUG -ge 1 ] && echo $XYMON $XYMSRV "drop $HOST $TESTNAME"
 +                                else
 +                                        $XYMON $XYMSRV "drop $HOST $TESTNAME"
 +                                fi
 +                        done
 +                        ;;
 +                clear)
 +                        # we make all of the known filesystems clear
 +                        REPORT2=`$XYMON $XYMSRV "xymondboard host=$HOST test=\Afs_\d+\Z fields=testname,color"`
 +                        if [ "$REPORT2" ]; then
 +                                REPLIST=""
 +                                for REPLINE in $REPORT2; do
 +                                        TESTNAME=`echo "$REPLINE" | head -1 | cut -d"|" -f1`    # eg fs_2
 +                                        FSCOL=`echo "$REPLINE" | head -1 | cut -d"|" -f2`
 +                                        if [ "$FSCOL" = "clear" ]; then
 +                                                [ 0$DEBUG -gt 0 ] && echo "Not going to clearify $TESTNAME, already clear"
 +                                        else
 +                                                [ 0$DEBUG -gt 0 ] && echo "Clearifying $TESTNAME ($FSCOL)"
 +                                                MSG="Filesystem $FS is $FSCOL, part of $COLOR disk test report:
 +$REPORT"
 +                                                if [ 0$DRYRUN -gt 0 ]; then
 +                                                        echo $XYMON $XYMSRV "status $HOST.$TESTNAME clear `date` $MSG"
 +                                                else
 +                                                        [ 0$DEBUG -ge 1 ] && $XYMON $XYMSRV "status $HOST.$TESTNAME clear `date` $MSG"
 +                                                fi
 +                                                REPLIST="$REPLIST $TESTNAME"
 +                                        fi
 +                                done
 +                                if [ 0$DEBUG -gt 0 ]; then
 +                                        [ "$REPLIST" ] && echo "Reported for: $REPLIST" || echo "Nothing to report"
 +                                fi
 +                        else
 +                                [ 0$DEBUG -gt 0 ] && echo "No fs reports for $HOST"
 +                        fi
 +                        ;;
 +                esac
 +        [ 0$DEBUG -gt 0 ] && echo
 +done
 +
 +</code>
 +</hidden>
 +
 +==== tasks.d/xymon-fs-test.cfg ====
 +
 +Copy this to the "tasks.d" directory on your Xymon server.
 +
 +<hidden onHidden="Show Code ⇲" onVisible="Hide Code ⇱">
 +<code>
 +[fs-test]
 +        ENVFILE $XYMONHOME/etc/xymonclient.cfg
 +        CMD $XYMONHOME/ext/fs-test.sh
 +        LOGFILE $XYMONSERVERLOGS/xymon-fstest.log
 +        INTERVAL 5m
 +</code>
 +</hidden>
 +
 +===== Known  Bugs and Issues =====
 +
 +===== To Do =====
 +* allow specifying filesystems to include/exclude in hosts.cfg
 +* allow specifying per-filesystem thresholds
 +
 +===== Credits =====
 +
 +===== Changelog =====
 +
 +  * **2013-02-19**
 +    * Initial release
 +  * **2013-02-28**
 +    * auto-cleanup temporary test names
  • monitors/fs-test.txt
  • Last modified: 2013/02/28 06:16
  • by 127.0.0.1