zonestat

zonestat

Author	Eric Meddaugh
Compatibility	Xymon 4.2
Requirements	Solaris 10
Download	None
Last Update	2009-10-26

Description

Monitor zones status under Solaris 10.

Installation

Client Configuration

Add script to client's ext folder

Add the following to client hobbitlaunch.cfg

[zonestat]
	ENVFILE $HOBBITCLIENTHOME/etc/hobbitclient.cfg
	CMD $HOBBITCLIENTHOME/ext/zonestat.sh 
	LOGFILE $HOBBITCLIENTHOME/logs/zonestat.log
	INTERVAL 5m

Server Configuration

Xymon only supports 1 data reader on the server. To make it as configurable as possible (we use multiple items to send stats to). Perl was used to help make it quicker. Inside the script has the graph definitions as well. You will have to add the information as documented below to the [rrddata] section to the hobbitlaunch.cfg file. Modify the hobbitlaunch.cfg file on the server under the rrddata section (add the –extra-tests and –extra-script segments):

# "rrddata" updates RRD files with information that arrives as "data"
# messages.  If you want RRD graphs of your monitoring BB data, then
# you want to run this.

[rrddata]
    ENVFILE /home/hobbit/server/etc/hobbitserver.cfg
    NEEDS hobbitd
    CMD hobbitd_channel --channel=data  
           --log=$BBSERVERLOGS/rrd-data.log hobbitd_rrd
           --extra-tests=mpstat,zonestat
           --extra-script=/home/hobbit/server/ext/rrd_data.pl
           --rrddir=$BBVAR/rrd

<note>Step 1 is not needed if you have the mpstat monitor installed.</note>

Add 'zonestat:zonestatcpu|zonestatmem|zonestatnproc|zonestatsize|zonersssize' to the bb-hosts entry.

0.0.0.0   myserver    # "TRENDS:*,zonestat:zonestatcpu|zonestatmem|zonestatnproc|zonestatsize|zonersssize"

Update hobbitgraph.cfg with:

Show Code ⇲

Hide Code ⇱

[zonestatcpu]
        TITLE Zone % CPU
        YAXIS % CPU
        FNPATTERN zonestat.(.*).rrd
        -u 100
        -r
        DEF:c@RRDIDX@=@RRDFN@:cpu:AVERAGE
        LINE1:c@RRDIDX@#@COLOR@:@RRDPARAM@
        GPRINT:c@RRDIDX@:LAST: \: %5.1lf (cur)
        GPRINT:c@RRDIDX@:MAX: \: %5.1lf (max)
        GPRINT:c@RRDIDX@:MIN: \: %5.1lf (min)
        GPRINT:c@RRDIDX@:AVERAGE: \: %5.1lf (avg)\n

[zonestatmem]
        TITLE Zone % Memory
        YAXIS % Memory
        FNPATTERN zonestat.(.*).rrd
        -u 100
        -r
        DEF:m@RRDIDX@=@RRDFN@:memory:AVERAGE
        LINE1:m@RRDIDX@#@COLOR@:@RRDPARAM@
        GPRINT:m@RRDIDX@:LAST: \: %5.1lf (cur)
        GPRINT:m@RRDIDX@:MAX: \: %5.1lf (max)
        GPRINT:m@RRDIDX@:MIN: \: %5.1lf (min)
        GPRINT:m@RRDIDX@:AVERAGE: \: %5.1lf (avg)\n

[zonestatnproc]
        TITLE Zone Num Proces
        YAXIS #
        FNPATTERN zonestat.(.*).rrd
        DEF:p@RRDIDX@=@RRDFN@:nproc:AVERAGE
        LINE1:p@RRDIDX@#@COLOR@:@RRDPARAM@
        GPRINT:p@RRDIDX@:LAST: \: %5.1lf (cur)
        GPRINT:p@RRDIDX@:MAX: \: %5.1lf (max)
        GPRINT:p@RRDIDX@:MIN: \: %5.1lf (min)
        GPRINT:p@RRDIDX@:AVERAGE: \: %5.1lf (avg)\n

[zonestatsize]
        # This graph only works on some OS'es (Solaris)
        TITLE Zone Memory Utilization
        YAXIS Size MB
        FNPATTERN zonestat.(.*).rrd
        DEF:s@RRDIDX@=@RRDFN@:size:AVERAGE
        LINE1:s@RRDIDX@#@COLOR@:@RRDPARAM@
        GPRINT:s@RRDIDX@:LAST: \: %5.1lf (cur)
        GPRINT:s@RRDIDX@:MAX: \: %5.1lf (max)
        GPRINT:s@RRDIDX@:MIN: \: %5.1lf (min)
        GPRINT:s@RRDIDX@:AVERAGE: \: %5.1lf (avg)\n

[zonersssize]
      # This graph only works on some OS'es (Solaris)
      TITLE Zone Resident Set Size
      YAXIS Size MB
      FNPATTERN zonestat.(.*).rrd
      DEF:s@RRDIDX@=@RRDFN@:rss:AVERAGE
      LINE1:s@RRDIDX@#@COLOR@:@RRDPARAM@
      GPRINT:s@RRDIDX@:LAST: \: %5.1lf (cur)
      GPRINT:s@RRDIDX@:MAX: \: %5.1lf (max)
      GPRINT:s@RRDIDX@:MIN: \: %5.1lf (min)
      GPRINT:s@RRDIDX@:AVERAGE: \: %5.1lf (avg)\n

Add 'zonestat' to the GRAPHS and TEST2RRD entry in hobbitserver.cfg.

Source

zonestat.sh

Show Code ⇲

Hide Code ⇱

#!/bin/sh

ZONE="global"
if [ -x "/bin/zonename" ] ; then
   ZONE=`/bin/zonename`
fi

if [ "${ZONE}" != "global" ] ; then
   exit 0
fi

OS=`uname -r`
REV=`echo "${OS}" | awk -F'.' '{printf $2}'`
# only run Solaris 10
if [ ${REV} -lt 10 ] ; then
   exit 0
fi

ZONELIST=`zoneadm list -v | egrep -v NAME | awk '{printf $1","}'`
ZONECNT=`zoneadm list -v | egrep -vc NAME`
ZONES=`zoneadm list -v | egrep -v NAME | awk '{printf $2" "}'`
NUM=`echo "${ZONECNT} + 5" | /bin/bc`

ZONENUM=`echo "${ZONECNT} + 2" | /bin/bc`

# only run if we actually are using zones
if [ ${ZONECNT} -le 1 ] ; then
   exit 0
fi

nohup sh -c "prstat -n 1,${NUM} -Zz ${ZONELIST} -c 300 2 1>$BBTMP/hobbit_zonestat.$MACHINEDOTS.$$ 2>&1; mv $BBTMP/hobbit_zonestat.$MACHINEDOTS.$$ $BBTMP/hobbit_zonestat.$MACHINEDOTS" </dev/null >/dev/null 2>&1 &
   sleep 5

if [ -f "$BBTMP/hobbit_zonestat.$MACHINEDOTS" ] ; then
   cat $BBTMP/hobbit_zonestat.$MACHINEDOTS | tail -${ZONENUM} | egrep -v "ZONEID|processes" > $BBTMP/hobbit_zonestat.$MACHINEDOTS.work

   echo "ZONEs: ${ZONES}" > $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
   while read ZONEID NPROC SIZE RSS MEMORY TIME CPU ZONE ; do

      if [ `echo "${SIZE}" | egrep -c "G"` -ge 1 ] ; then
         SIZE=`echo "${SIZE}" | sed -e's/G//'`
         SIZE=`echo "${SIZE} * 1024" | /bin/bc -l`
      elif [ `echo "${SIZE}" | egrep -c "M"` -ge 1 ] ; then
         SIZE=`echo "${SIZE}" | sed -e's/M//'`
      elif [ `echo "${SIZE}" | egrep -c "K"` -ge 1 ] ; then
         SIZE=`echo "${SIZE}" | sed -e's/K//'`
         SIZE=`echo "${SIZE} / 1024" | /bin/bc -l`
      fi

      if [ `echo "${RSS}" | egrep -c "G"` -ge 1 ] ; then
         RSS=`echo "${RSS}" | sed -e's/G//'`
         RSS=`echo "${RSS} * 1024" | /bin/bc -l`
      elif [ `echo "${RSS}" | egrep -c "M"` -ge 1 ] ; then
         RSS=`echo "${RSS}" | sed -e's/M//'`
      elif [ `echo "${RSS}" | egrep -c "K"` -ge 1 ] ; then
         RSS=`echo "${RSS}" | sed -e's/K//'`
         RSS=`echo "${RSS} / 1024" | /bin/bc -l`
      fi

      MEMORY=`echo "${MEMORY}" | sed -e's/\%//'`
      CPU=`echo "${CPU}" | sed -e's/\%//'`


      echo "${ZONE}nproc: ${NPROC}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
      echo "${ZONE}size: ${SIZE}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
      echo "${ZONE}rss: ${RSS}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
      echo "${ZONE}memory: ${MEMORY}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
      echo "${ZONE}cpu: ${CPU}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send

   done < $BBTMP/hobbit_zonestat.$MACHINEDOTS.work

      $BB $BBDISP "data $MACHINE.zonestat

`$CAT $BBTMP/hobbit_zonestat.$MACHINEDOTS.send`
"
   /bin/rm -f $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
   /bin/rm -f $BBTMP/hobbit_zonestat.$MACHINEDOTS.work
   /bin/rm $BBTMP/hobbit_zonestat.$MACHINEDOTS
fi

exit 0

zonestat.sh

Show Code ⇲

Hide Code ⇱

#!/bin/sh

#### This is an alternate zonestat.sh which handles reporting the zones as individual hosts.

#### You will need this entry in your sudoers to allow hobbit to zlogin to the zones:

# hobbit ALL = NOPASSWD: /usr/sbin/zlogin


SECS=300

ZONE="global"
if [ -x "/bin/zonename" ] ; then
   ZONE=`/bin/zonename`
fi

if [ "${ZONE}" != "global" ] ; then
   exit 0
fi

OS=`uname -r`
REV=`echo "${OS}" | awk -F'.' '{printf $2}'`
# only run Solaris 10
if [ ${REV} -lt 10 ] ; then
   exit 0
fi

ZONELIST=`zoneadm list -v | egrep -v NAME | awk '{printf $1","}'`
ZONECNT=`zoneadm list -v | egrep -vc NAME`
ZONES=`zoneadm list -v | egrep -v NAME | awk '{printf $2" "}'`
NUM=`echo "${ZONECNT} + 5" | /bin/bc`

ZONENUM=`echo "${ZONECNT} + 2" | /bin/bc`

# only run if we acutally are using zones
if [ ${ZONECNT} -le 1 ] ; then
   exit 0
fi

nohup sh -c "prstat -n 1,${NUM} -Zz ${ZONELIST} -c ${SECS} 2 1>$BBTMP/hobbit_zonestat.$MACHINEDOTS.$$ 2>&1; mv $BBTMP/hobbit_zonestat.$MACHINEDOTS.$$ $BBTMP/hobbit_zonestat.$MACHINEDOTS" </dev/null >/dev/null 2>&1 &
   sleep 5

if [ -f "$BBTMP/hobbit_zonestat.$MACHINEDOTS" ] ; then
   cat $BBTMP/hobbit_zonestat.$MACHINEDOTS | tail -${ZONENUM} | egrep -v "ZONEID|processes" > $BBTMP/hobbit_zonestat.$MACHINEDOTS.work

   echo "ZONEs: ${ZONES}" > $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
   while read ZONEID NPROC SIZE RSS MEMORY TIME CPU ZONE ; do

      if [ `echo "${SIZE}" | egrep -c "G"` -ge 1 ] ; then
         SIZE=`echo "${SIZE}" | sed -e's/G//'`
         SIZE=`echo "${SIZE} * 1024" | /bin/bc -l`
      elif [ `echo "${SIZE}" | egrep -c "M"` -ge 1 ] ; then
         SIZE=`echo "${SIZE}" | sed -e's/M//'`
      elif [ `echo "${SIZE}" | egrep -c "K"` -ge 1 ] ; then
         SIZE=`echo "${SIZE}" | sed -e's/K//'`
         SIZE=`echo "${SIZE} / 1024" | /bin/bc -l`
      fi

      if [ `echo "${RSS}" | egrep -c "G"` -ge 1 ] ; then
         RSS=`echo "${RSS}" | sed -e's/G//'`
         RSS=`echo "${RSS} * 1024" | /bin/bc -l`
      elif [ `echo "${RSS}" | egrep -c "M"` -ge 1 ] ; then
         RSS=`echo "${RSS}" | sed -e's/M//'`
      elif [ `echo "${RSS}" | egrep -c "K"` -ge 1 ] ; then
         RSS=`echo "${RSS}" | sed -e's/K//'`
         RSS=`echo "${RSS} / 1024" | /bin/bc -l`
      fi

      MEMORY=`echo "${MEMORY}" | sed -e's/\%//'`
      CPU=`echo "${CPU}" | sed -e's/\%//'`


      echo "${ZONE}nproc: ${NPROC}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
      echo "${ZONE}size: ${SIZE}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
      echo "${ZONE}rss: ${RSS}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
      echo "${ZONE}memory: ${MEMORY}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
      echo "${ZONE}cpu: ${CPU}" >> $BBTMP/hobbit_zonestat.$MACHINEDOTS.send

   done < $BBTMP/hobbit_zonestat.$MACHINEDOTS.work

   for ZONE in ${ZONES} ; do
      if [ "${ZONE}" = "global" ] ; then
         continue
      fi
      echo "ZONEs: ${ZONE}" > $BBTMP/hobbit_zonestat.$ZONE.send
      /usr/sbin/zoneadm -z ${ZONE} list -v > $BBTMP/hobbit_zonestatus.$ZONE.send
      egrep "^${ZONE}nproc:" $BBTMP/hobbit_zonestat.$MACHINEDOTS.send >> $BBTMP/hobbit_zonestat.$ZONE.send
      egrep "^${ZONE}size:" $BBTMP/hobbit_zonestat.$MACHINEDOTS.send >> $BBTMP/hobbit_zonestat.$ZONE.send
      egrep "^${ZONE}memory:" $BBTMP/hobbit_zonestat.$MACHINEDOTS.send >> $BBTMP/hobbit_zonestat.$ZONE.send
      egrep "^${ZONE}cpu:" $BBTMP/hobbit_zonestat.$MACHINEDOTS.send >> $BBTMP/hobbit_zonestat.$ZONE.send
      $BB $BBDISP "data $ZONE.zonestat

`$CAT $BBTMP/hobbit_zonestat.$ZONE.send`
"
      /bin/rm -f $BBTMP/hobbit_zonestat.$ZONE.send
      COLOR=green
      if [ `egrep -c running $BBTMP/hobbit_zonestatus.$ZONE.send` -eq 1 ] ; then
         COLOR="green"
      elif [ `egrep -c installed $BBTMP/hobbit_zonestatus.$ZONE.send` -eq 1 ] ; then
         COLOR="red"
      else
         COLOR="yellow"
      fi
      $BB $BBDISP "status $ZONE.zone $COLOR `$DATE` 

`$CAT $BBTMP/hobbit_zonestatus.$ZONE.send`
 "
      /bin/rm -f $BBTMP/hobbit_zonestatus.$ZONE.send

     echo "client $ZONE.$BBOSTYPE $CONFIGCLASS"  >>  $BBTMP/hobbit_zonestatus.$ZONE.send
     echo "[date]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
     /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /bin/date >> $BBTMP/hobbit_zonestatus.$ZONE.send
     echo "[uname]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
     /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /bin/uname -a >> $BBTMP/hobbit_zonestatus.$ZONE.send
     echo "[uptime]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
     /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /bin/uptime >> $BBTMP/hobbit_zonestatus.$ZONE.send
     echo "[who]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
     /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /bin/who >> $BBTMP/hobbit_zonestatus.$ZONE.send

     echo "[df]" >> $BBTMP/hobbit_zonestatus.$ZONE.send

# All of this because Solaris df cannot show multiple fs-types, or exclude certain fs types.
      FSTYPES=`/usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /bin/df -n -l|awk '{print $3}'|egrep -v "^proc|^fd|^mntfs|^ctfs|^devfs|^objfs|^nfs|"|sort|uniq`
      if test "$FSTYPES" = ""; then FSTYPES="lofs"; fi
      set $FSTYPES
      /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /bin/df -F $1 -k | grep -v " /var/run"  >> $BBTMP/hobbit_zonestatus.$ZONE.send
      shift
      while test "$1" != ""; do
        /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /bin/df -F $1 -k | grep -v " /var/run" | tail +2  >> $BBTMP/hobbit_zonestatus.$ZONE.send
        shift
      done
      echo "[mount]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
      /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /sbin/mount >> $BBTMP/hobbit_zonestatus.$ZONE.send

      echo "[ifconfig]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
      /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} /sbin/ifconfig -a >> $BBTMP/hobbit_zonestatus.$ZONE.send

      echo "[route]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
      /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} netstat -rn >> $BBTMP/hobbit_zonestatus.$ZONE.send

      echo "[ports]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
      /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} netstat -na -f inet -P tcp | tail +3 >> $BBTMP/hobbit_zonestatus.$ZONE.send
      /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} netstat -na -f inet6 -P tcp | tail +5 >> $BBTMP/hobbit_zonestatus.$ZONE.send

      echo "[ps]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
      /bin/ps -o pid,ppid,user,stime,s,pri,pcpu,time,pmem,rss,vsz,args -z $ZONE >> $BBTMP/hobbit_zonestatus.$ZONE.send
      echo "[top]" >> $BBTMP/hobbit_zonestatus.$ZONE.send
      /usr/local/bin/sudo /usr/sbin/zlogin ${ZONE} $TOP -b 20 >> $BBTMP/hobbit_zonestatus.$ZONE.send

      $BB $BBDISP "@" < $BBTMP/hobbit_zonestatus.$ZONE.send >$LOGFETCHCFG.$ZONE.tmp

   done

   /bin/rm -f $BBTMP/hobbit_zonestat.$MACHINEDOTS.send
   /bin/rm -f $BBTMP/hobbit_zonestat.$MACHINEDOTS.work
   /bin/rm $BBTMP/hobbit_zonestat.$MACHINEDOTS
fi

exit 0

rrd_data.pl

Show Code ⇲

Hide Code ⇱

#!/usr/local/bin/perl

use strict;

# Input parameters: Hostname, testname (column), and messagefile
my $hostname=$ARGV[0];
my $testname=$ARGV[1];
my $fname=$ARGV[2];

my ( $line,@line);
my ( @loop,$key,%buffer,$tmp );

open(IN,"$fname");
while(chomp($line=<IN>)) {
   @line=split(':',$line);
   $line[1] = join(' ',split(' ',$line[1]));
   $buffer{$line[0]}=$line[1];
}
close(IN);

if ( $testname eq "mpstat" ) {
##
## Reads in the mpstat.sh from client
##
   if ( defined($buffer{"CPUs"}) ) {

      @loop=split(' ',$buffer{"CPUs"});
      print "DS:CPU:GAUGE:600:0:32\n";

      print "DS:usr:GAUGE:600:0:100\n";
      print "DS:sys:GAUGE:600:0:100\n";
      print "DS:wt:GAUGE:600:0:100\n";
      print "DS:idl:GAUGE:600:0:100\n";

      foreach $key ( @loop ) {
         undef(@line);
         push(@line,$buffer{"${key}"});
         push(@line,$buffer{"CPU${key}usr"});
         push(@line,$buffer{"CPU${key}sys"});
         push(@line,$buffer{"CPU${key}wt"});
         push(@line,$buffer{"CPU${key}idl"});

         print "mpstat.CPU${key}.rrd\n";
         $line=join(':',@line);
         print "$line\n";
      }

   }
} elsif ( $testname eq "zonestat" ) {
##
## Reads in the zonestat.sh from client
##
   if ( defined($buffer{"ZONEs"}) ) {
      @loop=split(' ',$buffer{"ZONEs"});

      print "DS:nproc:GAUGE:600:0:U\n";
      print "DS:size:GAUGE:600:0:U\n";
      print "DS:rss:GAUGE:600:0:U\n";
      print "DS:memory:GAUGE:600:0:U\n";
      print "DS:cpu:GAUGE:600:0:U\n";

      foreach $key ( @loop ) {
         undef(@line);
         push(@line,$buffer{"${key}nproc"});
         push(@line,$buffer{"${key}size"});
         push(@line,$buffer{"${key}rss"});
         push(@line,$buffer{"${key}memory"});
         push(@line,$buffer{"${key}cpu"});

         $tmp = (split('/',$key))[1];
         print "zonestat.${key}.rrd\n";
         $line=join(':',@line);
         print "$line\n";
      }

   }
}

exit 0;

Table of Contents

zonestat

Description

Installation

Client Configuration

Server Configuration

Source

zonestat.sh

zonestat.sh

rrd_data.pl

Known Bugs and Issues

To Do

Credits

Changelog