====== WinEvtMsgs ======
^ Author | [[ dbaldwin@users.sf.net|David Baldwin ]] |
^ Compatibility | Xymon 4.2 |
^ Requirements | Perl, rsyslog/syslog-ng, [[monitors:XymonExt.pm]] |
^ Download | None |
^ Last Update | 2010-08-17 |
===== Description =====
Report on Windows Event logs forwarded with SNARE
note: xymon/hobbit user must have read access to logs (see FileGroup in rsyslog.conf)
[[http://www.intersectalliance.com/projects/SnareWindows/index.html|SNARE]] is a Windows event log forwarder over syslog.
===== Installation =====
=== Client side ===
Each Windows server needs the appropriate version of Snare installed (Vista/2008 different from older versions of Windows).
=== Server side ===
Install on central syslog server. Assumed this is the same as Xymon server (need to use bb-hosts 'evt' tag to denote tested hosts)
Can also report on cluster nodes and cluster resources (e.g. SQL or Exchange) - in such cases each cluster node forwards all events for all nodes.
bb-hosts example for SQL instances:
10.0.0.22 c2n1 # evt
10.0.0.23 c2n2 # evt
10.0.0.24 c2n3 # evt
10.0.0.26 sql1 # evt=10.0.0.22
10.0.0.27 sql2 # evt=10.0.0.22
10.0.0.28 sql3 # evt=10.0.0.22
[[http://www.rsyslog.com/|rsyslog]] or syslog-ng can be used on a central syslog server to accumulate logs for each host in a separate directory
Sample /etc/rsyslog.conf section to log each host into directory /var/log/rsyslog/IP/messages-YYYYMMDD
note: "rsyslogd -x" required to disable DNS lookup on remote IP
note: "rsyslogd -c3" required for V3 config (rsyslog V3)
# $ModLoad immark # provides --MARK-- message capability
$ModLoad imudp # provides UDP syslog reception
$UDPServerAddress * # this MUST be before the $UDPServerRun directive!
$UDPServerRun 514
$ModLoad imtcp # provides TCP syslog reception
$InputTCPServerRun 514
$ModLoad imuxsock # provides support for local system logging (e.g. via logger command)
$ModLoad imklog # provides kernel logging support (previously done by rklogd)
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
#--------------------ALL REMOTE LOGS----------------------------------------
$umask 0000
$DirOwner root
$DirGroup logview
$DirCreateMode 0750
$FileOwner root
$FileGroup logview
$FileCreateMode 0640
$template DynFile,"/var/log/rsyslog/%FROMHOST%/messages-%$YEAR%%$MONTH%%$DAY%"
:source , !isequal , "loghost" ?DynFile
Add to hobbitlaunch.cfg
# "winevt" checks forwarded windows event logs
[winevt]
ENVFILE /usr/lib/hobbit/server/etc/hobbitserver.cfg
NEEDS hobbitd
CMD $BBHOME/ext/winevtmsgs.pl
LOGFILE $BBSERVERLOGS/winevtmsgs.log
INTERVAL 5m
===== Source =====
==== winevtmsgs.pl ====
Install in server/ext directory
#!/usr/bin/perl -w
#CHKHOSTS evt
#
# winevtmsgs.pl
#
# Report on Windows Event logs forwarded with SNARE
#
# note: xymon/hobbit user must have read access to logs (see FileGroup in rsyslog.conf)
#
# SNARE is a Windows event log forwarder over syslog.
# http://www.intersectalliance.com/projects/SnareWindows/index.html
#
=begin rsyslog
# rsyslog or syslog-ng can be used on a central syslog server to accumulate logs for each host in a separate directory
#
# http://www.rsyslog.com/
# Sample /etc/rsyslog.conf section to log each host into directory /var/log/rsyslog/IP/messages-YYYYMMDD
# note: "rsyslogd -x" required to disable DNS lookup on remote IP
# note: "rsyslogd -c3" required for V3 config
# (rsyslog V3)
# $ModLoad immark # provides --MARK-- message capability
$ModLoad imudp # provides UDP syslog reception
$UDPServerAddress * # this MUST be before the $UDPServerRun directive!
$UDPServerRun 514
$ModLoad imtcp # provides TCP syslog reception
$InputTCPServerRun 514
$ModLoad imuxsock # provides support for local system logging (e.g. via logger command)
$ModLoad imklog # provides kernel logging support (previously done by rklogd)
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
#--------------------ALL REMOTE LOGS----------------------------------------
$umask 0000
$DirOwner root
$DirGroup logview
$DirCreateMode 0750
$FileOwner root
$FileGroup logview
$FileCreateMode 0640
$template DynFile,"/var/log/rsyslog/%FROMHOST%/messages-%$YEAR%%$MONTH%%$DAY%"
:source , !isequal , "loghost" ?DynFile
=cut
my $xymonlib;
BEGIN { $xymonlib = $ENV{BBHOME} && "$ENV{BBHOME}/ext" || "/usr/lib/hobbit/server/ext"; }
use strict;
use lib $xymonlib;
use POSIX;
use Fcntl; # For O_RDWR, O_CREAT, etc.
use DB_File;
use XymonExt;
use Getopt::Long;
use Data::Dumper;
$|=1;
my $debug = 0; # 0 = none 1 = low
# config
my $logdir = "/var/log/rsyslog";
my $test = "evt";
my $bbtest = "evt";
my $topredlim = 5;
# end of test config
my ($bbstatus,$bbmsg,$topred,$topredcnt);
my $filedump = 0;
my $exit_status = 0;
my $addr = '';
my $complete = 0;
my $previous = 0;
my %cluster;
my $evthf = "$BBTMP/winevtmsgs-hash";
my %logsp;
tie(%logsp, 'DB_File', $evthf, O_RDWR|O_CREAT, 0660)
or die "Couldn't tie DB file '$evthf': $!; aborting";
# Global status variables - don't forget to initialiase in main loop below to clear before checking new host ...
#------------------------------------------------------------------------------
while ( $ARGV[0] && $ARGV[0] =~ /^-/ ) {
if ( $ARGV[0] eq '-d' || $ARGV[0] eq '--debug' ) {
$debug++;
shift @ARGV;
next;
}
if ( $ARGV[0] eq '-f' || $ARGV[0] eq '--file' ) {
$filedump++;
shift @ARGV;
next;
}
if ( $ARGV[0] eq '-a' || $ARGV[0] eq '--address' ) {
shift @ARGV;
if ( ! $ARGV[0] ) { &usage; exit 5; }
$addr = $ARGV[0];
shift @ARGV;
next;
}
if ( $ARGV[0] eq '-c' || $ARGV[0] eq '--complete' ) {
$complete++;
shift @ARGV;
next;
}
if ( $ARGV[0] eq '-p' || $ARGV[0] eq '--previous' ) {
$previous++;
shift @ARGV;
next;
}
exit 5;
}
if ( $ARGV[0] ) {
die "Usage: $0 [-a|--address ] [-d|--debug] [-f|--file] [-c|--complete] [-p|--previous]\n";
}
#------------------------------------------------------------------------------
my %tests = do "/etc/hobbit/winevtmsgs.cfg";
sub match {
my ($val,$match) = @_;
return 0 unless defined $val;
if(ref $match eq "Regexp") {
return $val =~ $match;
} else {
return $val eq $match;
}
}
my %dbgc = ();
sub reportline {
my $line = shift;
my $rephost = shift;
my %fields = ();
@fields{qw{date evt eid src ser tsp evn cat usr na typ hst n1 n2 msg n3}} = split /#011/,$line;
my ($host) = ($fields{date} =~ /\s(\S+)$/);
print "DBGHD: $host $fields{hst} $rephost\n" if $debug && ! $dbgc{001}{$host}{$fields{hst}}++;
return if lc $rephost ne lc $fields{hst};
my $lcol = "";
MATCH: foreach my $tk ((grep {$_ ne "default"} keys %tests), "default") {
print "DBGKD: testing $host in test $tk\n" if $debug && ! $dbgc{000}{$host}{$tk}++;
my %tr = %{$tests{$tk}};
next if defined($tr{host}) && ! match($host,$tr{host});
print "DBG: matched $host in test $tk\n" if $debug && ! $dbgc{$host}{$tk}++;
if(defined $tr{ignore}) {
foreach my $ik (keys %{$tr{ignore}}) {
my %ti = %{$tr{ignore}{$ik}};
my $im = scalar keys %ti;
foreach my $ikt (keys %ti) {
$im &&= match($fields{$ikt}, $ti{$ikt});
}
if($im) {
print "DBGIGN($tk;$ik): $line\n" if $debug;
return;
}
}
}
foreach my $col (qw{red yellow green}) {
TEST: foreach my $ik (keys %{$tr{$col}}) {
print "DBGD: testing $host in $col test $ik\n" if $debug && ! $dbgc{111}{$host}{$col}{$ik}++;
my %ti = %{$tr{$col}{$ik}};
foreach my $ikt (keys %ti) {
print "DBGCK: ($ikt) $fields{$ikt}, $ti{$ikt}\n" if $debug > 1;
next TEST unless match($fields{$ikt}, $ti{$ikt});
}
print "DBG: matched $col in test $ik group $tk\n" if $debug;
$lcol = $col;
last MATCH; # exit on first match - otherwise default may override specific match - do we need priority also?
}
}
}
return if $lcol eq "";
my $msg = $fields{msg} || $fields{n1} || "";
my $evtmsg = "&$lcol $fields{tsp} ${fields{typ}}[$fields{src}:$fields{cat}]: (User: $fields{usr}) {$fields{evn}} $msg\n";
$topred .= $evtmsg if $lcol eq "red" && $topredlim > $topredcnt++;
$bbmsg .= $evtmsg;
XymonExt->UpdateStatus($lcol);
}
### Main loop
my @dflist = ();
if (length $addr == 0 || ! $debug) {
@dflist = XymonExt->HostsByTest($test);
} else {
@dflist = ($addr);
}
if($debug) {
printf "** DEBUG Host list ** %s\n",join(" ",@dflist);
}
my @testtime = localtime(time - 300 - $previous*24*60*60); # test log for previous 5 minutes
if($filedump) {
my $logdate = strftime "%Y%m%d",@testtime;
foreach my $f (keys %logsp) {
my $old = $f =~ /messages-(\d+)/ && $1 < $logdate;
print "$f: $logsp{$f} : $old\n";
if($filedump>1 && $old) {
delete $logsp{$f};
}
}
exit;
}
foreach my $bbhost (@dflist) {
my ($shrephost) = split /\./,$bbhost;
my $ipaddr = XymonExt->HostIP($bbhost);
my $altipaddr = XymonExt->HostItems($bbhost,"$bbtest=.*");
print "DBGIPs: $ipaddr $altipaddr\n" if $debug;
if( my ($ip) = ($altipaddr =~ /^$bbtest=(.*)$/)) {
if( $ip =~ /^[\d\.]+$/) {
$ipaddr = $ip;
} else {
push @{$cluster{$1}},($ipaddr);
}
}
XymonExt->InitStatus();
$bbmsg = "";
$topred = "";
$topredcnt = 0;
my $c = 0;
my $newsp = 0;
my $prevsp = 0;
my $logfile = strftime "$logdir/$ipaddr/messages-%Y%m%d",@testtime;
print "\n====\nDBG: logfile: $logfile\n" if $debug;
if(open LF,$logfile) {
if(exists $logsp{$logfile} && ! $complete) {
$prevsp = $logsp{$logfile};
print "DBG: logfile SP: $prevsp\n" if $debug;
my $seeksts = seek LF, $prevsp, 0;
print "DBGSTS: $seeksts\n" if $debug;
} else {
print "DBGNOSP: No saved SP for $logfile\n" if $debug;
}
my $tspat = strftime "^%b %e %H:%M",@testtime;
$tspat =~ s/[0-4]$/[0-4]/;
$tspat =~ s/[5-9]$/[5-9]/;
$tspat .= ":[0-5][0-9] ";
print "DBG: tspat: /$tspat/\n" if $debug;
my $tsre = qr{$tspat};
while(my $ll = ) {
print $ll if $debug > 1;
reportline($ll,$shrephost);
if( $ll =~ /$tsre/) {
print "DBGMAT: $prevsp new: $newsp\n" if $debug;
unless( $newsp ) {
$newsp = $prevsp;
print "DBG: set new SP: $newsp\n" if $debug;
}
$c++;
}
$prevsp = tell(LF) unless $newsp;
}
close LF;
print "DBG: $c\n" if $debug;
print "DBG: new logfile SP: $newsp\n" if $debug;
$logsp{$logfile} = $newsp if $newsp > 0;
my $scol = XymonExt->GetStatus();
my $warn = $scol eq "red" && "errors found" || $scol eq "yellow" && "warnings found" || "OK";
my $redcnt = $topredcnt > $topredlim && $topredlim || $topredcnt;
my $redmsg = $topredcnt ? "
Top $redcnt red messages:\n$topred\n
\n" : "";
$bbmsg = "Eventlog $warn\n${redmsg}All messages:\n$bbmsg\nLogfile: $logfile - $c lines matched /$tspat/\n";
} else {
$bbmsg = "ERROR\nCan't open $logfile\n";
if($testtime[2]*60+$testtime[1] > 15) { # allow 15 minutes after midnight for SOMETHING to be logged
XymonExt->UpdateStatus("red");
} else {
XymonExt->UpdateStatus("clear");
}
}
#------------------------------------------------------------------------------
if ($debug) {
print "BB: $bbhost.$bbtest sts: ".XymonExt->GetStatus()." MSG:\n$bbmsg";
} else {
XymonExt->Report($bbhost,$bbtest,XymonExt->GetStatus(),$bbmsg);
}
} # end main loop
exit $exit_status;
#==============================================================================
==== winevtmsgs.cfg ====
Install in /etc/hobbit/winevtmsgs.cfg
#!/usr/bin/perl -w
#------------------------------------------------------------------------------
# winevtmsgs.pl config file (/etc/hobbit/winevtmsgs.cfg)
#
# Field values can be defined as exact string match (plain string) or regexp (qr//).
#
# Rules must have unique key (name) - default is always processed last
# host field is matched against host name from syslog file line (generated by SNARE)
# matching rules use key: ignore, red, yellow, green
# ignore processed first, then red, yellow, green in that order
# allows for raising or lowering of default priority (Error=red, Warning=yellow, Information=green)
# other fields for matching purposes are from Event log line (* = displayed in report):
# date = syslog date/time stamp
# evt = "MSWinEventLog"
# eid = "0"
# * src = Security, Application, etc
# ser = event number (sequence)
# * tsp = time/date stamp (from Windows host)
# * evn = event ID
# * cat = category/subsystem
# * usr = account/username (or Unknown User)
# na = "User"
# * typ = Warning, Error, Information, etc
# hst = Windows host name or cluster node/resource name
# ? n1 = sometimes message (e.g. Exchange), often "None"
# n2 = ""
# * msg = event message (sometime blank - e.g. Exchange)
# n3 = another sequence number?
#
# Reporting line format is:
# colour tsp typ[src:cat]: (User: usr) {evn} msg
# (if msg is blank, n1 is reported instead - e.g. some Exchange messages)
#
# Tune as required based on "false" reports at inappropriate severity levels
#
# sample rule:
# DCs (Domain Controllers)
# host name specified by regexp
# ignore System:MRxSMB 8003 messages about Browser service
# yellow on System:KDC 26 messages (often Error, but not that significant)
# green on System:NETLOGON (various) messages (often Error) about deleted/etc computer accounts
#
# "DCs" => {
# "host" => qr/^(dc\d+)/i,
# "ignore" => {
# "Browser" => {
# "src" => "System",
# "cat" => "MRxSmb",
# "evn" => qr/^(8003)$/,
# },
# },
# "yellow" => {
# "KDC" => {
# "src" => "System",
# "cat" => "KDC",
# "evn" => "26",
# },
# },
# "green" => {
# "NoCompAcct" => {
# "src" => "System",
# "cat" => "NETLOGON",
# "evn" => qr/^(5719|572[23]|5805)$/,
# },
# },
# },
"DCs" => {
"host" => qr/^(dc\d+)/i,
"yellow" => {
"KDC" => {
"src" => "System",
"cat" => "KDC",
"evn" => "26",
},
},
"green" => {
"NoCompAcct" => {
"src" => "System",
"cat" => "NETLOGON",
"evn" => qr/^(5719|572[23]|5805)$/,
},
"Browser" => {
"src" => "System",
"cat" => "MRxSmb",
"evn" => qr/^(8003)$/,
},
"KDC mult acct" => {
"src" => "System",
"cat" => "KDC",
"evn" => qr/^(11)$/,
"msg" => qr/altirissdvsv/,
},
},
},
"WSUS replica" => {
"host" => qr/^(wsusrep)/i,
"yellow" => {
"WSUS synch" => {
"src" => "Application",
"cat" => "Windows Server Update Services",
"evn" => qr/^(10022)$/,
},
},
"green" => {
"WSUS self update" => {
"src" => "Application",
"cat" => "Windows Server Update Services",
"evn" => qr/^(13042)$/,
},
},
},
"Citrix" => {
"host" => qr/^(citrix\d+)/i,
"ignore" => {
"Print" => {
"src" => "System",
"cat" => "Print",
"evn" => qr/^([34])$/,
},
},
"green" => {
"NoPrinterDriver" => {
"src" => "Application",
"cat" => "MetaFrameEvents",
"evn" => qr/^(110[367]|1116)$/,
},
"TRIM" => {
"src" => "Application",
"cat" => "TRIM",
"evn" => qr/^(100)$/,
},
"TermServDev" => {
"src" => "System",
"cat" => "TermServDevices",
"evn" => qr/^(1111)$/,
},
"Office" => {
"src" => "Application",
"cat" => qr/^Microsoft Office/,
"evn" => qr/^([12]000)$/,
},
},
"yellow" => {
"msGina" => {
"src" => "Application",
"cat" => "MsGina",
"evn" => qr/^(1010)$/,
},
"Folder Redirection" => {
"src" => "Application",
"cat" => "Folder Redirection",
"evn" => qr/^(107)$/,
},
"Folder Redirection 2" => {
"src" => "Application",
"cat" => "Folder Redirection",
"evn" => qr/^(101)$/,
"msg" => qr/not enough space on the disk/,
},
"Userenv" => {
"src" => "Application",
"cat" => "Userenv",
"evn" => qr/^(10[45]3|1085|1096)$/,
},
"Application Error" => {
"src" => "Application",
"cat" => "Application Error",
"evn" => qr/^(1000)$/,
},
"Citrix Resource Management" => {
"src" => "Application",
"cat" => "Citrix Resource Management",
"evn" => qr/^(257)$/,
},
"UserInit" => {
"src" => "Application",
"cat" => "UserInit",
"evn" => qr/^(1000)$/,
},
},
},
"Citrix-WI" => {
"host" => qr/^(\w*cxwi)/i,
"yellow" => {
"Web Interface" => {
"src" => "Application",
"typ" => "Error",
"cat" => qr/^Web Interface/,
"evn" => qr/^(0)$/,
},
},
},
"Cluster1" => {
"host" => qr/^(c1n\d)/i,
"ignore" => {
"Exchange" => {
"src" => "Application",
"cat" => qr/^(MSExchangeIS|EXCDO)/,
"evn" => qr/^(8231|9646)$/,
},
},
"yellow" => {
"ExchangeMbox" => {
"src" => "Application",
"cat" => "MSExchangeIS Mailbox Store",
"evn" => qr/^(1022|1147|1203|10001)$/,
},
"EXCDO" => {
"src" => "Application",
"cat" => "EXCDO",
"evn" => qr/^(8206|8217)$/,
},
"MSExchangeFBPublish" => {
"src" => "Application",
"cat" => "MSExchangeFBPublish",
"evn" => qr/^(8275)$/,
},
"ClusSvc" => {
"src" => "System",
"cat" => "ClusSvc",
"evn" => qr/^(1069)$/,
},
},
},
"Exchange" => {
"host" => qr/^(exch\d)/i,
"yellow" => {
"MSExchange Availability" => {
"src" => "Application",
"cat" => "MSExchange Availability",
"evn" => qr/^(4011)$/,
},
"MSExchangeTransport" => {
"src" => "Application",
"cat" => "MSExchangeTransport",
"evn" => qr/^(12014)$/,
},
},
},
"Cluster2" => {
"host" => qr/^(c2n\d)/i,
"yellow" => {
"MSSQL" => {
"src" => "Application",
"cat" => qr/^MSSQL/,
"evn" => qr/^(17806)$/,
},
"WinMgmt" => {
"src" => "Application",
"cat" => "WinMgmt",
"evn" => qr/^(10)$/,
},
},
},
"All(pre-default)" => {
"green" => {
"BBWin" => {
"src" => "Application",
"cat" => "BigBrotherHobbitClient",
"evn" => qr/^(20)$/,
},
"Print" => {
"src" => "System",
"cat" => "Print",
"evn" => "6161",
},
"SChannel" => {
"src" => "System",
"cat" => "Schannel",
"evn" => qr/^(36874)$/,
},
},
"yellow" => {
"Microsoft-Windows-GroupPolicy" => {
"src" => "System",
"cat" => "Microsoft-Windows-GroupPolicy",
"evn" => "1058",
},
"Foundation Agents" => {
"src" => "System",
"cat" => "Foundation Agents",
"evn" => qr/^(1181)$/,
},
"Kerberos" => {
"src" => "System",
"cat" => "Kerberos",
"evn" => qr/^(4)$/,
},
"Userenv" => {
"src" => "Application",
"cat" => "Userenv",
"evn" => qr/^(1053)$/,
},
"W32Time" => {
"src" => "System",
"cat" => "W32Time",
"evn" => qr/^(29)$/,
},
"NETLOGON" => {
"src" => "System",
"cat" => "NETLOGON",
"evn" => qr/^(5719)$/,
},
"TermDD" => {
"src" => "System",
"cat" => "TermDD",
"evn" => qr/^(5[06])$/,
},
"AV" => {
"src" => "Application",
"cat" => "eTrust ITM",
"evn" => qr/^(23)$/,
},
"ASP.Net" => {
"src" => "Application",
"cat" => qr/^ASP.NET \d/,
"evn" => qr/^(1093)$/,
},
".NET Runtime Optimization Service" => {
"src" => "Application",
"cat" => ".NET Runtime Optimization Service",
"evn" => qr/^(1101)$/,
},
},
"red" => {
"SRMSVC" => {
"src" => "Application",
"cat" => "SRMSVC",
"evn" => "12324",
},
"HP Ethernet" => { # only Warning - I'd rather red!
"src" => "System",
"cat" => "HP Ethernet",
"evn" => "4",
},
},
},
"default" => {
"ignore" => {
"DCOM" => {
"src" => "System",
"cat" => "DCOM",
"evn" => qr/^(1000[05]|10015)$/,
},
"VMdebug" => {
"src" => "System",
"cat" => "vmdebug",
"evn" => "3",
},
"W3CTRS" => {
"src" => "Application",
"cat" => "W3CTRS",
"evn" => qr/^(200[13])$/,
},
"Perflib" => {
"src" => "Application",
"cat" => "Perflib",
"evn" => qr/^(1008|101[0578]|102[123]|200[23])$/,
},
"loadperf" => {
"src" => "Application",
"cat" => "LoadPerf",
"evn" => qr/^(3012)$/,
},
"NetBT" => {
"src" => "System",
"cat" => "NetBT",
"evn" => qr/^(4321)$/,
},
"Browser" => {
"src" => "System",
"cat" => "BROWSER",
"evn" => qr/^(80[01]9|8032)$/,
},
},
"red" => {
"Error" => {
"typ" => "Error",
},
},
"yellow" => {
"Warning" => {
"typ" => "Warning",
},
},
"green" => {
"Information" => {
"typ" => "Information",
},
},
},
#==============================================================================
===== Known Bugs and Issues =====
===== To Do =====
===== Credits =====
===== Changelog =====
* **2010-08-17**
* Initial release