====== WinEvtMsgs ====== ^ Author | [[ dbaldwin@users.sf.net|David Baldwin ]] | ^ Compatibility | Xymon 4.2 | ^ Requirements | Perl, rsyslog/syslog-ng, [[monitors:XymonExt.pm]] | ^ Download | None | ^ Last Update | 2010-08-17 | ===== Description ===== Report on Windows Event logs forwarded with SNARE note: xymon/hobbit user must have read access to logs (see FileGroup in rsyslog.conf) [[http://www.intersectalliance.com/projects/SnareWindows/index.html|SNARE]] is a Windows event log forwarder over syslog. ===== Installation ===== === Client side === Each Windows server needs the appropriate version of Snare installed (Vista/2008 different from older versions of Windows). === Server side === Install on central syslog server. Assumed this is the same as Xymon server (need to use bb-hosts 'evt' tag to denote tested hosts) Can also report on cluster nodes and cluster resources (e.g. SQL or Exchange) - in such cases each cluster node forwards all events for all nodes. bb-hosts example for SQL instances: 10.0.0.22 c2n1 # evt 10.0.0.23 c2n2 # evt 10.0.0.24 c2n3 # evt 10.0.0.26 sql1 # evt=10.0.0.22 10.0.0.27 sql2 # evt=10.0.0.22 10.0.0.28 sql3 # evt=10.0.0.22 [[http://www.rsyslog.com/|rsyslog]] or syslog-ng can be used on a central syslog server to accumulate logs for each host in a separate directory Sample /etc/rsyslog.conf section to log each host into directory /var/log/rsyslog/IP/messages-YYYYMMDD note: "rsyslogd -x" required to disable DNS lookup on remote IP note: "rsyslogd -c3" required for V3 config (rsyslog V3) # $ModLoad immark # provides --MARK-- message capability $ModLoad imudp # provides UDP syslog reception $UDPServerAddress * # this MUST be before the $UDPServerRun directive! $UDPServerRun 514 $ModLoad imtcp # provides TCP syslog reception $InputTCPServerRun 514 $ModLoad imuxsock # provides support for local system logging (e.g. via logger command) $ModLoad imklog # provides kernel logging support (previously done by rklogd) $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat #--------------------ALL REMOTE LOGS---------------------------------------- $umask 0000 $DirOwner root $DirGroup logview $DirCreateMode 0750 $FileOwner root $FileGroup logview $FileCreateMode 0640 $template DynFile,"/var/log/rsyslog/%FROMHOST%/messages-%$YEAR%%$MONTH%%$DAY%" :source , !isequal , "loghost" ?DynFile Add to hobbitlaunch.cfg # "winevt" checks forwarded windows event logs [winevt] ENVFILE /usr/lib/hobbit/server/etc/hobbitserver.cfg NEEDS hobbitd CMD $BBHOME/ext/winevtmsgs.pl LOGFILE $BBSERVERLOGS/winevtmsgs.log INTERVAL 5m ===== Source ===== ==== winevtmsgs.pl ==== Install in server/ext directory #!/usr/bin/perl -w #CHKHOSTS evt # # winevtmsgs.pl # # Report on Windows Event logs forwarded with SNARE # # note: xymon/hobbit user must have read access to logs (see FileGroup in rsyslog.conf) # # SNARE is a Windows event log forwarder over syslog. # http://www.intersectalliance.com/projects/SnareWindows/index.html # =begin rsyslog # rsyslog or syslog-ng can be used on a central syslog server to accumulate logs for each host in a separate directory # # http://www.rsyslog.com/ # Sample /etc/rsyslog.conf section to log each host into directory /var/log/rsyslog/IP/messages-YYYYMMDD # note: "rsyslogd -x" required to disable DNS lookup on remote IP # note: "rsyslogd -c3" required for V3 config # (rsyslog V3) # $ModLoad immark # provides --MARK-- message capability $ModLoad imudp # provides UDP syslog reception $UDPServerAddress * # this MUST be before the $UDPServerRun directive! $UDPServerRun 514 $ModLoad imtcp # provides TCP syslog reception $InputTCPServerRun 514 $ModLoad imuxsock # provides support for local system logging (e.g. via logger command) $ModLoad imklog # provides kernel logging support (previously done by rklogd) $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat #--------------------ALL REMOTE LOGS---------------------------------------- $umask 0000 $DirOwner root $DirGroup logview $DirCreateMode 0750 $FileOwner root $FileGroup logview $FileCreateMode 0640 $template DynFile,"/var/log/rsyslog/%FROMHOST%/messages-%$YEAR%%$MONTH%%$DAY%" :source , !isequal , "loghost" ?DynFile =cut my $xymonlib; BEGIN { $xymonlib = $ENV{BBHOME} && "$ENV{BBHOME}/ext" || "/usr/lib/hobbit/server/ext"; } use strict; use lib $xymonlib; use POSIX; use Fcntl; # For O_RDWR, O_CREAT, etc. use DB_File; use XymonExt; use Getopt::Long; use Data::Dumper; $|=1; my $debug = 0; # 0 = none 1 = low # config my $logdir = "/var/log/rsyslog"; my $test = "evt"; my $bbtest = "evt"; my $topredlim = 5; # end of test config my ($bbstatus,$bbmsg,$topred,$topredcnt); my $filedump = 0; my $exit_status = 0; my $addr = ''; my $complete = 0; my $previous = 0; my %cluster; my $evthf = "$BBTMP/winevtmsgs-hash"; my %logsp; tie(%logsp, 'DB_File', $evthf, O_RDWR|O_CREAT, 0660) or die "Couldn't tie DB file '$evthf': $!; aborting"; # Global status variables - don't forget to initialiase in main loop below to clear before checking new host ... #------------------------------------------------------------------------------ while ( $ARGV[0] && $ARGV[0] =~ /^-/ ) { if ( $ARGV[0] eq '-d' || $ARGV[0] eq '--debug' ) { $debug++; shift @ARGV; next; } if ( $ARGV[0] eq '-f' || $ARGV[0] eq '--file' ) { $filedump++; shift @ARGV; next; } if ( $ARGV[0] eq '-a' || $ARGV[0] eq '--address' ) { shift @ARGV; if ( ! $ARGV[0] ) { &usage; exit 5; } $addr = $ARGV[0]; shift @ARGV; next; } if ( $ARGV[0] eq '-c' || $ARGV[0] eq '--complete' ) { $complete++; shift @ARGV; next; } if ( $ARGV[0] eq '-p' || $ARGV[0] eq '--previous' ) { $previous++; shift @ARGV; next; } exit 5; } if ( $ARGV[0] ) { die "Usage: $0 [-a|--address ] [-d|--debug] [-f|--file] [-c|--complete] [-p|--previous]\n"; } #------------------------------------------------------------------------------ my %tests = do "/etc/hobbit/winevtmsgs.cfg"; sub match { my ($val,$match) = @_; return 0 unless defined $val; if(ref $match eq "Regexp") { return $val =~ $match; } else { return $val eq $match; } } my %dbgc = (); sub reportline { my $line = shift; my $rephost = shift; my %fields = (); @fields{qw{date evt eid src ser tsp evn cat usr na typ hst n1 n2 msg n3}} = split /#011/,$line; my ($host) = ($fields{date} =~ /\s(\S+)$/); print "DBGHD: $host $fields{hst} $rephost\n" if $debug && ! $dbgc{001}{$host}{$fields{hst}}++; return if lc $rephost ne lc $fields{hst}; my $lcol = ""; MATCH: foreach my $tk ((grep {$_ ne "default"} keys %tests), "default") { print "DBGKD: testing $host in test $tk\n" if $debug && ! $dbgc{000}{$host}{$tk}++; my %tr = %{$tests{$tk}}; next if defined($tr{host}) && ! match($host,$tr{host}); print "DBG: matched $host in test $tk\n" if $debug && ! $dbgc{$host}{$tk}++; if(defined $tr{ignore}) { foreach my $ik (keys %{$tr{ignore}}) { my %ti = %{$tr{ignore}{$ik}}; my $im = scalar keys %ti; foreach my $ikt (keys %ti) { $im &&= match($fields{$ikt}, $ti{$ikt}); } if($im) { print "DBGIGN($tk;$ik): $line\n" if $debug; return; } } } foreach my $col (qw{red yellow green}) { TEST: foreach my $ik (keys %{$tr{$col}}) { print "DBGD: testing $host in $col test $ik\n" if $debug && ! $dbgc{111}{$host}{$col}{$ik}++; my %ti = %{$tr{$col}{$ik}}; foreach my $ikt (keys %ti) { print "DBGCK: ($ikt) $fields{$ikt}, $ti{$ikt}\n" if $debug > 1; next TEST unless match($fields{$ikt}, $ti{$ikt}); } print "DBG: matched $col in test $ik group $tk\n" if $debug; $lcol = $col; last MATCH; # exit on first match - otherwise default may override specific match - do we need priority also? } } } return if $lcol eq ""; my $msg = $fields{msg} || $fields{n1} || ""; my $evtmsg = "&$lcol $fields{tsp} ${fields{typ}}[$fields{src}:$fields{cat}]: (User: $fields{usr}) {$fields{evn}} $msg\n"; $topred .= $evtmsg if $lcol eq "red" && $topredlim > $topredcnt++; $bbmsg .= $evtmsg; XymonExt->UpdateStatus($lcol); } ### Main loop my @dflist = (); if (length $addr == 0 || ! $debug) { @dflist = XymonExt->HostsByTest($test); } else { @dflist = ($addr); } if($debug) { printf "** DEBUG Host list ** %s\n",join(" ",@dflist); } my @testtime = localtime(time - 300 - $previous*24*60*60); # test log for previous 5 minutes if($filedump) { my $logdate = strftime "%Y%m%d",@testtime; foreach my $f (keys %logsp) { my $old = $f =~ /messages-(\d+)/ && $1 < $logdate; print "$f: $logsp{$f} : $old\n"; if($filedump>1 && $old) { delete $logsp{$f}; } } exit; } foreach my $bbhost (@dflist) { my ($shrephost) = split /\./,$bbhost; my $ipaddr = XymonExt->HostIP($bbhost); my $altipaddr = XymonExt->HostItems($bbhost,"$bbtest=.*"); print "DBGIPs: $ipaddr $altipaddr\n" if $debug; if( my ($ip) = ($altipaddr =~ /^$bbtest=(.*)$/)) { if( $ip =~ /^[\d\.]+$/) { $ipaddr = $ip; } else { push @{$cluster{$1}},($ipaddr); } } XymonExt->InitStatus(); $bbmsg = ""; $topred = ""; $topredcnt = 0; my $c = 0; my $newsp = 0; my $prevsp = 0; my $logfile = strftime "$logdir/$ipaddr/messages-%Y%m%d",@testtime; print "\n====\nDBG: logfile: $logfile\n" if $debug; if(open LF,$logfile) { if(exists $logsp{$logfile} && ! $complete) { $prevsp = $logsp{$logfile}; print "DBG: logfile SP: $prevsp\n" if $debug; my $seeksts = seek LF, $prevsp, 0; print "DBGSTS: $seeksts\n" if $debug; } else { print "DBGNOSP: No saved SP for $logfile\n" if $debug; } my $tspat = strftime "^%b %e %H:%M",@testtime; $tspat =~ s/[0-4]$/[0-4]/; $tspat =~ s/[5-9]$/[5-9]/; $tspat .= ":[0-5][0-9] "; print "DBG: tspat: /$tspat/\n" if $debug; my $tsre = qr{$tspat}; while(my $ll = ) { print $ll if $debug > 1; reportline($ll,$shrephost); if( $ll =~ /$tsre/) { print "DBGMAT: $prevsp new: $newsp\n" if $debug; unless( $newsp ) { $newsp = $prevsp; print "DBG: set new SP: $newsp\n" if $debug; } $c++; } $prevsp = tell(LF) unless $newsp; } close LF; print "DBG: $c\n" if $debug; print "DBG: new logfile SP: $newsp\n" if $debug; $logsp{$logfile} = $newsp if $newsp > 0; my $scol = XymonExt->GetStatus(); my $warn = $scol eq "red" && "errors found" || $scol eq "yellow" && "warnings found" || "OK"; my $redcnt = $topredcnt > $topredlim && $topredlim || $topredcnt; my $redmsg = $topredcnt ? "
Top $redcnt red messages:\n$topred\n
\n" : ""; $bbmsg = "Eventlog $warn\n${redmsg}All messages:\n$bbmsg\nLogfile: $logfile - $c lines matched /$tspat/\n"; } else { $bbmsg = "ERROR\nCan't open $logfile\n"; if($testtime[2]*60+$testtime[1] > 15) { # allow 15 minutes after midnight for SOMETHING to be logged XymonExt->UpdateStatus("red"); } else { XymonExt->UpdateStatus("clear"); } } #------------------------------------------------------------------------------ if ($debug) { print "BB: $bbhost.$bbtest sts: ".XymonExt->GetStatus()." MSG:\n$bbmsg"; } else { XymonExt->Report($bbhost,$bbtest,XymonExt->GetStatus(),$bbmsg); } } # end main loop exit $exit_status; #==============================================================================
==== winevtmsgs.cfg ==== Install in /etc/hobbit/winevtmsgs.cfg #!/usr/bin/perl -w #------------------------------------------------------------------------------ # winevtmsgs.pl config file (/etc/hobbit/winevtmsgs.cfg) # # Field values can be defined as exact string match (plain string) or regexp (qr//). # # Rules must have unique key (name) - default is always processed last # host field is matched against host name from syslog file line (generated by SNARE) # matching rules use key: ignore, red, yellow, green # ignore processed first, then red, yellow, green in that order # allows for raising or lowering of default priority (Error=red, Warning=yellow, Information=green) # other fields for matching purposes are from Event log line (* = displayed in report): # date = syslog date/time stamp # evt = "MSWinEventLog" # eid = "0" # * src = Security, Application, etc # ser = event number (sequence) # * tsp = time/date stamp (from Windows host) # * evn = event ID # * cat = category/subsystem # * usr = account/username (or Unknown User) # na = "User" # * typ = Warning, Error, Information, etc # hst = Windows host name or cluster node/resource name # ? n1 = sometimes message (e.g. Exchange), often "None" # n2 = "" # * msg = event message (sometime blank - e.g. Exchange) # n3 = another sequence number? # # Reporting line format is: # colour tsp typ[src:cat]: (User: usr) {evn} msg # (if msg is blank, n1 is reported instead - e.g. some Exchange messages) # # Tune as required based on "false" reports at inappropriate severity levels # # sample rule: # DCs (Domain Controllers) # host name specified by regexp # ignore System:MRxSMB 8003 messages about Browser service # yellow on System:KDC 26 messages (often Error, but not that significant) # green on System:NETLOGON (various) messages (often Error) about deleted/etc computer accounts # # "DCs" => { # "host" => qr/^(dc\d+)/i, # "ignore" => { # "Browser" => { # "src" => "System", # "cat" => "MRxSmb", # "evn" => qr/^(8003)$/, # }, # }, # "yellow" => { # "KDC" => { # "src" => "System", # "cat" => "KDC", # "evn" => "26", # }, # }, # "green" => { # "NoCompAcct" => { # "src" => "System", # "cat" => "NETLOGON", # "evn" => qr/^(5719|572[23]|5805)$/, # }, # }, # }, "DCs" => { "host" => qr/^(dc\d+)/i, "yellow" => { "KDC" => { "src" => "System", "cat" => "KDC", "evn" => "26", }, }, "green" => { "NoCompAcct" => { "src" => "System", "cat" => "NETLOGON", "evn" => qr/^(5719|572[23]|5805)$/, }, "Browser" => { "src" => "System", "cat" => "MRxSmb", "evn" => qr/^(8003)$/, }, "KDC mult acct" => { "src" => "System", "cat" => "KDC", "evn" => qr/^(11)$/, "msg" => qr/altirissdvsv/, }, }, }, "WSUS replica" => { "host" => qr/^(wsusrep)/i, "yellow" => { "WSUS synch" => { "src" => "Application", "cat" => "Windows Server Update Services", "evn" => qr/^(10022)$/, }, }, "green" => { "WSUS self update" => { "src" => "Application", "cat" => "Windows Server Update Services", "evn" => qr/^(13042)$/, }, }, }, "Citrix" => { "host" => qr/^(citrix\d+)/i, "ignore" => { "Print" => { "src" => "System", "cat" => "Print", "evn" => qr/^([34])$/, }, }, "green" => { "NoPrinterDriver" => { "src" => "Application", "cat" => "MetaFrameEvents", "evn" => qr/^(110[367]|1116)$/, }, "TRIM" => { "src" => "Application", "cat" => "TRIM", "evn" => qr/^(100)$/, }, "TermServDev" => { "src" => "System", "cat" => "TermServDevices", "evn" => qr/^(1111)$/, }, "Office" => { "src" => "Application", "cat" => qr/^Microsoft Office/, "evn" => qr/^([12]000)$/, }, }, "yellow" => { "msGina" => { "src" => "Application", "cat" => "MsGina", "evn" => qr/^(1010)$/, }, "Folder Redirection" => { "src" => "Application", "cat" => "Folder Redirection", "evn" => qr/^(107)$/, }, "Folder Redirection 2" => { "src" => "Application", "cat" => "Folder Redirection", "evn" => qr/^(101)$/, "msg" => qr/not enough space on the disk/, }, "Userenv" => { "src" => "Application", "cat" => "Userenv", "evn" => qr/^(10[45]3|1085|1096)$/, }, "Application Error" => { "src" => "Application", "cat" => "Application Error", "evn" => qr/^(1000)$/, }, "Citrix Resource Management" => { "src" => "Application", "cat" => "Citrix Resource Management", "evn" => qr/^(257)$/, }, "UserInit" => { "src" => "Application", "cat" => "UserInit", "evn" => qr/^(1000)$/, }, }, }, "Citrix-WI" => { "host" => qr/^(\w*cxwi)/i, "yellow" => { "Web Interface" => { "src" => "Application", "typ" => "Error", "cat" => qr/^Web Interface/, "evn" => qr/^(0)$/, }, }, }, "Cluster1" => { "host" => qr/^(c1n\d)/i, "ignore" => { "Exchange" => { "src" => "Application", "cat" => qr/^(MSExchangeIS|EXCDO)/, "evn" => qr/^(8231|9646)$/, }, }, "yellow" => { "ExchangeMbox" => { "src" => "Application", "cat" => "MSExchangeIS Mailbox Store", "evn" => qr/^(1022|1147|1203|10001)$/, }, "EXCDO" => { "src" => "Application", "cat" => "EXCDO", "evn" => qr/^(8206|8217)$/, }, "MSExchangeFBPublish" => { "src" => "Application", "cat" => "MSExchangeFBPublish", "evn" => qr/^(8275)$/, }, "ClusSvc" => { "src" => "System", "cat" => "ClusSvc", "evn" => qr/^(1069)$/, }, }, }, "Exchange" => { "host" => qr/^(exch\d)/i, "yellow" => { "MSExchange Availability" => { "src" => "Application", "cat" => "MSExchange Availability", "evn" => qr/^(4011)$/, }, "MSExchangeTransport" => { "src" => "Application", "cat" => "MSExchangeTransport", "evn" => qr/^(12014)$/, }, }, }, "Cluster2" => { "host" => qr/^(c2n\d)/i, "yellow" => { "MSSQL" => { "src" => "Application", "cat" => qr/^MSSQL/, "evn" => qr/^(17806)$/, }, "WinMgmt" => { "src" => "Application", "cat" => "WinMgmt", "evn" => qr/^(10)$/, }, }, }, "All(pre-default)" => { "green" => { "BBWin" => { "src" => "Application", "cat" => "BigBrotherHobbitClient", "evn" => qr/^(20)$/, }, "Print" => { "src" => "System", "cat" => "Print", "evn" => "6161", }, "SChannel" => { "src" => "System", "cat" => "Schannel", "evn" => qr/^(36874)$/, }, }, "yellow" => { "Microsoft-Windows-GroupPolicy" => { "src" => "System", "cat" => "Microsoft-Windows-GroupPolicy", "evn" => "1058", }, "Foundation Agents" => { "src" => "System", "cat" => "Foundation Agents", "evn" => qr/^(1181)$/, }, "Kerberos" => { "src" => "System", "cat" => "Kerberos", "evn" => qr/^(4)$/, }, "Userenv" => { "src" => "Application", "cat" => "Userenv", "evn" => qr/^(1053)$/, }, "W32Time" => { "src" => "System", "cat" => "W32Time", "evn" => qr/^(29)$/, }, "NETLOGON" => { "src" => "System", "cat" => "NETLOGON", "evn" => qr/^(5719)$/, }, "TermDD" => { "src" => "System", "cat" => "TermDD", "evn" => qr/^(5[06])$/, }, "AV" => { "src" => "Application", "cat" => "eTrust ITM", "evn" => qr/^(23)$/, }, "ASP.Net" => { "src" => "Application", "cat" => qr/^ASP.NET \d/, "evn" => qr/^(1093)$/, }, ".NET Runtime Optimization Service" => { "src" => "Application", "cat" => ".NET Runtime Optimization Service", "evn" => qr/^(1101)$/, }, }, "red" => { "SRMSVC" => { "src" => "Application", "cat" => "SRMSVC", "evn" => "12324", }, "HP Ethernet" => { # only Warning - I'd rather red! "src" => "System", "cat" => "HP Ethernet", "evn" => "4", }, }, }, "default" => { "ignore" => { "DCOM" => { "src" => "System", "cat" => "DCOM", "evn" => qr/^(1000[05]|10015)$/, }, "VMdebug" => { "src" => "System", "cat" => "vmdebug", "evn" => "3", }, "W3CTRS" => { "src" => "Application", "cat" => "W3CTRS", "evn" => qr/^(200[13])$/, }, "Perflib" => { "src" => "Application", "cat" => "Perflib", "evn" => qr/^(1008|101[0578]|102[123]|200[23])$/, }, "loadperf" => { "src" => "Application", "cat" => "LoadPerf", "evn" => qr/^(3012)$/, }, "NetBT" => { "src" => "System", "cat" => "NetBT", "evn" => qr/^(4321)$/, }, "Browser" => { "src" => "System", "cat" => "BROWSER", "evn" => qr/^(80[01]9|8032)$/, }, }, "red" => { "Error" => { "typ" => "Error", }, }, "yellow" => { "Warning" => { "typ" => "Warning", }, }, "green" => { "Information" => { "typ" => "Information", }, }, }, #============================================================================== ===== Known Bugs and Issues ===== ===== To Do ===== ===== Credits ===== ===== Changelog ===== * **2010-08-17** * Initial release