#!/usr/bin/perl -w

# parse /var/log/messages to remove the stuff which i know i
# don't care about... add to this as i find more stuff i don't
# care about

use strict;

use Time::Local;

# list of named "contains our address" domains which we're ignoring
# ... add to this as each one is checked to make sure we're not really
# supposed to be serving dns for it.
my %contains_our_address = (
	'1000words.org' => 1,
	'3brighton.ca' => 1,
	'alchemica.org' => 1,
	'alphabetties.com' => 1,
	'arockstar.com' => 1,
	'atomicsauce.com' => 1,
	'chibi.ca' => 1,
	'csua.com' => 1,
	'dougalnet.net' => 1,
	'dvdswapper.org' => 1,
	'earbud.net' => 1,
	'exactor.org' => 1,
	'extropian.org' => 1,
	'furtographer.com' => 1,
	'futurustic.com' => 1,
	'ghettotuna.com' => 1,
	'glenbardwest1989.com' => 1,
	'goodvibrations.com' => 1,
	'gp500racing.com' => 1,
	'hed-efx.com' => 1,
	'hed-efx.net' => 1,
	'hed-efx.org' => 1,
	'hedefx.com' => 1,
	'hedefx.net' => 1,
	'heh.to' => 1,
	'hirethisgeek.com' => 1,
	'humancreativity.com' => 1,
	'humdesign.com' => 1,
	'iconic.com' => 1,
	'inquiz.com' => 1,
	'internet-rock-star.com' => 1,
	'jdougal.com' => 1,
	'karimkhany.com' => 1,
	'kilobike.com' => 1,
	'kourosh.com' => 1,
	'liminaldesign.com' => 1,
	'leparkour.com' => 1,
	'loree.org' => 1,
	'megancash.com' => 1,
	'mindflick.net' => 1,
	'moesque.com' => 1,
	'monkeysee.com' => 1,
	'morph.org' => 1,
	'moslounge.com' => 1,
	'occamswriter.com' => 1,
	'ockhamswriter.com' => 1,
	'oengine.com' => 1,
	'oengine.org' => 1,
	'pacificsiam.com' => 1,
	'phatstep.com' => 1,
	'philegg.com' => 1,
	'portalkit.net' => 1,
	'rallykings.org' => 1,
	'rjnet.org' => 1,
	'robotswilldestroy.com' => 1,
	'scooterracing.com' => 1,
	'shackgroove.com' => 1,
	'shackgroove.org' => 1,
	'skullbaby.com' => 1,
	'strangeplanets.com' => 1,
	'stupidproduct.com' => 1,
	'sukornyk.com' => 1,
	'synchronicitysearch.com' => 1,
	'synchronicitysearch.net' => 1,
	'synchronicitysearch.org' => 1,
	'tiddleycovemorris.org' => 1,
	'vatgrown.com' => 1,
	'vgninja.com' => 1,
	'weddingsforone.com' => 1,
	'weddingforone.com' => 1,
	'worldforuminc.com' => 1,
);

# misconfigured DNS servers which respond on addresses other than which
# they are queried result in a "Response from unexpected source" log message.
# this is a list of domains we don't care to see such messages from.
my %ignore_unexpected_domains = (
	'985pro54.com' => 1,
	'adminwebhost.net' => 1,
	'adultoptinfo.com' => 1,
	'airton.ru' => 1,
	'al.us' => 1,
	'allstonefab.com' => 1,
	'aol.com' => 1,
	'aps-now.com' => 1,
	'asc.edu' => 1,
	'auctionwatch.com' => 1,
	'bigfish.com' => 1,
	'bsc.edu' => 1,
	'cais.com' => 1,
	'cais.net' => 1,
	'cbsig.net' => 1,
	'charter.net' => 1,
	'cn' => 1,
	'compuserve.com' => 1,
	'cs.com' => 1,
	'dailypromo.net' => 1,
	'dementia.org' => 1,
	'derekdeaton.com' => 1,
	'dezel.net' => 1,
	'direcpc.com' => 1,
	'donovandata.com' => 1,
	'dp-server.com' => 1,
	'ecampus.com' => 1,
	'emailgaul.com' => 1,
	'etienne.net' => 1,
	'europeanheritage.com' => 1,
	'fasttrackmonkey.com' => 1,
	'granitecanyon.com' => 1,
	'gu.nu' => 1,
	'hi-speedemail.net' => 1,
	'hoak.com' => 1,
	'hp.com' => 1,
	'hp.net' => 1,
	'hsmmailer.com' => 1,
	'iberbanda.es' => 1,
	'iboogie.tv' => 1,
	'ilovevideo.com' => 1,
	'inter.net' => 1,
	'jatis.com' => 1,
	'joc.com' => 1,
	'joyoftech.com' => 1,
	'jp' => 1,
	'lava.net' => 1,
	'lexis-nexis.com' => 1,
	'lisco.com' => 1,
	'macserialjunkie.com' => 1,
	'maxwebportal.com' => 1,
	'membersinfostation.com' => 1,
	'mxtech.org' => 1,
	'my' => 1,
	'nais.org' => 1,
	'nameserverhost.com' => 1,
	'netcenter.com' => 1,
	'netscape.com' => 1,
	'netscape.net' => 1,
	'onetravel.com' => 1,
	'optmaildirect.com' => 1,
	'pacifier.com' => 1,
	'proaxis.com' => 1,
	'prophotosupply.com' => 1,
	'quantumpro.com' => 1,
	'rubiconcomputing.com' => 1,
	'se' => 1,
	'spiraloasis.org' => 1,
	'suburbia.net' => 1,
	'tcminc.us' => 1,
	'tdn.com' => 1,
	'thefragile.com' => 1,
	'turner.com' => 1,
	'uconect.net' => 1,
	'uk' => 1,
	'ultra-mail.com' => 1,
	'united.com' => 1,
	'weather.com' => 1,
	'wmg.com' => 1,
	'wport.com' => 1,
);

# these are for "denied update" and "unapproved update" ... dunno
# what causes these errors yet.  or what to do about them.
my %ignore_updates = (
	'champine.net' => 1,
	'directoryservice.com' => 1,
	'everybodys-doing-it.com' => 1,
	'taylor.org' => 1,
	'toaster.net' => 1,
	'utopiabomb.com' => 1,
	'voyager.org' => 1,
	'mutex.org' => 1,
);

my $last_message_printed = 0;
my $save_notify;

# this is the maximum time delta in seconds which we accept between
# -- MARK -- lines ... if there's any gap larger than this then we
# print out a missing mark message.
my $mark_max_delta = 20*60+5;
my $last_mark;

# for date parsing
my %months = (
        'Jan' => 0,
        'Feb' => 1,
        'Mar' => 2,
        'Apr' => 3,
        'May' => 4,
        'Jun' => 5,
        'Jul' => 6,
        'Aug' => 7,
        'Sep' => 8,
        'Oct' => 9,
        'Nov' => 10,
        'Dec' => 11,
);

# what month/year is it now ... we'll be assuming that the log
# we're parsing is within a month of this timestamp.  (bleh
# syslog sux0rs)
my ($month_now, $year_now) = (localtime(time))[4, 5];


LOG:
while (<>) {
	next LOG if /^... .. ..:..:.. \S+ syslogd ([0-9.#-]+): restart\.$/;

	if (/^... .. ..:..:.. \S+ last message repeated \d+ times$/) {
		if ($last_message_printed) {
			print;
		}
		next LOG;
	}
	$last_message_printed = 0;

	# -- MARK -- parsing
	if (m#^(... .. ..:..:..) (\S+) -- MARK --$#) {
		my ($mon, $mday, $hour, $min, $sec) = $1 =~ m#(...) (..) (..):(..):(..)#;
		my $year = $year_now;
		$mon = $months{$mon};
		if ($month_now == 0 && $mon == 11) {
			# assume we're parsing a log around the end of the year
			# and the year for this entry is really the previous year
			--$year;
		}
		my $timestamp = timelocal($sec,$min,$hour,$mday,$mon,$year);

		if (!defined($last_mark)) {
			$last_mark = $timestamp;
		}
		elsif ($timestamp - $last_mark > $mark_max_delta) {
			chomp;
			print "$_ !! last mark was " . ($timestamp - $last_mark) . " seconds ago!\n";
		}
		$last_mark = $timestamp;
		next LOG;
	}

	my ($date, $host, $daemon, $msg) =
		m#^(... .. ..:..:..) (\S+) ([^\s\[]+)(?:\[\d+\])?: (.*)$#;
	if (!defined($msg)) {
		warn "oddball: $_";
		next LOG;
	}

	if ($daemon eq 'in.ftpd' or $daemon eq 'ftpd') {
		next LOG if $msg =~ /^connect from \d+\.\d+\.\d+\.\d+$/;
	}

	elsif ($daemon eq 'sshd') {
		next LOG if $msg =~ /^Generating new 768 bit RSA key\.$/;
		next LOG if $msg =~ /^RSA key generation complete\./;
		next LOG if $msg =~ /^Could not reverse map address \S+$/;
		next LOG if $msg =~ /^packet_set_maxsize: setting to 4096$/;
		next LOG if $msg =~ /^Disconnecting: Command terminated on signal 1\.$/;
		next LOG if $msg =~ /^Connection closed by \S+$/;
		next LOG if $msg =~ /^reverse mapping checking getaddrinfo for /;
		next LOG if $msg =~ /^Address \S+ maps to \S+, but this does not map back to the address - POSSIBLE BREAKIN ATTEMPT!$/;
		next LOG if $msg =~ /^Setting tty modes failed: Invalid argument$/;

		# buh?
		next LOG if $msg =~ /^error: channel \d+: chan_shutdown_read: shutdown\(\) failed for fd\d+ \[i\d+ o\d+\]: Transport endpoint is not connected$/;

		# XXX: could track users here... but for now let's just ignore
		next LOG if $msg =~ /^Accepted (rsa|password|publickey) for \S+ from \S+ port \d+( ssh2)?$/;
		# XXX: buh?
		next LOG if $msg =~ m#^Warning: /home/[^/]+/.ssh/authorized_keys, line \d+: keysize mismatch: actual 1023 vs. announced 1024\.$#;
		# why is sshd so noisy?
		next LOG if $msg =~ m#^Received disconnect from \S+: \d+: All open channels closed$#;
		next LOG if $msg =~ m#^Received disconnect from \S+: \d+: Disconnect requested by Windows SSH Client\.$#;
		next LOG if $msg =~ m#^Received disconnect from \S+: \d+: No further authentication methods available\.$#;
		next LOG if $msg =~ m#^Received disconnect from \S+: Session canceled by user$#;
		next LOG if $msg =~ m#^channel \d+: rcvd big packet \d+, maxpack 16384$#;
		next LOG if $msg =~ m#^fatal: Read from socket failed: Connection reset by peer$#;
		next LOG if $msg =~ m#^fatal: buffer_get: trying to get more bytes than in buffer$#;
		next LOG if $msg =~ m#^Unknown message during authentication: type 4$#;
		next LOG if $msg =~ m#^fatal: Timeout before authentication for \S+$#;
		next LOG if $msg =~ m#^subsystem request for sftp$#;
		next LOG if $msg =~ m#^PAM pam_set_item: NULL pam handle passed$#;
	}

	elsif ($daemon eq 'named') {
		next LOG if $msg =~ /^"\S+ IN (NS|MX|A)" points to a CNAME/;
		next LOG if $msg =~ /^Lame server on /;
		next LOG if $msg =~ /^Cleaned cache of \d+ RRsets$/;
		next LOG if $msg =~ /^(USAGE|NSTATS|XSTATS) /;
		next LOG if $msg =~ /^(sysquery|ns_forw|ns_resp): query\([a-zA-Z0-9.-]+\) All possible A RR's lame$/;
		next LOG if $msg =~ /^(sysquery|ns_forw|ns_resp): query\([a-zA-Z0-9.-]+\) NS points to CNAME/;
		next LOG if $msg =~ /^(sysquery|ns_forw|ns_resp|ns_req): sendto\(/;
		next LOG if $msg =~ /^bad referral /;
		next LOG if $msg =~ /^slave zone "[^"]+" \(IN\) loaded \(serial \d+\)$/;
		next LOG if $msg =~ /^slave zone "[^"]+" expired$/;
		next LOG if $msg =~ /^dumping nameserver data$/;
		next LOG if $msg =~ /^finished dumping nameserver data$/;
		next LOG if $msg =~ /^approved AXFR from/;
		next LOG if $msg =~ /^zone transfer \(AXFR\) of/;
		next LOG if $msg =~ /^master zone.*loaded/;
		next LOG if $msg =~ /No default TTL set using SOA minimum instead$/;
		next LOG if $msg =~ /^Sent NOTIFY for /;
		next LOG if $msg =~ /^Received NOTIFY answer from /;
		next LOG if $msg =~ /^invalid RR type \S+ in (authority|additional) section/;
		next LOG if $msg =~ /^unrelated additional info/;
		next LOG if $msg =~ /^wrong ans. name/;
		# grr
		next LOG if $msg =~ /^rcvd NOTIFY for "jdougal\.com", name not one of our zones/;
		# XXX: figure out what these updates are coming from?
		my $domain;
		if (($domain) = $msg =~ /^(?:unapproved|denied) update from \S+ for (\S+)/) {
			$domain =~ s#"##g;
			$domain =~ tr/A-Z/a-z/;
			next LOG if defined($ignore_updates{$domain});
		}
		# ignore these lamers
		if (($domain) = $msg =~ /^(?:sysquery|ns_resp|ns_forw): query\((\S+)\) (?:contains our address|forwarding loop)/) {
			$domain =~ tr/A-Z/a-z/;
			while ($domain =~ /\./) {
				next LOG if (defined($contains_our_address{$domain}));
				$domain =~ s#^[^\.]+\.(.*)$#$1#;
			}
		}
		# save the notify thingers in case the next line indicates
		# its not one of our zones
		if ($msg =~ /^rcvd NOTIFY\(\S+, IN, SOA\) from \S+$/) {
			$save_notify = $_;
			next LOG;
		}
		if ($msg =~ /^rcvd NOTIFY for "[^"]*", name not one of our zones/) {
			print $save_notify;
		}
		# unexpected sources
		if ($msg =~ /Response from unexpected source \(\[([\d.]+)\]\.\d+\) for query "(\S+) IN (\S+)"/) {
			my $ip = $1;
			my $name = $2;
			my $type = $3;

			# i don't know what is issuing these ipv6 lookups, grr.
			next LOG if ($type eq 'AAAA' or $type eq 'A6');

			$name =~ tr/A-Z/a-z/;
			while ($name =~ /\./) {
				next LOG if (defined($ignore_unexpected_domains{$name}));
				$name =~ s#^[^\.]+\.(.*)$#$1#;
			}
		}
		# some lamer
		next LOG if $msg =~ /^Malformed response from \[213\.171\.(208|212)\.222\]\.53/;
	}

	elsif ($daemon eq 'named-xfer') {
		# XXX: could make sure this is a valid master from named.conf
		next LOG if $msg =~ /^send AXFR query 0 to /;
		next LOG if $msg =~ /^recv\(len=\d+\): Connection timed out$/;
		next LOG if $msg =~ /^connect\(\S+\) for zone \S+ failed: No route to host$/;
	}

	elsif ($daemon eq 'PAM_unix') {
		next LOG if $msg =~ /\(cron\)/;
		next LOG unless $msg =~ /session (opened|closed) for user root/;
	}

	elsif ($daemon eq '/USR/SBIN/CRON') {
		next LOG if $msg =~ /^\(\S+\) CMD /;
	}

	elsif ($daemon eq 'crontab') {
		if ($msg =~ /^\((\S+)\) LIST \((\S+)\)$/) {
			next LOG if ($1 eq $2);
		}
	}

	elsif ($daemon eq 'in.telnetd') {
		next LOG if $msg =~ /^connect from \d+\.\d+\.\d+\.\d+$/;
	}
	elsif ($daemon eq 'telnetd') {
		next LOG if $msg =~ /^ttloop:  peer died: Invalid or incomplete multibyte or wide character $/;
		next LOG if $msg =~ /^ttloop:  read: Connection reset by peer $/;
		next LOG if $msg =~ /^ttloop:  read: Connection timed out $/;
		next LOG if $msg =~ /^ttloop: peer died: EOF $/;
	}

	elsif ($daemon eq 'xntpd') {
		next LOG if $msg =~ /^kernel pll status change/;
	}

	elsif ($daemon eq 'login') {
		next LOG if $msg =~ /^FAILED LOGIN \S+ FROM \S+ FOR , User not known to the underlying authentication module$/;
	}

	elsif ($daemon eq 'inetd') {
		next LOG if $msg =~ /^pid \d+: exit status \d+$/
	}

	elsif ($daemon eq 'su') {
		next LOG if $msg =~ m#^\+ pts/\d+ atk-atk2$#;
	}

	$last_message_printed = 1;

	print;
}