#!/usr/bin/perl -w
#
# logfilterd
#
# $Id: logfilterd,v 1.12 1999/04/07 04:45:33 adam Exp $
#
# (c) 1998 Adam Spiers <adam@spiers.net>
#
# Filters /var/log/messages for important stuff and takes 
# various appropriate actions on the results.
#
# This version has been written specially for thelonious.new.ox.ac.uk,
# but it could easily be adapted for other machines.
#
# Please see the accompanying files COPYRIGHT and COPYING for
# copyright details.
#

require 5.004;
use strict qw(vars);

# {{{ Import modules and symbols

use Carp qw(confess verbose);
use Net::Domain qw(hostname);
use Date::Manip;
use Set::IntSpan;
use Mail::Send;
use Fcntl qw(:flock);
use Getopt::Long;
use DB_File;

# }}}

my $test = 0;
my $email_to = 'logs';

# {{{ Store pid in /var/run/logfilterd.pid

if (!$test){
    my $pidfile = '/var/run/logfilterd.pid';
    open(PIDFILE, ">$pidfile") or die "open >$pidfile: $!";
    print PIDFILE $$;
    close(PIDFILE);
}

# }}}
# {{{ Process options

my %options = ();
GetOptions(\%options, 'clean', 'no-follow', 'no-email');

# }}}
# {{{ Define file locations

# These first few are self-explanatory:

my $test_dir = '/home/nmcgroga/perl/logfilterd';

my $input_log = '/var/log/messages';
$input_log = "$test_dir/messages" if $test;


my $output_dir = '/var/log';
$output_dir = "$test_dir" if $test;

my $output_log  = "$output_dir/filtered-messages";
my $error_log   = "$output_dir/logfilterd-errors";
my $sshd_log    = "$output_dir/sshd";
my $ftpd_log    = "$output_dir/ftpd";
my $ntpd_log    = "$output_dir/ntpd";
my $imapd_log   = "$output_dir/imapd";
my $popd_log    = "$output_dir/popd";
my $dhcpd_log   = "$output_dir/dhcpd";

# Anything to do with the printing subsystem
my $print_log   = "$output_dir/print";

# Abacus Sentry, port scan detection system
my $sentry_log  = "$output_dir/sentry";

# Strange stuff which isn't too worrying and which, while it might
# be worth investigating at some stage, isn't worth putting in
# the main output (left-overs) log file, $output_log.
my $strange_log = "$output_dir/strange";

# All IP firewalling
my $IP_fw       = "$output_dir/firewall";


# Location of file which contains timestamp of most recently
# filtered messages from input file
my $recent      = "/var/run/logfilterd.most-recent";
$recent      = "$output_dir/logfilterd.most-recent" if $test;

# Location of db file containing the hash of dhcp leases.
my $dhcpd_leases_db = "/var/run/logfilterd.dhcpd_leases.db";
$dhcpd_leases_db = "$output_dir/logfilterd.dhcpd_leases.db" if $test;

# }}}
# {{{ Initialise data

my $hostname   = hostname();
my %contexts   = ();
my $context_id = 0;

my %context_timeouts = (
			sentry  => &ParseDateDelta('5 minutes')
		       );

my $most_recent_logline_time;

my %dhcpd_leases;

# }}}
# {{{ Open file-handles

# Input file

open(LOG, $input_log) or die "Couldn't open $input_log.\n";

# Output files

use vars qw(*ERROR *OUTPUT 
	    *SSHD *FTPD *NTPD *PRINT *SENTRY *STRANGE *IP_FW *IMAPD *POPD
	    *DHCPD);

my %output_files = (
		    ERROR   =>  $error_log,
		    OUTPUT  =>  $output_log,
		    SSHD    =>  $sshd_log,
		    FTPD    =>  $ftpd_log,
		    NTPD    =>  $ntpd_log,
		    PRINT   =>  $print_log,
		    SENTRY  =>  $sentry_log,
		    STRANGE =>  $strange_log,
		    IP_FW   =>  $IP_fw,
		    IMAPD   =>  $imapd_log,
		    POPD    =>  $popd_log,
		    DHCPD   =>  $dhcpd_log,
		   );

if ($options{'clean'}) {
  unlink (values %output_files);
  exit 0;
}


foreach my $fh (keys %output_files) {
  my $file = $output_files{$fh};

  # Open file
  open($fh, ">>$file") or die "Couldn't open $file.\n";

  # flock file
  flock($fh, LOCK_EX) or die "Couldn't get exclusive lock on $file.\n";

  # Turn on auto-flush
  select((select($fh), $| = 1)[0]);

  print $fh ">>>>>>>>>>>>>>>>>>  logfilterd started on ",
	    &UnixDate(&ParseDate(scalar(localtime())), '%b %e %H:%M:%S'),
	    "  <<<<<<<<<<<<<<<<<<\n";
}

# Tie a db file to the hash of DHCP leases to ensure that we maintain
# state across restarts.
my $dhcpd_db_obj=tie(%dhcpd_leases, "DB_File", $dhcpd_leases_db);

# }}}
# {{{ Find out timestamp of most recently filtered messages from input file

my $ignore_threshold;

if (open(RECENT, $recent)) {
  $ignore_threshold = <RECENT>;
  close(RECENT);
}

# }}}
# {{{ Trap HUP, TERM and KILL signals

$SIG{'HUP'} = $SIG{'TERM'} = $SIG{'KILL'} = sub { 
  die "Caught a SIG$_[0]";	# die() will execute END block
};

# }}}
# {{{ Jump to first unfiltered line

&jump($ignore_threshold) if defined($ignore_threshold);

sub jump {
  my ($ignore_threshold) = @_;

  my $time_threshold = &ParseDate($ignore_threshold)
    or die "&jump: Couldn't parse threshold $ignore_threshold";

  my ($size, $blocksize, $blocks) = ((stat(LOG))[7, 11, 12]);
  $blocksize ||= 8192;
  die "stat: $!" unless defined($size) && defined($blocks);

  my ($min, $max) = (0, int($size / $blocksize));
  my ($mid, $time, $message);

  # Perform binary search with granularity of one block

  while ($max - $min > 1) {
    $mid = int(($max + $min) / 2);
    seek(LOG, $mid * $blocksize, 0);
    $_ = <LOG> if $mid;		# probably a partial line
    $_ = <LOG>;
    chomp;

    if (/^([A-Z][a-z]{2} [ 0-3][0-9] \d\d:\d\d:\d\d) $hostname /) {
      ($time, $message) = ($1, $');
    } else {
      die "&jump: couldn't parse line:\n$_";
    }

    my $logdate = &ParseDate($time)
      or die "&jump: Couldn't parse log line time $time";

    if ($logdate lt $time_threshold) {
      $min = $mid;
    } else {
      $max = $mid;
    }
  }

  # Perform linear search within block found by binary search

  $min *= $blocksize;
  seek(LOG, $min, 0);
  <LOG> if $min;      # probably a partial line if not at start of file

  while (<LOG>) {
    chomp;
    if (/^([A-Z][a-z]{2} [ 0-3][0-9] \d\d:\d\d:\d\d) $hostname /) {
      ($time, $message) = ($1, $');
    } else {
      die "&jump: couldn't parse line:\n$_";
    }

    my $logdate = &ParseDate($time)
      or die "&jump: Couldn't parse log line time $time";

    last if $logdate ge $time_threshold;

    $min = tell(LOG);		# store old position
  }

  seek(LOG, $min, 0);
}

# }}}

&process(\*LOG);

exit 0 if $options{'no-follow'};

while (1) {
  &process(\*LOG);
  sleep(1);
}

# {{{ END block

END {

  # Close input file

  close(LOG);

  # Close output files

  foreach my $fh (keys %output_files) {
    next unless defined fileno($fh);
    print $fh ">>>>>>>>>>>>>>>>>>  logfilterd stopped on ",
	      &UnixDate(&ParseDate(scalar(localtime())), '%b %e %H:%M:%S'),
	      "  <<<<<<<<<<<<<<<<<<\n";
    close($fh);
  }

  # Write out timestamp of most recently filtered message from input file

  if ($most_recent_logline_time) {
    open(RECENT, ">$recent") or die "Couldn\'t open $recent for writing: $!";
    print RECENT &ParseDate($most_recent_logline_time);
    close(RECENT);
  }

  # Untie the DHCP hash
  untie(%dhcpd_leases);
}

# }}}

##############################################################################
  
# {{{ &process (main filtering loop)

sub process {
  my $input = shift;

  my $message;

  while (<$input>) {
    chomp;

    if (/^([A-Z][a-z]{2} [ 0-3][0-9] \d\d:\d\d:\d\d) $hostname /) {
      ($most_recent_logline_time, $message) = ($1, $');
      &filter_line($most_recent_logline_time, $message);
    } else {
      &error("Warning: couldn't parse line:\n$_");
    }
  }
}

# }}}
# {{{ &filter_line

# {{{ process_dhcpd

sub process_dhcpd {
    my $time = shift;
    $_ = shift;
    my $msg = '';

    /^dhcpd: DHCPACK on (.*) to (.*) via/ and do {
	my ($ip_addr, $mac_addr) = ($1, $2);
	if (!$dhcpd_leases{$ip_addr} || 
	    $dhcpd_leases{$ip_addr} ne $mac_addr) {
	    $msg = "$time $ip_addr to $mac_addr\n";
	    $dhcpd_leases{$ip_addr} = $mac_addr;
	}
    };

    return $msg;
}

# }}}
     
sub filter_line {
  my $time = shift;
  $_ = shift;

  study;

  # Log all messages specific to certain services

  print SSHD    "$time $_\n" if /sshd/;
  print FTPD    "$time $_\n" if /ftpd/;
  print NTPD    "$time $_\n" if /ntpd/;
  print PRINT   "$time $_\n" if /^(nprintd|filter|lpd)\[\d+\]/;
  print SENTRY  "$time $_\n" if /^abacus_sentry/;
  print IP_FW   "$time $_\n" if /^IP fw-in acc eth0 /;
  print IMAPD   "$time $_\n" if /^imapd/;
  print POPD	"$time $_\n" if /^ipop[23]d/;
  print DHCPD   &process_dhcpd($time, $_) if /^dhcpd/;

  # icmplog/mon

  /^icmplog: echo reply/ and return;

  /^icmplog:\ echo\ (reply\ )?from\ (
      thelonious.new.ox.ac.uk		|
      router173.new.ox.ac.uk		|
      router.new.ox.ac.uk		|
      newjcr.new.ox.ac.uk		|
      163.1.145.245			|
      163.1.145.246			|
      163.1.145.252 
    )
  /x and return;

  # qmail

  /^(qmail|splogger):\ \d{9}\.\d{6}\ (
      delivery\ \d+:\ success:.*procmail
    )
  /x and do {
    print OUTPUT "$time $_\n";
    return;
  };

  /^(qmail|splogger):\ \d{9}\.\d{6}\ (
      new\ msg\ \d+ |
      info\ msg\ \d+:\ bytes\ \d+\ from\ <.*>\ qp\ \d+\ uid\ \d+  |
      starting\ delivery\ \d+:\ (
        msg\ \d+\ to\ (local|remote)\ 
      )								  |
      delivery\ \d+:\ (
        success:\ (
          did_						|
          .*_accepted_message\.\/Remote_host_said:_250	|
          msg\ \d+\ to\ (local|remote)\ 
        )|
	deferral:\ (
          Sorry,_I_wasn\'t_able_to_establish_an_
            SMTP_connection\._\(\#4\.4\.1\)\/				  |
          Sorry,_I_couldn\'t_find_any_host_by_that_name\._\(\#4\.1\.2\)\/ |
	  CNAME_lookup_failed_temporarily\._\(\#4\.4\.3\)\/		  |
          Connected_to_.*_but_greeting_failed\.\/
        )|
        failure:\ (
          Sorry,_no_mailbox_here_by_that_name\._\(\#5\.1\.1\)\/	      |
          Sorry,_I_couldn\'t_find_any_host_named.*\._\(\#5\.1\.2\)\/
        )
      )|
      bounce\ msg\ \d+\ qp\ \d+ |
      end\ msg\ \d+
    )
  /x and return;

  # Ignore loads of common boring stuff

  # dhcpd -- bin it all!
  /^dhcpd:/ and return;

  # ident

  /^identd\[\d+\]: Returned: \d+ , 113 : NO-USER/ and do {
    print STRANGE "$time $_\n";
    return;
  };

  /^identd\[\d+\]:\ (
      Connection\ from\ 			|
      Successful\ lookup:\ 			|
      from:\ [\d.]+\ \(\ .*\ \)\ for:\ 	 	|
      Returned:\ \d+\ ,\ \d+\ :\ NO-USER
    )
  /x and return;

  # sshd

  /^sshd\[\d+\]:\ log:\ (
      Server\ listening\ on\ port\ 22\.					|
      Generating\ (new\ )?768\ bit\ RSA\ key\.				|
      RSA\ key\ generation\ complete\.					|
      Rhosts\ with\ RSA\ host\ authentication\ accepted\ for\ 		|
      Connection\ from\ 						|
      executing\ remote\ command\ as\ 					|
      (RSA|Password)\ authentication\ for\ \w+\ accepted\.		|
      fwd\ X11\ connect\ from\ 						|
      Closing\ connection\ to\ 						|
      RhostsRsa\ authentication\ not\ available\ for\ connections\ 
        from\ unprivileged\ port\.					|
      Could\ not\ reverse\ map\ address\ [\d.]+				|
      ROOT\ LOGIN\ as\ 'root'\ from\ miles\.new\.ox\.ac\.uk
    )
  /x and return;

  /^(sshd\[\d+\]|syslog):\ fatal:\ (
      Local:\ Command\ terminated\ on\ signal\ 9\.			|
      Connection\ closed\ by\ remote\ host\.				|
      Read\ error\ from\ remote\ host:\ (
        Connection\ reset\ by\ peer		|
        Connection\ timed\ out
      )									|
      Timeout\ before\ authentication\.
    )
  /x and return;

  # PAM_pwdb
  
  /^PAM_pwdb\[\d+\]:\ (
      \(\w+\)\ session\ (open|clos)ed\ for\ user\ |
      \d+\ authentication\ failures?;\ 	  |
      check\ pass;\ user\ unknown		  |
      bad\ username
    )     
  /x and return;

  # syslog

  /^syslog:\ log:\ (
      fwd\ X11\ connect\ from\ 			|
      Password\ authentication\ for\ \w+\ accepted\.	|
      Closing\ connection\ to\ 
    )
  /x and return;     

  # logins

  /^(login(\[\d+\])?|syslog):\ (
      LOGIN\ ON\ (
        tty[p-z]				|
        tty\d\ BY\ (adam|nick|nmcgroga|stephen|ben|cclading)
      )|
      failed\ login\ from\ |
      FAILED\ LOGIN\ .*\ FROM\ .*\ FOR\ .*,\ (
        User\ not\ known\ to\ the\ underlying\ authentication\ module     |
        Error\ in\ service\ module				          |
        Authentication\ service\ cannot\ retrieve\ authentication\ info\. |
        Authentication\ failure
      )
    )
  /x and return;


  # Miscellaneous messages from various services

  /^(in\.(telnet|finger|n?talk|ftp)|imap|ipop[23])d\[\d+\]: connect from / 
    and return;

  /^talkd\[\d+\]: recv: Connection refused/ and return;

  /^fingerd\[\d+\]: fingered / and return;

  /^portmap\[\d+\]: connect from [\d.]+ to ((un)?set|getport)\((nfs|mountd)\)/
    and return;

  m@^pttyd\[\d+\]: request for /dev/pty@ and return;

  /^ntpd\[\d+\]:\ (
      ntpd\ 4\.0						|
      tickadj\ =\ 1,\ tick\ =\ 10000,\ tvu_maxslew\ =\ 99	|
      precision\ =\ \d+\ usec					|
      synchronized\ to						|
      kernel\ pll\ status\ change
    )
  /x and return;

  /^imapd\[\d+\]:\ (
      (
        Authenticated				|
        Killed\ \(lost\ mailbox\ lock\)		|
        (Connection\ reset\ by\ peer|command\ stream\ end\ of\ file)
          ,\ while\ reading\ line		|
        Autologout
      )\ user=.*\ host=
    )
  /x and return;

  /^ipop3d\[\d+\]: (Login user|Logout user)[= ].*?host[= ]/ and return;

  # Ignore printing subsystem for now

  /^(nprintd|filter|lpd)\[\d+\]/ and return;

  # ftpd

  /^syslog: .*FTP session closed/ and return;

  /^ftpd: .*connected: (IDLE|QUIT)/ and return;

  /^ftpd(\[\d+\])?:\ (
      .*(ANONYMOUS\ )?FTP\ LOGIN\ FROM |
      User\ .*\ timed\ out\ after      |
      .*\ of\ .*\ \[.*\]\ deleted\     |
      .* FTP\ session\ closed
    )
  /x and return;
  
  # Broken DNS entries

  /^syslog: gethostby\*\.getanswer: asked for ".*", got / and return;

  # Ignore all mon stuff

  /^mon\[\d+\]: / and return;

  # Abacus Sentry

  /^abacus_sentry\[\d+\]:\ attackalert:\ (
      UDP\ scan\ from\ host:\ (
        ellington\.new\.ox\.ac\.uk\/163\.1\.145\.58\ to\ UDP\ port:\ 32770 |
	0\.0\.0\.0\/0\.0\.0\.0\ to\ UDP\ port:\ 67			   |
        (.*\.new\.ox\.ac\.uk|163\.1\.145\.\d+)
          \/163\.1\.145\.\d+\ to\ UDP\ port:\ (7|68|13[7-9])
      )|
      Host:\ .*\ is\ already\ blocked\ Ignoring			   |
      Ignoring\ (TCP|UDP)\ response\ per\ configuration\ file\ setting\.
    )
  /x and return;

  /^abacus_sentry\[\d+\]:\ adminalert:\ (
#     ERROR:\ Socket\ \d+ is\ in\ use\ and\ will\ not\ be\ monitored\.\ 
#       Attemping\ to\ continue				        |
#     Sentry\ [\d.]+\ is\ starting\.				|
      Advanced\ mode\ will\ monitor\ first\ \d+\ ports		|
      Advanced\ mode\ will\ manually\ exclude\ port		|
      Going\ into\ stealth\ listen\ mode\ on			|
      Advanced\ Stealth\ scan\ detection\ mode\ activated\.\ 
        Ignored\ TCP\ port:\ \d+
    )
  /x and return; 

  # Check for context time-outs

  if (exists $contexts{'sentry'}) {
    foreach my $host (keys %{$contexts{'sentry'}}) {
      my $context = $contexts{'sentry'}{$host};

      my $last_msg_time = $context->{'last-msg-time'};
      my $plus = &DateCalc($last_msg_time, $context_timeouts{'sentry'});

      if (&ParseDate($time) gt $plus) {
	&finish_sentry_context($time, $host);
      }
    }
  }

  /^abacus_sentry\[\d+\]:\ attackalert:\ 
      (UDP|SYN\/Normal|FIN\ stealth)\ scan\ from\ host:\ 
        ([^\/ ]+?\/[^\/ ]*?)\ to\ (TCP|UDP)\ port:\ (\d+)
  /x and do {
    my ($scan_type, $source_host, $protocol, $port) = ($1, $2, $3, $4);

    # Search for existing contexts

    if (exists $contexts{'sentry'}{$source_host}) {
      # Found context; add message to it

      my $context = $contexts{'sentry'}{$source_host};
      $context->{'last-msg-time'} = &ParseDate($time);
      push @{$context->{'dest-ports'}{$scan_type}}, "$protocol:$port";
      return;

    } else {

      # No context found; create one

      $contexts{'sentry'}{$source_host} = { id => ++$context_id };

      my $new_context = $contexts{'sentry'}{$source_host};
      $new_context->{'start-time'}    = &ParseDate($time);
      $new_context->{'last-msg-time'} = &ParseDate($time);
      $new_context->{'dest-ports'}{$scan_type} = [ "$protocol:$port" ];

      print OUTPUT "$time logfilterd[sentry/$context_id]: ",
                   "Detected port scan initiating from $source_host\n";
      return;
    }
  };

  # Kernel IP firewalling false alarms

  /^kernel:\ IP\ fw-in\ acc\ eth0\ (
      UDP\ 0\.0\.0\.0:68\ 255\.255\.255\.255:67\ 	|
      TCP\ (163\.1\.145\.171|192.76.27.252):\d+\ 163\.1\.145\.129:(143)
    )
  /x and return;

  # Miscellaneous

  /^last message repeated \d+ times$/ and return;

  # Default action

  print OUTPUT "$time $_\n";
}

# }}}

# {{{ &finish_sentry_context

sub finish_sentry_context {
  my ($time, $host) = @_;

  my $context = $contexts{'sentry'}{$host};
  my $start_time = &UnixDate($context->{'start-time'},    '%b %e %H:%M:%S');
  my $end_time   = &UnixDate($context->{'last-msg-time'}, '%b %e %H:%M:%S');

  my $email    = '';
  my $loglines = '';

  $email .= <<"EOF";
Port scan (context id \#$context->{id}) detected from host:
   $host,

starting at $start_time, and 
ending   at $end_time.

EOF

  foreach my $scan_type (keys %{$context->{'dest-ports'}}) {
    my $ports = $context->{'dest-ports-IntSpan'}{$scan_type} = {};

    # Add all destination ports to sets, one set for each protocol
    foreach my $port (@{$context->{'dest-ports'}{$scan_type}}) {
      my ($protocol, $number) = split(/:/, $port);
      if (exists $ports->{$protocol}) {
	insert { $ports->{$protocol} } $number;
      } else {
	$ports->{$protocol} = new Set::IntSpan $number;
      }
    }

    # Print out run list for each's protocol's ports
    foreach my $protocol (keys %$ports) {
      my $run = $ports->{$protocol}->run_list;
      $run =~ s/,/, /g;
      $run =~ s/-/--/g;
      $email .= "$scan_type scan employed on $protocol ports: $run\n";
      $loglines .= "$time logfilterd[sentry/$context->{id}]: " .
		   "$scan_type scan employed on $protocol ports: $run\n";
    }  
  }

  $loglines .= "$time logfilterd[sentry/$context->{id}]: " .
               "Port scan from $host finished at " .
	       &UnixDate($context->{'last-msg-time'}, '%b %e %H:%M:%S') .
	       "\n";


  print OUTPUT $loglines;

  my $send_email_alert = 0;

  # Do not send e-mail for singleton scans to port 7

 CHECK:
  foreach my $scan_type (keys %{$context->{'dest-ports-IntSpan'}}) {
    foreach my $protocol (keys 
			  %{$context->{'dest-ports-IntSpan'}{$scan_type}}
			 )
    {
      $send_email_alert = 1, last CHECK
	if ! equal
	       { $context->{'dest-ports-IntSpan'}{$scan_type}{$protocol} }
	       7;
    }
  }

  if ($send_email_alert && ! $options{'no-email'}) {
    my $msg = 
      new Mail::Send (
		      Subject => 'logfilterd ALERT: port scan detection',
		      To      => $email_to,
		     );

    my $fh = $msg->open;
    print $fh $email;
    $fh->close;
  }

  # Remove context
  delete $contexts{'sentry'}{$host};
}

# }}}

# {{{ &error

sub error {
  my ($error) = @_;

  print ERROR  substr(localtime(), 0, 19), " $error\n";
  print STDERR substr(localtime(), 0, 19), " $error\n" if $test;
}

# }}}