#!/usr/bin/perl -w
#
# httpd-guardian - detect DoS attacks by monitoring requests
# Apache Security, http://www.apachesecurity.net
# Copyright (C) 2005 Ivan Ristic <ivanr@webkreator.com>
#
# $Id: httpd-guardian,v 1.6 2005/12/04 11:30:35 ivanr Exp $
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#

# This script is designed to monitor all web server requests through
# the piped logging mechanism. It keeps track of the number of requests
# sent from each IP address. Request speed is calculated at 1 minute and
# 5 minute intervals. Once a threshold is reached, httpd-guardian can
# either emit a warning or execute a script to block the IP address.
#
# Error message will be sent to stderr, which means they will end up
# in the Apache error log.
#
# Usage (in httpd.conf)
# ---------------------
#
# Without mod_security, Apache 1.x:
#
#   LogFormat "%V %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %{UNIQUE_ID}e \"-\" %T 0 \"%{modsec_message}i\" 0" guardian
#   CustomLog "|/path/to/httpd-guardian" guardian
#
# or without mod_security, Apache 2.x:
#
#   LogFormat "%V %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %{UNIQUE_ID}e \"-\" %T %D \"%{modsec_message}i\" 0" guardian
#   CustomLog "|/path/to/httpd-guardian" guardian
#
# or with mod_security (better):
#
#   SecGuardianLog "|/path/to/httpd-guardian"
#
# NOTE: In order for this script to be effective it must be able to
#       see all requests coming to the web server. This will not happen
#       if you are using per-virtual host logging. In such cases either
#       use the ModSecurity 1.9 SecGuardianLog directive (which was designed
#       for this very purpose).
#
#
# Usage (with Spread)
# -------------------
#
# 1) First you need to make sure you have Spread running on the machine
#    where you intend to run httpd-guardian on.
#
# 2) Then uncomment line "use Spread;" in this script, and change
#    $USE_SPREAD to "1".
#
# 3) The default port for Spread is 3333. Change it if you want to
#    and then start httpd-guardian. We will be looking for messages
#    in the Spread group called "httpd-guardian".

# TODO Add support to ignore certain log entries based on a
#      regex applied script_name.
#
# TODO Warn about session hijacking.
#
# TODO Track ip addresses, sessions, and individual users.
#
# TODO Detect status code anomalies.
#
# TODO Track accesses to specific pages.
#
# TODO Open proxy detection.
#
# TODO Check IP addresses with blacklists (e.g.
#      http://www.spamhaus.org/XBL/).
#
# TODO Is there a point to keep per-vhost state?
#
# TODO Enhance the script to tail a log file - useful for test
#      runs, in preparation for deployment.
#
# TODO Can we track connections as Apache creates and destroys them?
#
# TODO Command-line option to support multiple log formats. E.g. common,
#      combined, vcombined, guardian.
#
# TODO Command-line option not to save state
#

use strict;
use Time::Local;
# SPREAD UNCOMMENT
# use Spread;


# -- Configuration----------------------------------------------------------

my $USE_SPREAD = 0;
my $SPREAD_CLIENT_NAME = "httpd-guardian";
my $SPREAD_DAEMON = "3333";
my $SPREAD_GROUP_NAME = "httpd-guardian";
my $SPREAD_TIMEOUT = 1;

# If defined, execute this command when a threshold is reached
# block the IP address for one hour.
# $PROTECT_EXEC = "/sbin/blacklist block %s 3600";
# $PROTECT_EXEC = "/sbin/samtool -block -ip %s -dur 3600 snortsam.example.com";
#my $PROTECT_EXEC;

# For testing only:
my $PROTECT_EXEC = "/usr/bin/logger Possible DoS Attack from %s";

# Max. speed allowed, in requests per
# second, measured over an 1-minute period
#my $THRESHOLD_1MIN = 2; # 120 requests in a minute

# For testing only:
my $THRESHOLD_1MIN = 0.01;

# Max. speed allowed, in requests per
# second, measured over a 5-minute period
my $THRESHOLD_5MIN = 1; # 360 requests in 5 minutes

# If defined, httpd-guardian will make a copy
# of the data it receives from Apache
# $COPY_LOG = "";
my $COPY_LOG;

# Remove IP address data after a 10-minute inactivity
my $STALE_INTERVAL = 400;

# Where to save state (at this point only useful
# for monitoring what the script does)
my $SAVE_STATE_FILE = "/tmp/httpd-guardian.state";

# How often to save state (in seconds).
my $SAVE_STATE_INTERVAL = 10;

my $DEBUG = 0;


# -----------------------------------------------------------------

my %months = (
    "Jan" => 0,
    "Feb" => 1,
    "Mar" => 2,
    "Apr" => 3,
    "May" => 4,
    "Jun" => 5,
    "Jul" => 6,
    "Aug" => 7,
    "Sep" => 8, 
    "Oct" => 9,
    "Nov" => 10,
    "Dec" => 11
);

# -- log parsing regular expression


# 127.0.0.1 192.168.2.11 - - [05/Jul/2005:16:56:54 +0100]
# "GET /favicon.ico HTTP/1.1" 404 285 "-"
# "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.8) Gecko/20050511 Firefox/1.0.4"
# - "-" 0 0 "-" 0

my $logline_regex = "";

# hostname
$logline_regex .= "^(\\S+)";
# remote host, remote username, local username
$logline_regex .= "\\ (\\S+)\\ (\\S+)\\ (\\S+)";
# date, time, and gmt offset
$logline_regex .= "\\ \\[([^:]+):(\\d+:\\d+:\\d+)\\ ([^\\]]+)\\]";
# request method + request uri + protocol (as one field)
$logline_regex .= "\\ \"(.*)\"";
# status, bytes out
$logline_regex .= "\\ (\\d+)\\ (\\S+)";
# referer, user_agent
$logline_regex .= "\\ \"(.*)\"\\ \"(.*)\"";
# uniqueid, session, duration, duration_msec
$logline_regex .= "\\ (\\S+)\\ \"(.*)\"\\ (\\d+)\\ (\\d+)";
# modsec_message, modsec_rating
$logline_regex .= "\\ \"(.*)\"\\ (\\d+)";

# the rest (always keep this part of the regex)
$logline_regex .= "(.*)\$";

my $therequest_regex = "(\\S+)\\ (.*?)\\ (\\S+)";

# use strict
my %ipaddresses = ();
my %request;
my $current_time;
my $last_state_save;

sub parse_logline {
    $_ = shift;

    my %request = ();
    $request{"invalid"} = 0;

    my @parsed_logline = /$logline_regex/x;
    if (@parsed_logline == 0) {
        return (0,0);
    }

    (
        $request{"hostname"},
        $request{"remote_ip"},
        $request{"remote_username"},
        $request{"username"},
        $request{"date"},
        $request{"time"},
        $request{"gmt_offset"},
        $request{"the_request"},
        $request{"status"},
        $request{"bytes_out"},
        $request{"referer"},
        $request{"user_agent"},
        $request{"unique_id"},
        $request{"session_id"},
        $request{"duration"},
        $request{"duration_msec"},
        $request{"modsec_message"},
        $request{"modsec_rating"},
        $request{"the_rest"}
    ) = @parsed_logline;

    if ($DEBUG == 2) {
        print "\n";
        print "hostname = " . $request{"hostname"} . "\n";
        print "remote_ip = " . $request{"remote_ip"} . "\n";
        print "remote_username = " . $request{"remote_username"} . "\n";
        print "username = " . $request{"username"} . "\n";
        print "date = " . $request{"date"} . "\n";
        print "time = " . $request{"time"} . "\n";
        print "gmt_offset = " . $request{"gmt_offset"} . "\n";
        print "the_request = " . $request{"the_request"} . "\n";
        print "status = " . $request{"status"} . "\n";
        print "bytes_out = " . $request{"bytes_out"} . "\n";
        print "referer = " . $request{"referer"} . "\n";
        print "user_agent = " . $request{"user_agent"} . "\n";
        print "unique_id = " . $request{"unique_id"} . "\n";
        print "session_id = " . $request{"session_id"} . "\n";
        print "duration = " . $request{"duration"} . "\n";
        print "duration_msec = " . $request{"duration_msec"} . "\n";
        print "modsec_message = " . $request{"modsec_message"} . "\n";
        print "modsec_rating = " . $request{"modsec_rating"} . "\n";
        print "\n\n";
    }

    # parse the request line
    $_ = $request{"the_request"};
    my @parsed_therequest = /$therequest_regex/x;
    if (@parsed_therequest == 0) {
        $request{"invalid"} = "1";
        $request{"request_method"} = "";
        $request{"request_uri"} = "";
        $request{"protocol"} = "";
    } else {
        (
            $request{"request_method"},
            $request{"request_uri"},
            $request{"protocol"}
        ) = @parsed_therequest;
    }

    if ($request{"bytes_out"} eq "-") {
        $request{"bytes_out"} = 0;
    }

    # print "date=" . $request{"date"} . "\n";
    (
        $request{"time_mday"},
        $request{"time_mon"},
        $request{"time_year"}
    ) = ( $request{"date"} =~ m/^(\d+)\/(\S+)\/(\d+)/x );

    # print "time=" . $request{"time"} . "\n";
    (
        $request{"time_hour"},
        $request{"time_min"},
        $request{"time_sec"}
    ) = ( $request{"time"} =~ m/(\d+):(\d+):(\d+)/x );

    $request{"time_mon"} = $months{$request{"time_mon"}};

    $request{"time_epoch"} = timelocal(
        $request{"time_sec"},
        $request{"time_min"},
        $request{"time_hour"},
        $request{"time_mday"},
        $request{"time_mon"},
        $request{"time_year"}
    );

    # print %request;

    my $offset = index($request{"request_uri"}, "?");
    if ($offset != -1) {
        $request{"script_name"} = substr($request{"request_uri"}, 0, $offset);
        $request{"query_string"} = substr($request{"request_uri"}, $offset + 1);
    } else {
        $request{"script_name"} = $request{"request_uri"};
        $request{"query_string"} = "";
    }

    $request{"request_uri"} =~ s/\%([A-Fa-f0-9]{2})/pack('C', hex($1))/seg;
    $request{"query_string"} =~ s/\%([A-Fa-f0-9]{2})/pack('C', hex($1))/seg;

    return %request;
}

sub update_ip_address() {
    my $ipd = $ipaddresses{$request{"remote_ip"}};
    if (defined($$ipd{"counter"})) {
        $$ipd{"counter"} = $$ipd{"counter"} + 1;

        if ($DEBUG) {
            print STDERR "httpd-guardian: Incrementing counter for " . $request{"remote_ip"} . " (" . $$ipd{"counter"} . ")\n";
        }

        my($exec) = 0;

        # check the 1 min counter
        if ($current_time - $$ipd{"time_1min"} > 60) {
            # check the counters
            my $speed = ($$ipd{"counter"} - $$ipd{"counter_1min"}) / ($current_time - $$ipd{"time_1min"});
            if ($speed > $THRESHOLD_1MIN) {
                print STDERR "httpd-guardian: IP address " . $ipaddresses{$request{"remote_ip"}} . " reached the 1 min threshold (speed = $speed req/sec, threshold = $THRESHOLD_1MIN req/sec)\n";
                $exec = 1;
            }

            # reset the 1 min counter
            $$ipd{"time_1min"} = $current_time;
            $$ipd{"counter_1min"} = $$ipd{"counter"};
        }

        # check the 5 min counter
        if ($current_time - $$ipd{"time_5min"} > 360) {
            # check the counters
            my $speed = ($$ipd{"counter"} - $$ipd{"counter_5min"}) / ($current_time - $$ipd{"time_5min"});
            if ($speed > $THRESHOLD_5MIN) {
                print STDERR "httpd-guardian: IP address " . $request{"remote_ip"} . " reached the 5 min threshold (speed = $speed req/sec, threshold = $THRESHOLD_5MIN req/sec)\n";
                $exec = 1;
            }

            # reset the 5 min counter
            $$ipd{"time_5min"} = $current_time;
            $$ipd{"counter_5min"} = $$ipd{"counter"};
        }
    
        if (($exec == 1)&&(defined($PROTECT_EXEC))) {
            my $cmd = sprintf($PROTECT_EXEC, $request{"remote_ip"});
            print STDERR "httpd-guardian: Executing: $cmd\n";
            system($cmd);
        }

    } else {
        # start tracking this email address
        my %ipd = ();
        $ipd{"counter"} = 1;
        $ipd{"counter_1min"} = 1;
        $ipd{"time_1min"} = $current_time;
        $ipd{"counter_5min"} = 1;
        $ipd{"time_5min"} = $current_time;
        $ipaddresses{$request{"remote_ip"}} = \%ipd;
    }
}

sub process_log_line {
    update_ip_address();
}

sub remove_stale_data {
    while(my($key, $value) = each(%ipaddresses)) {
        if ($current_time - $$value{"time_1min"} > $STALE_INTERVAL) {
            if ($DEBUG) {
                print STDERR "httpd-guardian: Removing key $key\n";
            }
            delete($ipaddresses{$key});
        }
    }
}

sub save_state {
    if (!defined($SAVE_STATE_FILE)) {
        return;
    }

    if (!defined($last_state_save)) {
        $last_state_save = 0;
    }

    if ($current_time - $last_state_save > $SAVE_STATE_INTERVAL) {
        open(FILE, ">$SAVE_STATE_FILE") || die("Failed to save state to $SAVE_STATE_FILE");
        print FILE "# $current_time\n";
        print FILE "# IP Address\x09Counter\x09\x091min (time)\x095min (time)\n";
        while(my($key, $value) = each(%ipaddresses)) {
            print FILE ("$key" . "\x09" . $$value{"counter"} . "\x09\x09" . $$value{"counter_1min"} . " (" . $$value{"time_1min"} . ")\x09" . $$value{"counter_5min"} . " (" . $$value{"time_5min"} . ")\n");
        }
        close(FILE);
        $last_state_save = $current_time;
    }
}

# load state from $SAVE_STATE_FILE, store the data into $ipaddresses
sub load_state {
    return unless ( defined $SAVE_STATE_FILE );
    return unless ( -e $SAVE_STATE_FILE && -r $SAVE_STATE_FILE );
    open my $fd, "<", $SAVE_STATE_FILE
        or die "cannot open state file for reading : $SAVE_STATE_FILE : $!";
    while (<$fd>) {
        s/^\s+//;
        next if /^#/;
        #--------------------------------------------------
        # # 1133599679
        # # IP Address  Counter     1min (time)         5min (time)
        # 211.19.48.12  396         396 (1133599679)    395 (1133599379)
        #-------------------------------------------------- 
        my ($addr, $counter, $time1, $time5) = split /\t+/, $_; # TAB
        my ($counter_1min, $time_1min) = split /\s+/, $time1;
        my ($counter_5min, $time_5min) = split /\s+/, $time5;
        $ipaddresses{$addr} = {
            counter         => $counter,
            counter_1min    => $counter_1min,
            time_1min       => chop_brace($time_1min),
            counter_5min    => $counter_5min,
            time_5min       => chop_brace($time_5min),
        }
    }
    close $fd;
}

# return strings between braces
sub chop_brace {
	my $str = shift;
	$str =~ /\((.*)\)/;
	return $1;
}
sub process_line {
    my $line = shift(@_);

    if (defined($COPY_LOG)) {
        print COPY_LOG_FD $line;
    }

    if ($DEBUG) {
        print STDERR "httpd-guardian: Received: $line";
    }

    %request = parse_logline($line);
    if (!defined($request{0})) {
        # TODO verify IP address is in correct format

        # extract the time from the log line, to allow the
        # script to be used for batch processing too
        $current_time = $request{"time_epoch"};

        remove_stale_data();
        process_log_line();
        save_state();
    } else {
        print STDERR "Failed to parse line: " . $line;
    }
}

# -----------------------------------

load_state();
if (defined($COPY_LOG)) {
    open(COPY_LOG_FD, ">>$COPY_LOG") || die("Failed to open $COPY_LOG for writing");
    # enable autoflush on the file descriptor
    $| = 1, select $_ for select COPY_LOG_FD;  
}

if ($USE_SPREAD) {
    my($sperrno);
    my %args;

    $args{"spread_name"} = $SPREAD_DAEMON;
    $args{"private_name"} = $SPREAD_CLIENT_NAME;

    my($mbox, $privategroup) = Spread::connect(\%args);
    if (!defined($mbox)) {
        die "Failed to connect to Spread daemon: $sperrno\n";
    }

    Spread::join($mbox, $SPREAD_GROUP_NAME);

    for(;;) {
        my($st, $s, $g, $mt, $e, $msg);
        while(($st, $s, $g, $mt, $e, $msg) = Spread::receive($mbox, $SPREAD_TIMEOUT)) {
            if ((defined($st))&&($st == 2)&&(defined($msg))) {
                process_line($msg . "\n");
            }
        }
    }

} else {
    while(<STDIN>) {
        process_line($_);    
    }
}

if (defined($COPY_LOG)) {
    close(COPY_LOG_FD);
}