1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
#############################################################################
# Copyright (c) 2015 Balabit
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License version 2 as published
# by the Free Software Foundation, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
# As an additional exemption you are allowed to compile & link against the
# OpenSSL libraries as published by the OpenSSL project. See the file
# COPYING for details.
#
#############################################################################
# Parse apache access.log
#
# Formats recognized:
#
# LogFormat "%v:%p %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
# virtualhost:443 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
#
# LogFormat "%v:%p %h %l %u %t \"%r\" %>s %b" vhost_common
# virtualhost:443 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
#
# LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined
# 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
#
# LogFormat "%h %l %u %t \"%r\" %>s %b" common
# 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
block parser apache-accesslog-parser-vhost(prefix() template()) {
channel {
filter { match("^[A-Za-z0-9\-\._]+:[0-9]+ " template(`template`)); };
parser {
csv-parser(
dialect(escape-backslash-with-sequences)
flags(strip-whitespace)
delimiters(" ")
template(`template`)
quote-pairs('""[]')
columns("2", "`prefix`clientip", "`prefix`ident",
"`prefix`auth", "`prefix`timestamp",
"`prefix`rawrequest", "`prefix`response",
"`prefix`bytes", "`prefix`referrer",
"`prefix`agent"));
csv-parser(
prefix(`prefix`)
template("$2")
delimiters(":")
dialect(escape-none)
columns("vhost", "port"));
};
};
};
# combined & common format without vhost
# LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
# LogFormat "%h %l %u %t \"%r\" %>s %b" common
block parser apache-accesslog-parser-combined(prefix() template()) {
channel {
parser {
csv-parser(
prefix(`prefix`)
dialect(escape-backslash-with-sequences)
flags(strip-whitespace)
delimiters(" ")
template(`template`)
quote-pairs('""[]')
columns("clientip", "ident", "auth",
"timestamp", "rawrequest", "response",
"bytes", "referrer", "agent"));
};
};
};
block parser apache-accesslog-parser(prefix(".apache.") template("${MESSAGE}")) {
# parse into a logstash-like schema
# https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns#L90
channel {
# parser for formats including vhost:port
if {
parser { apache-accesslog-parser-vhost(prefix(`prefix`) template(`template`)); };
# parser for standard formats
} else {
parser { apache-accesslog-parser-combined(prefix(`prefix`) template(`template`)); };
};
# mungle values to match Kibana/elastic schema and common to all
# supported formats.
parser {
date-parser(format("%d/%b/%Y:%H:%M:%S %z")
template("${`prefix`timestamp}"));
};
if {
# Sometimes the rawrequest is not a proper HTTP request (e.g. when someone
# submits an request like this):
#
# _default_:443 106.75.178.169 "-" - [22/Nov/2022:00:03:53 +0100] "{\"params\": [\"miner1\", \"bf\", \"00000001\", \"504e86ed\", \"b2957c02\"], \"id\": 4, \"method\": \"mining.submit\"}\n" 400 226 "-" "-"
parser {
csv-parser(
prefix(`prefix`)
template("${`prefix`rawrequest}")
delimiters(" ")
dialect(escape-none)
flags(strip-whitespace, drop-invalid)
columns("verb", "request", "httpversion"));
};
rewrite {
subst("^HTTP/(.*)$", "$1", value("`prefix`httpversion"));
};
};
};
};
|
