3 ###########################################################################
7 # Purpose: nf2csv parses iptables log messages and prints them on stdout
8 # in comma separated value format. This is most useful for
9 # generating data for the AfterGlow project
10 # (http://afterglow.sourceforge.net) to visualize iptables log
13 # nf2csv is part of the psad project (http://www.cipherdyne.org/psad)
15 # Author: Michael Rash (mbr@cipherdyne.org)
17 # Credits: (see the CREDITS file)
19 # Copyright (C) 2006 Michael Rash (mbr@cipherdyne.org)
21 # License (GNU Public License):
23 # This program is distributed in the hope that it will be useful,
24 # but WITHOUT ANY WARRANTY; without even the implied warranty of
25 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 # GNU General Public License for more details.
28 # You should have received a copy of the GNU General Public License
29 # along with this program; if not, write to the Free Software
30 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
33 ###########################################################################
36 use Getopt::Long 'GetOptions';
39 my $version = 'psad-2.3-pre1';
41 ### regex to match an ip address
42 my $ip_re = qr|(?:[0-2]?\d{1,2}\.){3}[0-2]?\d{1,2}|;
44 ### main packet data structure
50 'intf' => '', ### FIXME in and out interfaces?
77 ### extra fields for internals (fwsnort sid matching,
78 ### iptables logging prefixes and chains, etc.)
87 my $csv_print_uniq = 0;
88 my $csv_line_limit = 0;
89 my $csv_start_line = 0;
92 my $csv_neg_regex = 0;
98 ### make Getopts case sensitive
99 Getopt::Long::Configure('no_ignore_case');
101 &usage(1) unless (GetOptions(
102 'Messages-file=s' => \$nf_log_file, # Specify the path to file containing
103 'fields=s' => \$csv_fields, # Specify list of CSV fields.
104 'uniq-lines' => \$csv_print_uniq, # Only print unique lines in CSV
106 'max-lines=i' => \$csv_line_limit, # Limit the number of CSV output
108 'start-line=i' => \$csv_start_line, # Starting line in CSV file.
109 'end-line=i' => \$csv_end_line, # Ending line in CSV file.
110 'regex=s' => \$csv_regex, # Require additional regex match.
111 'neg-regex=s' => \$csv_neg_regex, # Require additional negative regex
112 'debug' => \$debug, # Run in debug mode.
113 'Version' => \$print_ver, # Print the nf2csv version and exit.
114 'help' => \$help, # Display help.
118 ### Print the version number and exit if -V given on the command line.
120 print "[+] psad v$version by Michael Rash <mbr\@cipherdyne.org>\n";
124 ### see what we should be searching for
125 my ($tokens_ar, $match_criteria_ar) = &csv_tokens();
127 $csv_regex = qr/$csv_regex/ if $csv_regex;
128 $csv_neg_regex = qr/$csv_neg_regex/ if $csv_neg_regex;
130 my %csv_uniq_lines = ();
132 if ($csv_start_line) {
133 die "[*] Cannot have start line > end line."
134 if $csv_start_line > $csv_end_line;
141 open MSGS, "< $nf_log_file" or die "[*] Could not open ",
149 if ($csv_start_line) {
150 next MSG unless $line_ctr >= $csv_start_line;
153 last MSG if $line_ctr == $csv_end_line;
155 next MSG unless $pkt_str =~ /IN.*OUT/;
158 my %pkt = %pkt_NF_init;
160 my $rv = &parse_NF_pkt_str(\%pkt, $pkt_str);
164 next MSG unless $pkt{'raw'} =~ m|$csv_regex|;
166 if ($csv_neg_regex) {
167 next MSG unless $pkt{'raw'} !~ m|$csv_neg_regex|;
169 $pkt{'log_prefix'} =~ s/\W//g;
170 $pkt{'log_prefix'} =~ s/\s//g;
171 my @matched_fields = ();
172 for (my $i=0; $i <= $#$tokens_ar; $i++) {
173 my $tok = $tokens_ar->[$i];
174 if ($match_criteria_ar) {
175 my $match_hr = $match_criteria_ar->[$i];
176 if (defined $match_hr->{'num'}) {
177 unless ($pkt{$tok} =~ m|^\d+$|
178 and $pkt{$tok} == $match_hr->{'num'}) {
181 } elsif (defined $match_hr->{'gt'}) {
182 unless ($pkt{$tok} =~ m|^\d+$|
183 and $pkt{$tok} > $match_hr->{'gt'}) {
186 } elsif (defined $match_hr->{'lt'}) {
187 unless ($pkt{$tok} =~ m|^\d+$|
188 and $pkt{$tok} < $match_hr->{'lt'}) {
191 } elsif (defined $match_hr->{'str'}) {
192 unless ($pkt{$tok} eq $match_hr->{'str'}) {
195 } elsif (defined $match_hr->{'re'}) {
196 unless ($pkt{$tok} =~ m|$match_hr->{'re'}m|) {
199 } elsif (defined $match_hr->{'net'}) {
200 if ($pkt{$tok} =~ m|$ip_re|) {
201 unless (ipv4_in_network($match_hr->{'net'}, $pkt{$tok})) {
207 } elsif (defined $match_hr->{'ip'}) {
208 unless ($pkt{$tok} eq $match_hr->{'ip'}) {
212 push @matched_fields, $pkt{$tok};
214 push @matched_fields, $pkt{$tok};
217 next MSG unless @matched_fields;
220 $str .= "$_, " for @matched_fields;
222 $str .= "$_ " for @matched_fields;
227 if ($csv_print_uniq) {
228 $csv_uniq_lines{$str} = '';
232 if ($csv_line_limit > 0) {
233 last if $ctr >= $csv_line_limit;
237 if ($csv_print_uniq) {
238 print "$_\n" for keys %csv_uniq_lines;
242 #============================= end main ===============================
247 my @match_criteria = ();
250 my @tok_tmp = split /\s+/, $csv_fields;
251 for my $tok_str (@tok_tmp) {
252 my $token = $tok_str;
254 if ($tok_str =~ m|(\w+):(\S+)|) {
258 $token = 'src' if $token eq 'SRC';
259 $token = 'dst' if $token eq 'DST';
260 $token = 'sp' if $token eq 'SPT';
261 $token = 'dp' if $token eq 'DPT';
262 $token = 'tos' if $token eq 'TOS';
263 $token = 'win' if $token eq 'WIN';
264 $token = 'itype' if $token eq 'TYPE';
265 $token = 'icode' if $token eq 'CODE';
266 $token = 'ttl' if $token eq 'TTL';
267 $token = 'ip_id' if $token eq 'ID';
268 $token = 'icmp_seq' if $token eq 'SEQ';
269 $token = 'proto' if $token eq 'PROTO';
270 $token = 'ip_len' if $token eq 'LEN';
271 $token = 'intf' if $token eq 'IN' or $token eq 'OUT';
272 unless (defined $pkt_NF_init{$token}) {
273 print "[*] $token is not a valid packet field; valid ",
275 for my $key (sort keys %pkt_NF_init) {
280 push @tokens, $token;
284 if ($search =~ m|^\d+$|) {
285 $search_hsh{'num'} = $search;
286 } elsif ($search =~ m|^>(\d+)$|) {
287 $search_hsh{'gt'} = $1;
288 die "[*] $token value must be >= 0"
290 } elsif ($search =~ m|^<(\d+)$|) {
291 $search_hsh{'lt'} = $1;
292 die "[*] $token value must be >= 0"
294 } elsif ($search =~ m|^/(.*?)/$|) {
295 $search_hsh{'re'} = qr|$1|;
296 } elsif ($search =~ m|^\"(.*?)\"$|) {
297 $search_hsh{'str'} = $1;
298 } elsif ($search =~ m|^$ip_re/$ip_re$|) {
299 $search_hsh{'net'} = $search;
300 } elsif ($search =~ m|^$ip_re/\d+$|) {
301 $search_hsh{'net'} = $search;
302 } elsif ($search =~ m|^$ip_re$|) {
303 $search_hsh{'ip'} = $search;
305 die "[*] Unrecognized value for $token";
307 push @match_criteria, \%search_hsh;
309 push @match_criteria, {};
327 return \@tokens, \@match_criteria;
330 sub parse_NF_pkt_str() {
331 my ($pkt_hr, $pkt_str) = @_;
333 print STDERR "\n", $pkt_str if $debug;
335 $pkt_hr->{'raw'} = $pkt_str;
337 if ($pkt_str =~ /.*kernel:\s+(.*?)\s*IN=/) {
338 $pkt_hr->{'log_prefix'} = $1;
341 ### get the in/out interface and iptables chain (the code below
342 ### allows the iptables log message to contain the PHYSDEV stuff):
343 ### Feb 25 12:13:27 bridge kernel: INBOUND TCP: IN=br0 PHYSIN=eth0 OUT=br0
344 ### PHYSOUT=eth1 SRC=63.147.183.21 DST=11.11.79.100 LEN=48 TOS=0x00
345 ### PREC=0x00 TTL=113 ID=19664 DF PROTO=TCP SPT=4918 DPT=135 WINDOW=64240
346 ### RES=0x00 SYN URGP=0
347 if ($pkt_str =~ /\sIN=(\S+).*\sOUT=\s/) {
348 $pkt_hr->{'intf'} = $1;
349 $pkt_hr->{'chain'} = 'INPUT';
350 } elsif ($pkt_str =~ /\sIN=(\S+).*\sOUT=\S/) {
351 $pkt_hr->{'intf'} = $1;
352 $pkt_hr->{'chain'} = 'FORWARD';
353 } elsif ($pkt_str =~ /\sIN=\s+.*\sOUT=(\S+)/) {
354 $pkt_hr->{'intf'} = $1;
355 $pkt_hr->{'chain'} = 'OUTPUT';
358 if ($pkt_str =~ /\sMAC=(\S+)/) {
360 if ($mac_str =~ /^((?:\w{2}\:){6})((?:\w{2}\:){6})/) {
361 $pkt_hr->{'dst_mac'} = $1;
362 $pkt_hr->{'src_mac'} = $2;
365 if ($pkt_hr->{'src_mac'}) {
366 $pkt_hr->{'src_mac'} =~ s/:$//;
367 print STDERR "[+] src mac addr: $pkt_hr->{'src_mac'}\n" if $debug;
369 if ($pkt_hr->{'dst_mac'}) {
370 $pkt_hr->{'dst_mac'} =~ s/:$//;
371 print STDERR "[+] dst mac addr: $pkt_hr->{'dst_mac'}\n" if $debug;
374 unless ($pkt_hr->{'intf'} and $pkt_hr->{'chain'}) {
375 print STDERR "[-] err packet: could not determine ",
376 "interface and chain.\n" if $debug;
380 ### get the syslog logging host for this packet
381 if ($pkt_str =~ /^\s*((?:\S+\s+){2}\S+)\s+(\S+)\s+kernel:/) {
382 $pkt_hr->{'timestamp'} = $1;
383 $pkt_hr->{'syslog_host'} = $2;
385 $pkt_hr->{'timestamp'} = localtime();
386 $pkt_hr->{'syslog_host'} = 'unknown';
389 ### try to extract a snort sid (generated by fwsnort) from
391 if ($pkt_str =~ /SID(\d+)/) {
392 $pkt_hr->{'fwsnort_sid'} = $1;
395 ### get IP options if --log-ip-options is used
396 ### (they appear before the PROTO= field).
397 if ($pkt_str =~ /OPT\s+\((\S+)\)\s+PROTO=/) {
398 $pkt_hr->{'ip_opts'} = $1;
401 ### May 18 22:21:26 orthanc kernel: DROP IN=eth2 OUT=
402 ### MAC=00:60:1d:23:d0:01:00:60:1d:23:d3:0e:08:00 SRC=192.168.20.25
403 ### DST=192.168.20.1 LEN=60 TOS=0x10 PREC=0x00 TTL=64 ID=47300 DF
404 ### PROTO=TCP SPT=34111 DPT=6345 WINDOW=5840 RES=0x00 SYN URGP=0
405 if ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+)\s+TOS=(\S+)
406 \s*.*\s+TTL=(\d+)\s+ID=(\d+)\s*.*\s+PROTO=TCP\s+
407 SPT=(\d+)\s+DPT=(\d+)\s.*\s*WINDOW=(\d+)\s+
410 ($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
411 $pkt_hr->{'tos'}, $pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'},
412 $pkt_hr->{'sp'}, $pkt_hr->{'dp'}, $pkt_hr->{'win'},
414 = ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10);
416 ### the reserve bits are not reported by ulogd, but normal
417 ### iptables syslog messages contain them.
418 $pkt_hr->{'flags'} =~ s/\s*RES=\S+\s*//;
420 $pkt_hr->{'proto'} = 'tcp';
423 $pkt_hr->{'flags'} = 'NULL' unless $pkt_hr->{'flags'};
425 unless ($pkt_hr->{'flags'} !~ /WIN/ &&
426 $pkt_hr->{'flags'} =~ /ACK/ ||
427 $pkt_hr->{'flags'} =~ /SYN/ ||
428 $pkt_hr->{'flags'} =~ /RST/ ||
429 $pkt_hr->{'flags'} =~ /URG/ ||
430 $pkt_hr->{'flags'} =~ /PSH/ ||
431 $pkt_hr->{'flags'} =~ /FIN/ ||
432 $pkt_hr->{'flags'} eq 'NULL') {
434 print STDERR "[-] err packet: bad tcp flags.\n" if $debug;
437 $pkt_hr->{'frag_bit'} = 1 if $pkt_str =~ /\sDF\s+PROTO/;
439 ### don't pickup IP options if --log-ip-options is used
440 ### (they appear before the PROTO= field).
441 if ($pkt_str =~ /URGP=\S+\s+OPT\s+\((\S+)\)/) {
442 $pkt_hr->{'tcp_opts'} = $1;
445 ### make sure we have a "reasonable" packet (note that nmap
446 ### can scan port 0 and iptables can report this fact)
447 unless ($pkt_hr->{'ip_len'} >= 0 and $pkt_hr->{'tos'}
448 and $pkt_hr->{'ttl'} >= 0 and $pkt_hr->{'ip_id'} >= 0
449 and $pkt_hr->{'proto'} and $pkt_hr->{'sp'} >= 0
450 and $pkt_hr->{'dp'} >= 0 and $pkt_hr->{'win'} >= 0
451 and $pkt_hr->{'flags'}) {
455 ### May 18 22:21:26 orthanc kernel: DROP IN=eth2 OUT=
456 ### MAC=00:60:1d:23:d0:01:00:60:1d:23:d3:0e:08:00
457 ### SRC=192.168.20.25 DST=192.168.20.1 LEN=28 TOS=0x00 PREC=0x00
458 ### TTL=40 ID=47523 PROTO=UDP SPT=57339 DPT=305 LEN=8
460 } elsif ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+)\s+TOS=(\S+)
461 \s.*TTL=(\d+)\s+ID=(\d+)\s*.*\s+PROTO=UDP\s+
462 SPT=(\d+)\s+DPT=(\d+)\s+LEN=(\d+)/x) {
464 ($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
465 $pkt_hr->{'tos'}, $pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'},
466 $pkt_hr->{'sp'}, $pkt_hr->{'dp'}, $pkt_hr->{'udp_len'})
467 = ($1,$2,$3,$4,$5,$6,$7,$8,$9);
469 $pkt_hr->{'proto'} = 'udp';
471 ### make sure we have a "reasonable" packet (note that nmap
472 ### can scan port 0 and iptables can report this fact)
473 unless ($pkt_hr->{'ip_len'} >= 0
474 and $pkt_hr->{'tos'} and $pkt_hr->{'ttl'} >= 0
475 and $pkt_hr->{'ip_id'} >= 0 and $pkt_hr->{'proto'}
476 and $pkt_hr->{'sp'} >= 0 and $pkt_hr->{'dp'} >= 0
477 and $pkt_hr->{'udp_len'} >= 0) {
482 ### Nov 27 15:45:51 orthanc kernel: DROP IN=eth1 OUT= MAC=00:a0:cc:e2:1f:f2:00:
483 ### 20:78:10:70:e7:08:00 SRC=192.168.10.20 DST=192.168.10.1 LEN=84 TOS=0x00
484 ### PREC=0x00 TTL=64 ID=0 DF PROTO=ICMP TYPE=8 CODE=0 ID=61055 SEQ=256
486 } elsif ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+).*
487 TTL=(\d+)\s+ID=(\d+).*PROTO=ICMP\s+TYPE=(\d+)\s+
488 CODE=(\d+)\s+ID=(\d+)\s+SEQ=(\d+)/x) {
490 ($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
491 $pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'}, $pkt_hr->{'itype'},
492 $pkt_hr->{'icode'}, $pkt_hr->{'icmp_id'}, $pkt_hr->{'icmp_seq'})
493 = ($1,$2,$3,$4,$5,$6,$7,$8,$9);
495 $pkt_hr->{'proto'} = 'icmp';
496 $pkt_hr->{'sp'} = $pkt_hr->{'dp'} = 0;
498 unless ($pkt_hr->{'ip_len'} >= 0 and $pkt_hr->{'ttl'} >= 0
499 and $pkt_hr->{'proto'} and $pkt_hr->{'itype'} >= 0
500 and $pkt_hr->{'icode'} >= 0 and $pkt_hr->{'ip_id'} >= 0
501 and $pkt_hr->{'icmp_seq'} >= 0) {
507 ### Sometimes the iptables log entry gets messed up due to
508 ### buffering issues so we write it to the error log.
509 print STDERR "[-] err packet: no regex match.\n" if $debug;
516 my $exitcode = shift;
520 [+] Version: $version
521 [+] By Michael Rash (mbr\@cipherdyne.org, http://www.cipherdyne.org)
523 Usage: nf2csv [options]
526 -f, --fields <fields> - Restrict output to a list of
528 -u, --uniq-lines - Only print unique lines
529 -m, --max-lines <num> - Specify the maximum number of
530 output lines to print.
531 -M, --messages-file <file> - Specify the path to the iptables
532 logfile (use in conjunction with
534 -s, --start-line <line> - Starting line within iptables log
536 -e, --end-line <line> - Ending line within iptables log
538 -r, --regex <regex> - Require iptables log messages to
539 match an additional regex.
540 -n, --neg-regex <regex> - Require iptables log messages to
541 not match an additional regex.
542 -d, --debug - Run in debugging mode.
543 -V, --Version - Print the version and exit.
544 -h --help - Display usage on STDOUT and exit.