#!/usr/bin/perl -w # # This file is part of the exilog suite. # # http://duncanthrax.net/exilog/ # # (c) Tom Kistner 2004 # # See LICENSE for licensing information. # package exilog_cgi_messages; use strict; use exilog_config; use exilog_cgi_html; use exilog_cgi_param; use exilog_sql; use exilog_util; use Net::Netmask; use Time::Local; use Data::Dumper; BEGIN { use Exporter; use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); # set the version for version checking $VERSION = 0.1; @ISA = qw(Exporter); @EXPORT = qw( &messages ); %EXPORT_TAGS = (); # your exported package globals go here, # as well as any optionally exported functions @EXPORT_OK = qw(); } sub _select_all { # All tables my @tables = ( 'deliveries','errors','unknown','deferrals','messages','rejects','queue' ); # Only server and timestamp as criteria. # Since these are present on every table # the queries are all the same ... my $criteria = { 'timestamp' => (edt($param,'tr') ? _make_tr() : undef), 'server' => (edt($param,'sr') ? $param->{'sr'} : undef ) }; my @results = (); foreach my $table (@tables) { next unless (ina($param->{'qw'},$table)); push @results, @{ sql_select( $table, [ 'server','message_id','timestamp' ], $criteria ) }; }; return \@results; }; sub _select_ident { if (!edt($param,'qs')) { return []; } my $criteria = { 'timestamp' => (edt($param,'tr') ? _make_tr() : undef), 'server' => (edt($param,'sr') ? $param->{'sr'} : undef ), 'host_ident' => dos2sql($param->{'qs'}) }; # Only messages table return sql_select( 'messages', [ 'server','message_id','timestamp' ], $criteria ); }; sub _select_msgid { if (!edt($param,'qs')) { return []; } # Only messages table return sql_select( 'messages', [ 'server','message_id','timestamp' ], { 'msgid' => dos2sql($param->{'qs'}) } ); }; sub _select_message_id { if (!edt($param,'qs')) { return []; } my @results = (); my @tables = ( 'deliveries','errors','unknown','deferrals','messages','rejects','queue' ); my $criteria = { 'message_id' => dos2sql($param->{'qs'}) }; foreach my $table (@tables) { push @results, @{ sql_select( $table, [ 'server','message_id','timestamp' ], $criteria ) }; }; # check bounce parent field too push @results, @{ sql_select( 'messages', [ 'server','message_id','timestamp' ], { 'bounce_parent' => dos2sql($param->{'qs'}) } ) }; return \@results; }; sub _select_addr { my $p = shift || 'all'; if (!edt($param,'qs')) { return []; } my @queries; push @queries, { 'table' => 'messages', 'criteria' => { 'mailfrom' => dos2sql($param->{'qs'}) } }, { 'table' => 'rejects', 'criteria' => { 'mailfrom' => dos2sql($param->{'qs'}) } } if (($p eq 'sender') || ($p eq 'all')); push @queries, { 'table' => 'rejects', 'criteria' => { 'rcpt' => dos2sql($param->{'qs'}) } }, { 'table' => 'deliveries', 'criteria' => { 'rcpt' => dos2sql($param->{'qs'}) } }, { 'table' => 'deliveries', 'criteria' => { 'rcpt_final' => dos2sql($param->{'qs'}) } }, { 'table' => 'deferrals', 'criteria' => { 'rcpt' => dos2sql($param->{'qs'}) } }, { 'table' => 'deferrals', 'criteria' => { 'rcpt_final' => dos2sql($param->{'qs'}) } }, { 'table' => 'errors', 'criteria' => { 'rcpt' => dos2sql($param->{'qs'}) } }, { 'table' => 'errors', 'criteria' => { 'rcpt_final' => dos2sql($param->{'qs'}) } } if (($p eq 'rcpt') || ($p eq 'all')); my @results = (); foreach my $query (@queries) { next unless (ina($param->{'qw'},$query->{table})); # add standard criteria $query->{criteria}->{'timestamp'} = (edt($param,'tr') ? _make_tr() : undef); $query->{criteria}->{'server'} = (edt($param,'sr') ? $param->{'sr'} : undef ); push @results, @{ sql_select( $query->{table}, [ 'server','message_id','timestamp' ], $query->{criteria} ) }; }; return \@results; }; sub _select_host { my $p = shift || 'all'; if (!edt($param,'qs')) { return []; } my @queries; if ($param->{'qs'} =~ /^[0-9A-Fa-f.:]+$/) { # IPv4 or IPv6 address push @queries, { 'table' => 'messages', 'criteria' => { 'host_addr' => dos2sql($param->{'qs'}) } }, { 'table' => 'rejects', 'criteria' => { 'host_addr' => dos2sql($param->{'qs'}) } } if (($p eq 'incoming') || ($p eq 'all')); push @queries, { 'table' => 'deliveries', 'criteria' => { 'host_addr' => dos2sql($param->{'qs'}) } }, { 'table' => 'deferrals', 'criteria' => { 'host_addr' => dos2sql($param->{'qs'}) } }, { 'table' => 'errors', 'criteria' => { 'host_addr' => dos2sql($param->{'qs'}) } }, { 'table' => 'unknown', 'criteria' => { 'line' => '%'.dos2sql($param->{'qs'}).'%' } } if (($p eq 'outgoing') || ($p eq 'all')); } elsif ($param->{'qs'} =~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/[0-9]{1,2}$/) { # check if we can make a valid Net::Netmask object out of this my $block = new2 Net::Netmask($param->{'qs'}); if (!defined($block)) { return "Invalid CIDR specification"; }; # Network specification push @queries, { 'table' => 'messages' }, { 'table' => 'rejects' } if (($p eq 'incoming') || ($p eq 'all')); push @queries, { 'table' => 'deliveries' }, { 'table' => 'deferrals' }, { 'table' => 'errors' } if (($p eq 'outgoing') || ($p eq 'all')); my @results = (); foreach my $query (@queries) { next unless (ina($param->{'qw'},$query->{table})); # add standard criteria $query->{criteria}->{'timestamp'} = (edt($param,'tr') ? _make_tr() : undef); $query->{criteria}->{'server'} = (edt($param,'sr') ? $param->{'sr'} : undef ); push @results, @{ sql_select( $query->{table}, [ 'server','message_id','timestamp','host_addr' ], $query->{criteria} ) }; }; # now weed out those that don't match the CIDR specification my @valid = (); foreach my $result (@results) { if ($block->match($result->{host_addr})) { delete $result->{host_addr}; push @valid, $result; }; }; return \@valid; } else { # assume hostname my $prefix_wc = ""; my $suffix_wc = ""; $prefix_wc = '%' if ($param->{'qs'} !~ /^\%/); $suffix_wc = '%' if ($param->{'qs'} !~ /\%$/); push @queries, { 'table' => 'messages', 'criteria' => { 'host_helo' => dos2sql($param->{'qs'}) } }, { 'table' => 'messages', 'criteria' => { 'host_rdns' => dos2sql($param->{'qs'}) } }, { 'table' => 'rejects', 'criteria' => { 'host_helo' => dos2sql($param->{'qs'}) } }, { 'table' => 'rejects', 'criteria' => { 'host_rdns' => dos2sql($param->{'qs'}) } } if (($p eq 'incoming') || ($p eq 'all')); push @queries, { 'table' => 'deliveries', 'criteria' => { 'host_dns' => dos2sql($param->{'qs'}) } }, { 'table' => 'deferrals', 'criteria' => { 'host_dns' => dos2sql($param->{'qs'}) } }, { 'table' => 'errors', 'criteria' => { 'host_dns' => dos2sql($param->{'qs'}) } }, { 'table' => 'unknown', # the blank makes sure that we do not match domains in addresses 'criteria' => { 'line' => $prefix_wc.' '.dos2sql($param->{'qs'}).$suffix_wc } } if (($p eq 'outgoing') || ($p eq 'all')); }; my @results = (); foreach my $query (@queries) { next unless (ina($param->{'qw'},$query->{table})); # add standard criteria $query->{criteria}->{'timestamp'} = (edt($param,'tr') ? _make_tr() : undef); $query->{criteria}->{'server'} = (edt($param,'sr') ? $param->{'sr'} : undef ); push @results, @{ sql_select( $query->{table}, [ 'server','message_id','timestamp' ], $query->{criteria} ) }; }; return \@results; }; sub messages { _print_Messages_selector(); # Check CGI input for event selection. # We need at least a query type ('qt'), # otherwise we only display the selector. my $selected = []; if (edt($param,'qt')) { # Call event selection function for this query type. _print_progress_bar("Collecting message IDs ..."); # cut off parameter part (separated with dash) my ($function,$parameter) = split /\-/, $param->{'qt'}; no strict "refs"; $selected = &{ "_select_".$function }($parameter); if (ref($selected) ne 'ARRAY') { # error _update_progress_bar($selected); return; }; } else { # no query type ('qt'), just return return; }; # Now we have a set of selected messages in an array: # # [0]-->{server} # |->{timestamp} # \->{message_id} # [1]-->{server} # |->{timestamp} # \->{message_id} # ... # Perform dupe check. We may have a lot of duplicate IDs # in the list. It is faster to weed them out this way ... _update_progress_bar("Performing dupe check ..."); my $dupe = {}; my @duped = (); foreach my $message (@{ $selected }) { if (exists($dupe->{$message->{server}}->{$message->{message_id}})) { # Make sure we use the largest timestamp we can find if ($dupe->{$message->{server}}->{$message->{message_id}}->{timestamp} < $message->{timestamp}) { $dupe->{$message->{server}}->{$message->{message_id}}->{timestamp} = $message->{timestamp}; }; next; }; $dupe->{$message->{server}}->{$message->{message_id}} = $message; push @duped, $message; }; undef $dupe; undef $selected; if ((scalar @duped) == 0) { _update_progress_bar("No matching events found."); return; }; if (((scalar @duped) > 500) && ($param->{'sm'} !~ /^Confirm/)) { _update_progress_bar("Warning: ".(scalar @duped)." messages/events found. Narrow down your selection or submit the query again."); print ' '; return; }; # Initialize stats counters my $stats = { 'num_messages' => { 'desc' => "Messages", 'order' => 1, 'num' => 0 }, 'num_rejects' => { 'desc' => "Rejects", 'order' => 5, 'num' => 0 }, 'num_deliveries' => { 'desc' => "Deliveries", 'order' => 2, 'num' => 0 }, 'num_errors' => { 'desc' => "Errors", 'order' => 3, 'num' => 0 }, 'total_turnover' => { 'desc' => "Total Turnover", 'order' => 4, 'size' => 0 } }; # Now we need to build the complete message set. # This requires a large number of SELECTs. _update_progress_bar("Sorting ..."); my $c = 0; foreach my $message (sort { $b->{timestamp} <=> $a->{timestamp} } @duped) { # Update the progress bar every 50 entries if (($c % 50) == 0) { _update_progress_bar("Grabbing event data (".$c." of ".scalar @duped." events done) ..."); }; $c++; # Remove timestamp, we'll re-add it later for marking # the "sort" timestamp. my $sort_timestamp = $message->{timestamp}; delete $message->{timestamp}; # Check the message ID. if ($message->{message_id} !~ /^.{6}\-.{6}\-.{2}$/) { # This is a pre-DATA reject/warning. # Render it as a reject. my $complete = @{ sql_select( 'rejects', ['*'], $message ) }[0]; $complete->{sort_timestamp} = $sort_timestamp; print render_reject($complete); $stats->{num_rejects}->{num}++; } else { # Try to grab complete arrival ('messages' table) my $complete = @{ sql_select( 'messages', ['*'], $message ) }[0]; # If there is an arrival, this set has a "real" # message ID. Scan other tables for events. if (defined($complete)) { $complete->{rejects} = sql_select( 'rejects', ['*'], $message ); $complete->{deliveries} = sql_select( 'deliveries', ['*'], $message ); $complete->{errors} = sql_select( 'errors', ['*'], $message ); $complete->{deferrals} = sql_select( 'deferrals', ['*'], $message ); $complete->{unknown} = sql_select( 'unknown', ['*'], $message ); $complete->{queue} = sql_select( 'queue', ['*'], $message ); $complete->{sort_timestamp} = $sort_timestamp; print render_message($complete); $stats->{num_messages}->{num}++; $stats->{total_turnover}->{size} += $complete->{size}; $stats->{num_rejects}->{num} += (scalar @{ $complete->{rejects} }); $stats->{num_deliveries}->{num} += (scalar @{ $complete->{deliveries} }); $stats->{total_turnover}->{size} += ($complete->{size} * (scalar @{ $complete->{deliveries} })); $stats->{num_errors}->{num} += (scalar @{ $complete->{errors} }); } # If there is no associated arrival, this is either # a POST-DATA reject (in rejects table) or another # post-DATA warning (in unknown table). Since both # can occur, we render this as a message. else { $complete->{server} = $message->{server}; $complete->{message_id} = $message->{message_id}; $complete->{rejects} = sql_select( 'rejects', ['*'], $message ); $complete->{unknown} = sql_select( 'unknown', ['*'], $message ); $complete->{sort_timestamp} = $sort_timestamp; print render_message($complete); $stats->{num_rejects}->{num}++; }; }; }; _update_progress_bar(_render_stats($stats)); }; sub _make_tr { my $str = $q->param('tr') || 0; unless ($str eq 'custom') { my $unit = chop $str; my $now = time(); my $units = { '0' => 0, 'm' => 60, 'h' => 3600, 'd' => 86400 }; my $then = $now + $units->{$unit}*$str; unless ($now == $then) { # The "unlimited" case $param->{'tds'} = stamp_to_date($then,1); $param->{'tde'} = stamp_to_date($now,1); } return $then; } else { $param->{'tds'} =~ s/ +$//; $param->{'tds'} =~ s/^ +//; $param->{'tde'} =~ s/ +$//; $param->{'tde'} =~ s/^ +//; my ($sd,$st) = split / +/, $param->{'tds'}; my ($ed,$et) = split / +/, $param->{'tde'}; if (!$st && $sd =~ /\:/) { $st = $sd; $sd = ''; } if (!$et && $ed =~ /\:/) { $et = $ed; $ed = ''; } $ed = $sd unless($ed); my $fsd = _parse_date($sd, $st || '00:00:00'); my $fed = _parse_date($ed, $et || '23:59:59'); $param->{'tds'} = stamp_to_date($fsd); $param->{'tde'} = stamp_to_date($fed); return $fsd." ".$fed; } } sub _parse_date { my $d = shift; my $t = shift; my ($dn,$tn) = split / /, stamp_to_date(time,1); my ($year,$month,$day) = split /\-/, $dn; my ($hour,$minute,$second) = split /\:/, $tn; if ($d =~ /^([0-9]{4})\-([0-9]{2})\-([0-9]{2})$/) { $year = $1; $month = $2; $day = $3; } elsif ($d =~ /^([0-9]{2})\-([0-9]{2})$/) { $month = $1; $day = $2; } if ($t =~ /^([0-9]{2})\:([0-9]{2})\:([0-9]{2})$/) { $hour = $1; $minute = $2; $second = $3; } elsif ($t =~ /^([0-9]{2})\:([0-9]{2})$/) { $hour = $1; $minute = $2; } return date_to_stamp($year.'-'.$month.'-'.$day, $hour.':'.$minute.':'.$second); } sub _render_stats { my $stats = shift || {}; my @items = (); foreach (sort {$stats->{$a}->{order} <=> $stats->{$b}->{order}} keys %{ $stats }) { if (exists($stats->{$_}->{num}) && $stats->{$_}->{num}) { push @items, $stats->{$_}->{desc}.": ".$stats->{$_}->{num}; } elsif (exists($stats->{$_}->{size}) && $stats->{$_}->{size}) { push @items, $stats->{$_}->{desc}.": ".human_size($stats->{$_}->{size}); }; }; return join(" | ",@items); }; sub _print_progress_bar { my $str = shift || ""; print render_header( $q->div({-name=>"progress",-id=>"progress", -align=>"center"}, $str ) ); print "\n\n"; }; sub _update_progress_bar { my $str = shift || ""; print ' '; print "\n\n"; }; sub _print_Messages_selector { _make_tr(); # Calendar popup DIVs print "\n". ' ' ."\n"; print $q->div({-class=>"top_spacer"}, $q->div({-align=>"left",-style=>"padding: 10px; border: 1px solid black; background: #eeeeee;"}, $q->table({-cellspacing=>0,-cellpadding=>4,-border=>0}, $q->Tr( $q->td({-align=>"left",-style=>"width: 16px;"}, $q->img({-src=>$config->{web}->{webroot}."icons/event_type.png"}) ), $q->td({-align=>"left",-style=>"width: 100px;"}, "Search Type" ), $q->td({-align=>"left"}, $q->popup_menu({ -name=>"qt", -id=>"qt", -style=>"width: 400px;", -values=>[ 'all', 'addr-all', 'addr-sender', 'addr-rcpt', 'host-all', 'host-incoming', 'host-outgoing', 'msgid', 'ident', 'message_id' ], -labels=>{ 'all' => "Show everything", 'addr-all' => "Address (All)", 'addr-sender' => "Address (Sender)", 'addr-rcpt' => "Address (Recipient)", 'host-all' => "Host (all)", 'host-incoming' => "Host (incoming)", 'host-outgoing' => "Host (outgoing)", 'msgid' => "Message-ID (Header)", 'ident' => "Ident String (incoming messages)", 'message_id' => "Message-ID (Exim)" }, -default=>(exists($param->{'qt'}) ? ($param->{'qt'} || 'all') : 'all'), -onChange=>"javascript:switch_controls(document.getElementById('qt').options[document.getElementById('qt').selectedIndex].value);", -override=>1}) ) ) ) . $q->span({-id=>"term"},''). $q->div({-id=>"term_hidden",-style=>"visibility: hidden; position: absolute;"}, $q->table({-cellspacing=>0,-cellpadding=>4,-border=>0}, $q->Tr( $q->td({-align=>"left",-style=>"width: 16px;"}, $q->img({-src=>$config->{web}->{webroot}."icons/find.png"}) ), $q->td({-align=>"left",-style=>"width: 100px;"}, "Search Term" ), $q->td({-align=>"left"}, $q->textfield( { -name=>"qs", -style=>"width: 400px;", -value=>(exists($param->{'qs'}) ? ($param->{'qs'} || '') : ''), -override=>1 } ) ) ) ) ) . $q->span({-id=>"events"},''). $q->div({-id=>"events_hidden",-style=>"visibility: hidden; position: absolute;"}, $q->table({-cellspacing=>0,-cellpadding=>4,-border=>0}, $q->Tr( $q->td({-align=>"left",-valign=>"top",-style=>"width: 16px;"}, $q->img({-src=>$config->{web}->{webroot}."icons/address.png"}) ), $q->td({-align=>"left",-valign=>"top",-style=>"width: 100px;"}, "Event types" ), $q->td({-align=>"left",-style=>"padding:2px 4px 4px 4px;"}, eval { my @where = ( 'messages', 'errors', 'deliveries', 'deferrals', 'rejects', 'queue' ); my $labels = { 'messages' => 'Arrivals', 'errors' => 'Errors', 'deliveries' => 'Deliveries', 'deferrals' => 'Deferrals', 'rejects' => 'Rejects', 'queue' => 'Queued' }; my $html = ""; my $num = 0; foreach my $w (@where) { if (($num % 3) == 0) { $html .= '