added WAN statistics in collector

scripts/eye-statd.pl

@@ -17,6 +17,7 @@ use eyelib::config;
 use eyelib::main;
 use eyelib::net_utils;
 use eyelib::database;
+use eyelib::snmp;
 use Socket qw(AF_INET6 inet_ntop);
 use IO::Socket;
 use Data::Dumper;
@@ -25,8 +26,10 @@ use threads;
 my @router_ref = ();
 my @interfaces = ();
 
-my %router_svi;
+my %routers_svi;
+my %routers_by_ip;
 my %routers;
+
 my %wan_dev;
 my %lan_dev;
 
@@ -35,6 +38,7 @@ my $saving = 0;
 
 #user statistics for cached data
 my %user_stats;
+my %wan_stats;
 
 my $MAXREAD = 9216;
 
@@ -91,11 +95,16 @@ $timeshift = get_option($hdb,55)*60;
 #router device_id by known device ip
 foreach my $row (@router_ref) {
     $routers{$row->{id}}=$row;
-    my @auth_list = get_records_sql($hdb,"SELECT ip FROM User_auth WHERE deleted=0 AND user_id=".$row->{user_id});
-    foreach my $auth (@auth_list) {
-	$router_svi{$auth->{ip}}{id}=$row->{id};
-	$router_svi{$auth->{ip}}{save}=$row->{netflow_save};
-	}
+    my $l3_list = getIpAdEntIfIndex($row->{ip},$row->{community},$row->{snmp_version});
+
+    #create hash for interface snmp index => ip-address at interface =1;
+    foreach my $router_ip (keys %$l3_list) { $routers_svi{$row->{id}}{$l3_list->{$router_ip}}{$router_ip}=1; }
+
+    #create hash by all ip-addresses for router
+    foreach my $router_ip (keys %$l3_list) {
+        $routers_by_ip{$router_ip}->{id}=$row->{id};
+        $routers_by_ip{$router_ip}->{save}=$row->{netflow_save};
+        }
     }
 
 #snmp index for WAN/LAN interface by device id
@@ -254,7 +263,7 @@ sub parse_netflow_v9_template_flowset {
 		my $template = [@template_ints[($i+2) .. ($i+2+$fldcount*2-1)]];
 		$netflow9_templates->{$ipaddr}->{$source_id}->{$template_id}->{'template'} = $template;
 		
-		# Calculate total length of template data
+		# total length of template data
 		my $totallen = 0;
 		for (my $j = 1; $j < scalar @$template; $j += 2) {
 			$totallen += $template->[$j];
@@ -382,20 +391,19 @@ sub save_flow {
 	$flow->{direction} = '0';
 	my $router_id;
 	#skip unknown router
-	if (exists $router_svi{$router_ip}) { 
-		$router_id = $router_svi{$router_ip}{id};
+	if (exists $routers_by_ip{$router_ip}) { 
+		$router_id = $routers_by_ip{$router_ip}{id};
 		$flow->{router_ip} = $router_ip;
 		$flow->{device_id} = $router_id;
-		$flow->{save} = $router_svi{$router_ip}{save};
+		$flow->{save} = $routers_by_ip{$router_ip}{save};
 		} else { return; }
-	#skip input traffic for router
-	if (exists $wan_dev{$router_id}->{$flow->{snmp_out}} and exists $wan_dev{$router_id}->{$flow->{snmp_in}}) { return; }
+
 	#skip local traffic for router
 	if (!exists $wan_dev{$router_id}->{$flow->{snmp_out}} and ! exists $wan_dev{$router_id}->{$flow->{snmp_in}}) { return; }
 
+        #detect traffic direction
 	if (exists $wan_dev{$router_id}->{$flow->{snmp_out}}) { $flow->{direction} = 1; }
 
-#	print Dumper($flow) if ($debug);
 	push(@traffic,$flow);
 	flush_traffic(0);
 }
@@ -410,6 +418,8 @@ $saving++;
 
 my $pid = fork();
 
+INIT();
+
 if (!defined $pid) {
     $saving = 0;
     print "cannot fork! Save traffic and exit...\n";
@@ -421,14 +431,12 @@ if (!defined $pid) {
         return;
     }
 
+
 #create oper-cache
 my @flush_table = ();
 
 push(@flush_table,@traffic);
 
-#clean main cache
-INIT();
-
 my $hdb=init_db();
 
 #saved packet by users
@@ -439,6 +447,7 @@ my %routers_found;
 
 #last packet timestamp
 my $last_time = time();
+my $start_time;
 
 foreach my $traf_record (@flush_table) {
 
@@ -452,6 +461,54 @@ if ($traf_record->{save}) {
 
 $routers_found{$router_id} = 1;
 
+#save start netflow time
+if (!$start_time) { $start_time = $traf_record->{starttime}; }
+
+#--- router statistics
+
+if (exists $wan_dev{$router_id}->{$traf_record->{snmp_out}} and exists $wan_dev{$router_id}->{$traf_record->{snmp_in}}) {
+    #input or output
+    if (exists $routers_svi{$router_id}{$traf_record->{snmp_out}}{$traf_record->{src_ip}}) {
+        #output
+        if (exists $wan_stats{$router_id}{$traf_record->{snmp_out}}{out}) {
+            $wan_stats{$router_id}{$traf_record->{snmp_out}}{out}+=$traf_record->{octets};
+            } else {
+            $wan_stats{$router_id}{$traf_record->{snmp_out}}{out}=$traf_record->{octets};
+            }
+        next;
+        }
+    if (exists $routers_svi{$router_id}{$traf_record->{snmp_in}}{$traf_record->{dst_ip}}) {
+        #input
+        if (exists $wan_stats{$router_id}{$traf_record->{snmp_in}}{in}) {
+            $wan_stats{$router_id}{$traf_record->{snmp_in}}{in}+=$traf_record->{octets};
+            } else {
+            $wan_stats{$router_id}{$traf_record->{snmp_in}}{in}=$traf_record->{octets};
+            }
+        next;
+        }
+    #unknown packet
+    next;
+    } else {
+    #forward
+    if ($traf_record->{direction}) {
+        #out
+        if (exists $wan_stats{$router_id}{$traf_record->{snmp_out}}{forward_out}) {
+            $wan_stats{$router_id}{$traf_record->{snmp_out}}{forward_out}+=$traf_record->{octets};
+            } else {
+            $wan_stats{$router_id}{$traf_record->{snmp_out}}{forward_out}+=$traf_record->{octets};
+            }
+        } else {
+        #in
+        if (exists $wan_stats{$router_id}{$traf_record->{snmp_in}}{forward_in}) {
+            $wan_stats{$router_id}{$traf_record->{snmp_in}}{forward_in}+=$traf_record->{octets};
+            } else {
+            $wan_stats{$router_id}{$traf_record->{snmp_in}}{forward_in}+=$traf_record->{octets};
+            }
+        }
+    }
+
+#--- user statistics
+
 #outbound traffic
 if ($traf_record->{direction}) {
     if (exists $user_stats{$traf_record->{src_ip}}) {
@@ -531,14 +588,14 @@ my $nmin = int($min/10)*10;
 my $netflow_file_name = $netflow_file_path.sprintf "%04d%02d%02d-%02d%02d.csv",$year+1900,$month+1,$day,$hour,$nmin;
 if ($saved_netflow{$dev_id} and scalar @{$saved_netflow{$dev_id}}) {
     use File::Path;
-    File::Path::make_path($netflow_file_path);
+    make_path($netflow_file_path);
     if ( -e $netflow_file_name) {
         open (ND,">>$netflow_file_name") || die("Error open file $netflow_file_name!!! die...");
         binmode(ND,':utf8');
         } else {
         open (ND,">$netflow_file_name") || die("Error open file $netflow_file_name!!! die...");
         binmode(ND,':utf8');
-        print ND join(';',"time","proto","snmp_in","snmp_out","src_ip","dst_ip","xsrc_ip","xdst_ip","src_port","dst_port","octets","pkts")."\n";
+        print ND join(';',"time","device_id","proto","snmp_in","snmp_out","src_ip","dst_ip","xsrc_ip","xdst_ip","src_port","dst_port","octets","pkts")."\n";
         }
     foreach my $row (@{$saved_netflow{$dev_id}}) {
         next if (!$row);
@@ -608,6 +665,28 @@ foreach my $user_ip (keys %user_stats) {
 	}
     }
 
+#print Dumper(\%wan_stats) if ($debug);
+
+# update database
+foreach my $router_id (keys %wan_stats) {
+    #last flow for user
+    my ($sec,$min,$hour,$day,$month,$year) = (localtime($start_time))[0,1,2,3,4,5];
+    #flow time string
+    my $flow_date = $hdb->quote(sprintf "%04d-%02d-%02d %02d:%02d:%02d",$year+1900,$month+1,$day,$hour,$min,$sec);
+    #per interface stats
+    foreach my $int_id (keys %{$wan_stats{$router_id}}) {
+	if (!$wan_stats{$router_id}{$int_id}{in})  { $wan_stats{$router_id}{$int_id}{in} = 0; }
+	if (!$wan_stats{$router_id}{$int_id}{out})  { $wan_stats{$router_id}{$int_id}{out} = 0; }
+	if (!$wan_stats{$router_id}{$int_id}{forward_in})  { $wan_stats{$router_id}{$int_id}{forward_in} = 0; }
+	if (!$wan_stats{$router_id}{$int_id}{forward_out})  { $wan_stats{$router_id}{$int_id}{forward_out} = 0; }
+	#skip empty stats
+        if ($wan_stats{$router_id}{$int_id}{in} + $wan_stats{$router_id}{$int_id}{out} + $wan_stats{$router_id}{$int_id}{forward_in} + $wan_stats{$router_id}{$int_id}{forward_out} ==0) { next; }
+	#current stats
+	my $tSQL="INSERT INTO Wan_stats (time,router_id,interface_id,in,out,forward_in,forward_out) VALUES($flow_date,'$router_id',$wan_stats{$router_id}{$int_id}{in},$wan_stats{$router_id}{$int_id}{out},$wan_stats{$router_id}{$int_id}{forward_in},$wan_stats{$router_id}{$int_id}{forward_out})";
+	push (@batch_sql_traf,$tSQL);
+	}
+    }
+
 #update statistics in DB
 batch_db_sql($hdb,\@batch_sql_traf);
 

scripts/eyelib/snmp.pm

@@ -28,6 +28,7 @@ get_vlan_at_port
 get_switch_vlans
 get_snmp_ifindex
 get_ifmib_index_table
+getIpAdEntIfIndex
 get_interfaces
 get_router_state
 snmp_get_req
@@ -41,6 +42,7 @@ $ifIndex
 $ifIndex_map
 $arp_oid
 $ipNetToMediaPhysAddress
+$ipAdEntIfIndex
 $fdb_table_oid
 $fdb_table_oid2
 $cisco_vlan_oid
@@ -53,11 +55,12 @@ $snmp_timeout
 BEGIN
 {
 
-our $ifAlias      ='.1.3.6.1.2.1.31.1.1.1.18';
-our $ifName       ='.1.3.6.1.2.1.31.1.1.1.1';
-our $ifDescr      ='.1.3.6.1.2.1.2.2.1.2';
-our $ifIndex      ='.1.3.6.1.2.1.2.2.1.1';
-our $ifIndex_map  ='.1.3.6.1.2.1.17.1.4.1.2';
+our $ifAlias        ='.1.3.6.1.2.1.31.1.1.1.18';
+our $ifName         ='.1.3.6.1.2.1.31.1.1.1.1';
+our $ifDescr        ='.1.3.6.1.2.1.2.2.1.2';
+our $ifIndex        ='.1.3.6.1.2.1.2.2.1.1';
+our $ifIndex_map    ='.1.3.6.1.2.1.17.1.4.1.2';
+our $ipAdEntIfIndex ='.1.3.6.1.2.1.4.20.1.2';
 
 #RFC1213::atPhysAddress
 our $arp_oid      ='.1.3.6.1.2.1.3.1.1.2';
@@ -361,6 +364,26 @@ sub get_snmp_ifindex {
 
 #-------------------------------------------------------------------------------------
 
+#get ip interfaces
+sub getIpAdEntIfIndex {
+    my ($host,$community,$snmp) = @_;
+    my $port = 161;
+    ### open SNMP session
+    my ($snmp_session, $error) = Net::SNMP->session( -hostname  => $host, -community => $community, -version => $snmp, -timeout => $snmp_timeout );
+    return if (!defined($snmp_session));
+    $snmp_session->translate([-timeticks]);
+    my $if_ipaddr = $snmp_session->get_table($ipAdEntIfIndex);
+    my $l3_list;
+    foreach my $row (keys(%$if_ipaddr)) {
+        my $ipaddr = $row;
+        $ipaddr=~s/$ipAdEntIfIndex\.//;
+        $l3_list->{$ipaddr}=$if_ipaddr->{$row};
+    }
+    return $l3_list;
+}
+
+#-------------------------------------------------------------------------------------
+
 sub get_interfaces {
     my ($host,$community,$snmp,$skip_empty) = @_;
 #    return if (!HostIsLive($host));

scripts/updates/2-7-0/device_l3_interfaces.sql

@@ -1 +1,2 @@
 ALTER TABLE `device_l3_interfaces` ADD `snmpin` INT NULL DEFAULT NULL AFTER `device_id`;
+ALTER TABLE `device_l3_interfaces` ADD `deleted` BOOLEAN NOT NULL DEFAULT FALSE AFTER `name`;

scripts/updates/2-7-0/wan_stats.sql

@@ -0,0 +1,2 @@
+CREATE TABLE `Wan_stats` ( `id` INT NOT NULL AUTO_INCREMENT , `time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , `router_id` INT NULL DEFAULT NULL , `interface_id` INT NULL DEFAULT NULL , `in` BIGINT UNSIGNED NOT NULL DEFAULT '0' , `out` BIGINT UNSIGNED NOT NULL DEFAULT '0' , `forward_in` BIGINT UNSIGNED NOT NULL DEFAULT '0' , `forward_out` BIGINT UNSIGNED NOT NULL DEFAULT '0' , PRIMARY KEY (`id`), INDEX `times` (`time`)) ENGINE = InnoDB;
+ALTER TABLE `Wan_stats` ADD UNIQUE `main` (`time`, `router_id`, `interface_id`);