added save netflow as csv

added verification of the existence of a computer in the AD domain when updating DNS by DHCP request

Readme.en.md

@@ -8,7 +8,7 @@ libnet-patricia-perl libnetaddr-ip-perl libconfig-tiny-perl libnet-dns-perl libd
 libnet-netmask-perl libtext-iconv-perl libnet-snmp-perl libnet-telnet-perl libdbi-perl \
 libdbd-mysql-perl libparallel-forkmanager-perl libproc-daemon-perl libdatetime-format-dateparse-perl \
 libnetwork-ipv4addr-perl libnet-openssh-perl libfile-tail-perl php-fpm php-db libapache2-mod-fcgid \
-libcrypt-cbc-perl libcryptx-perl php-pgsql libdbd-pg-perl
+libcrypt-cbc-perl libcryptx-perl php-pgsql libdbd-pg-perl libfile-path-tiny-perl
 
 2. Download the source code and spread it in catalogs:
 

Readme.ru.md

@@ -8,7 +8,7 @@ libnet-patricia-perl libnetaddr-ip-perl libconfig-tiny-perl libnet-dns-perl libd
 libnet-netmask-perl libtext-iconv-perl libnet-snmp-perl libnet-telnet-perl libdbi-perl \
 libdbd-mysql-perl libparallel-forkmanager-perl libproc-daemon-perl libdatetime-format-dateparse-perl \
 libnetwork-ipv4addr-perl libnet-openssh-perl libfile-tail-perl php-fpm php-db libapache2-mod-fcgid \
-libcrypt-cbc-perl libcryptx-perl php-pgsql libdbd-pg-perl
+libcrypt-cbc-perl libcryptx-perl php-pgsql libdbd-pg-perl libfile-path-tiny-perl
 
 2. Качаем исходники и раскидываем по каталогам:
 

scripts/eye-statd.pl

@@ -20,6 +20,7 @@ use eyelib::database;
 use Socket qw(AF_INET6 inet_ntop);
 use IO::Socket;
 use Data::Dumper;
+use File::Path::Tiny;
 use threads;
 
 my @router_ref = ();
@@ -39,6 +40,7 @@ my %user_stats;
 my $MAXREAD = 9216;
 
 my $timeshift = get_option($dbh,55)*60;
+my $save_path =  get_option($dbh,72);
 
 my $thread_count = $cpu_count;
 
@@ -80,6 +82,8 @@ InitSubnets();
 
 init_option($hdb);
 
+$save_path = get_option($dbh,72);
+
 $timeshift = get_option($hdb,55)*60;
 
 @router_ref = get_records_sql($hdb,"SELECT * FROM devices WHERE deleted=0 AND device_type=2 AND snmp_version>0 ORDER by ip" );
@@ -90,7 +94,8 @@ foreach my $row (@router_ref) {
     $routers{$row->{id}}=$row;
     my @auth_list = get_records_sql($hdb,"SELECT ip FROM User_auth WHERE deleted=0 AND user_id=".$row->{user_id});
     foreach my $auth (@auth_list) {
-	$router_svi{$auth->{ip}}=$row->{id};
+	$router_svi{$auth->{ip}}{id}=$row->{id};
+	$router_svi{$auth->{ip}}{save}=$row->{netflow_save};
 	}
     }
 
@@ -379,9 +384,10 @@ sub save_flow {
 	my $router_id;
 	#skip unknown router
 	if (exists $router_svi{$router_ip}) { 
-		$router_id = $router_svi{$router_ip};
+		$router_id = $router_svi{$router_ip}{id};
 		$flow->{router_ip} = $router_ip;
 		$flow->{device_id} = $router_id;
+		$flow->{save} = $router_svi{$router_ip}{save};
 		} else { return; }
 	#skip input traffic for router
 	if (exists $wan_dev{$router_id}->{$flow->{snmp_out}} and exists $wan_dev{$router_id}->{$flow->{snmp_in}}) { return; }
@@ -428,6 +434,7 @@ my $hdb=init_db();
 
 #saved packet by users
 my @detail_traffic = ();
+my @saved_netflow = ();
 
 my %routers_found;
 
@@ -440,6 +447,10 @@ my ($auth_id,$l_src_ip,$l_dst_ip,$user_ip,$router_id);
 
 $router_id = $traf_record->{device_id};
 
+if ($traf_record->{save}) {
+    push(@saved_netflow,join(';',$traf_record->{starttime},$traf_record->{device_id},$traf_record->{proto},$traf_record->{snmp_in},$traf_record->{snmp_out},$traf_record->{src_ip},$traf_record->{dst_ip},$traf_record->{xsrc_ip},$traf_record->{xdst_ip},$traf_record->{src_port},$traf_record->{dst_port},$traf_record->{octets},$traf_record->{pkts}));
+    }
+
 $routers_found{$router_id} = 1;
 
 #outbound traffic
@@ -510,10 +521,27 @@ push(@detail_traffic,\@detail_array);
 
 @flush_table=();
 
-#save statistics
-
 #start hour
-my ($min,$hour,$day,$month,$year) = (localtime($last_time))[1,2,3,4,5];
+my ($sec,$min,$hour,$day,$month,$year) = (localtime($last_time))[0,1,2,3,4,5];
+
+#save netflow
+$save_path=~s/\/$//;
+my $netflow_file_path = $save_path.'/'.sprintf "%04d/%02d/%02d/%02d/",$year+1900,$month+1,$day,$hour;
+my $netflow_file_name = $netflow_file_path.sprintf "%04d%02d%02d-%02d%02d%02d.csv",$year+1900,$month+1,$day,$hour,$min,$sec;
+if (scalar @saved_netflow) {
+    File::Path::Tiny::::mk($netflow_file_path);
+    open (ND,">$netflow_file_name") || die("Error open file $netflow_file_name!!! die...");
+    binmode(ND,':utf8');
+    print ND join(';',"time","device_id","proto","snmp_in","snmp_out","src_ip","dst_ip","xsrc_ip","xdst_ip","src_port","dst_port","octets","pkts")."\n";
+    foreach my $row (@saved_netflow) {
+        next if (!$row);
+        print ND $row."\n";
+        }
+    close ND;
+    @saved_netflow=();
+    }
+
+#save statistics
 
 #start stat time
 my $hour_date1 = $hdb->quote(sprintf "%04d-%02d-%02d %02d:00:00",$year+1900,$month+1,$day,$hour);

scripts/eyelib/database.pm

@@ -62,6 +62,7 @@ resurrection_auth
 new_auth
 StrToIp
 get_first_line
+is_ad_computer
 update_dns_record
 update_dns_record_by_dhcp
 create_dns_cname
@@ -203,7 +204,7 @@ if ($config_ref{DBTYPE} eq 'mysql') {
 
 my $table= shift;
 my $data = shift;
-my $fh = File::Temp->new(UNLINK=>1);
+my $fh = File::Temp->new(UNLINK=>0);
 my $fname = $fh->filename;
 binmode($fh,':utf8');
 foreach my $row (@$data) {
@@ -220,7 +221,7 @@ close $fh;
 my $query = qq{ LOAD DATA LOCAL INFILE '$fname' INTO TABLE $table FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n'; };
 $db->do($query);
 $db->disconnect;
-File::Temp::cleanup();
+#File::Temp::cleanup();
 }
 
 #---------------------------------------------------------------------------------------------------------------
@@ -935,6 +936,37 @@ if ($@) { log_error("Error dns commands: $@"); }
 
 }
 
+#---------------------------------------------------------------------------------------------------------------
+sub is_ad_computer {
+
+my $hdb = shift;
+my $computer_name = shift;
+
+my $ad_check = get_option($hdb,73);
+if (!$ad_check) { return 1; }
+
+my $ad_zone = get_option($hdb,33);
+
+if ($computer_name =~/\./) {
+    if ($computer_name!~/\.$ad_zone$/i) { return 0; }
+    }
+
+if ($computer_name =~/^(.+)\./) {
+    $computer_name = $1;
+    }
+
+my $ad_computer_name = trim($computer_name).'$';
+
+my $name_in_cache = get_record_sql($hdb,"SELECT * FROM ad_comp_cache WHERE name='".$computer_name."'");
+if ($name_in_cache) { return 1; }
+
+my $name_found=do_exec('/usr/bin/getent passwd '.$ad_computer_name);
+if (!$name_found) { return 0; }
+
+do_sql($hdb,"INSERT INTO ad_comp_cache(name) VALUES('".$computer_name."') ON DUPLICATE KEY UPDATE name='".$computer_name."';");
+return 1;
+}
+
 #---------------------------------------------------------------------------------------------------------------
 
 sub update_dns_record_by_dhcp {
@@ -957,7 +989,7 @@ log_debug("Subnets: ".Dumper($subnets_dhcp->{$dhcp_record->{network}->{subnet}})
 log_debug("enable_ad_dns_update: ".$enable_ad_dns_update);
 log_debug("DNS update flags - zone: ".$ad_zone.",dns: ".$ad_dns.", update_hostname_from_dhcp: ".$update_hostname_from_dhcp.", enable_ad_dns_update: ".$enable_ad_dns_update);
 
-my $maybe_update_dns=(($dhcp_record->{type}=~/add/i or $dhcp_record->{type}=~/old/i) and $dhcp_record->{hostname_utf8} and $dhcp_record->{hostname_utf8} !~/UNDEFINED/i and $enable_ad_dns_update and $subnets_dhcp->{$dhcp_record->{network}->{subnet}}->{dhcp_update_hostname});
+my $maybe_update_dns=(is_ad_computer($hdb,$dhcp_record->{hostname_utf8}) and ($dhcp_record->{type}=~/add/i or $dhcp_record->{type}=~/old/i) and $dhcp_record->{hostname_utf8} and $dhcp_record->{hostname_utf8} !~/UNDEFINED/i and $enable_ad_dns_update and $subnets_dhcp->{$dhcp_record->{network}->{subnet}}->{dhcp_update_hostname});
 if (!$maybe_update_dns) {
     db_log_debug($hdb,"FOUND Auth_id: $auth_record->{id}. DNS update don't needed.");
     return 0;
@@ -2013,6 +2045,13 @@ $year += 1900;
 my $now_str=sprintf "%04d-%02d-%02d %02d:%02d:%02d",$year,$month,$day,$hour,$min,$sec;
 my $clean_variables_date=$db->quote($now_str);
 do_sql($db,"DELETE FROM `variables` WHERE clear_time<=$clean_variables_date");
+
+#clean old AD computer cache
+my $now = DateTime->now(time_zone=>'local');
+my $day_dur = DateTime::Duration->new( days => 1 );
+my $clean_date = $now - $day_dur;
+my $clean_str = $dbh->quote($clean_date->ymd("-")." 00:00:00");
+do_sql($db,"DELETE FROM `ad_comp_cache` WHERE last_found<=$clean_str");
 }
 
 #---------------------------------------------------------------------------------------------------------------

scripts/updates/2-7-0/Traffic_detail.sql

@@ -0,0 +1,4 @@
+ALTER TABLE `Traffic_detail` CHANGE `proto` `proto` TINYINT UNSIGNED NULL DEFAULT NULL;
+ALTER TABLE `Traffic_detail` CHANGE `src_ip` `src_ip` INT UNSIGNED NOT NULL;
+ALTER TABLE `Traffic_detail` CHANGE `dst_ip` `dst_ip` INT UNSIGNED NOT NULL;
+ALTER TABLE `Traffic_detail` CHANGE `pkt` `pkt` INT UNSIGNED NOT NULL DEFAULT '0';

scripts/updates/2-7-0/ad_computer_cache.sql

@@ -0,0 +1 @@
+CREATE TABLE `ad_comp_cache` ( `id` INT NOT NULL AUTO_INCREMENT , `name` VARCHAR(63) NOT NULL , `last_found` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP , PRIMARY KEY (`id`), UNIQUE `comp_name` (`name`)) ENGINE = InnoDB;

scripts/updates/2-7-0/config_options.sql

@@ -0,0 +1,7 @@
+INSERT INTO `config_options` (`id`, `option_name`, `description.russian`, `description.english`, `uniq`, `type`, `default_value`, `min_value`, `max_value`) VALUES ('72', 'netflow_path', 'Каталог для хранения данных, полученных по netflow от маршрутизаторов', 'The directory for storing data received via netflow from routers', '1', 'text', '/opt/Eye/netflow', '0', '0');
+INSERT INTO `config_options` (`id`, `option_name`, `description.russian`, `description.english`, `uniq`, `type`, `default_value`, `min_value`, `max_value`) VALUES ('73', 'check_computer_exists', 'Проверять существование компьютера в домене перед обновлением DNS по DHCP запросу', 'Verify the existence of a computer in the domain before updating DNS by DHCP request', '1', 'bool', '1', '0', '0');
+ALTER TABLE `config_options` ADD `draft` BOOLEAN NOT NULL DEFAULT FALSE AFTER `description.english`;
+UPDATE `config_options` SET `draft` = '1' WHERE `config_options`.`id` = 65;
+UPDATE `config_options` SET `draft` = '1' WHERE `config_options`.`id` = 66;
+UPDATE `config_options` SET `draft` = '1' WHERE `config_options`.`id` = 67;
+

scripts/updates/2-7-0/devices.sql

@@ -0,0 +1 @@
+ALTER TABLE `devices` ADD `netflow_save` BOOLEAN NOT NULL DEFAULT FALSE AFTER `discovery`;