removed old migration script

the scanner has been rewritten for parallel polling of devices by snmp

docs/addons/fix-device-models.pl

@@ -4,6 +4,9 @@
 # Copyright (C) Roman Dmitiriev, rnd@rajven.ru
 #
 
+#Скрипт нужен для переноса юзеровских девайсов в системные. НЕ ЗАПУСКАТЬ В ПОЛЬЗОВАТЕЛЬСКОЙ ИНСТАЛЛЯЦИИ!!! 
+#Запуск в пользовательской инсталляции приведёт к ошибкам при следующем обновлении версии БД
+
 use FindBin '$Bin';
 use lib "$Bin/";
 use Data::Dumper;
@@ -21,7 +24,7 @@ my @dev_models = get_records_sql($dbh,"SELECT * FROM device_models WHERE id>=100
 foreach my $row (@dev_models) {
 print "Dev: $row->{id} $row->{model_name}  =>";
 #search hardcoded models with some name
-my $model = get_record_sql($dbh,"SELECT * FROM device_models WHERE id <>".$row->{id}." AND model_name='".trim($row->{model_name})."'");
+my $model = get_record_sql($dbh,"SELECT * FROM device_models WHERE id <>".$row->{id}." AND LOWER(model_name)='".lc(trim($row->{model_name}))."'");
 if ($model) { 
     print "... found id: $model->{id}. Migrated.";
     do_sql($dbh,"DELETE FROM device_models WHERE id=".$row->{id});
@@ -47,7 +50,7 @@ my @dev_vendors = get_records_sql($dbh,"SELECT * FROM vendors WHERE id>=10000");
 foreach my $row (@dev_vendors) {
 print "Dev: $row->{id} $row->{name}  =>";
 #search hardcoded vendors with some name
-my $vendor = get_record_sql($dbh,"SELECT * FROM vendors WHERE id <>".$row->{id}." AND name='".trim($row->{name})."'");
+my $vendor = get_record_sql($dbh,"SELECT * FROM vendors WHERE id <>".$row->{id}." AND LOWER(name)='".lc(trim($row->{name}))."'");
 if ($vendor) {
     print "... found id: $vendor->{id}. Migrated.";
     do_sql($dbh,"DELETE FROM vendors WHERE id=".$row->{id});

docs/addons/migrate-devices.pl

@@ -1,48 +0,0 @@
-#!/usr/bin/perl
-
-#
-# Copyright (C) Roman Dmitiriev, rnd@rajven.ru
-#
-
-use FindBin '$Bin';
-use lib "$Bin/";
-use Data::Dumper;
-use Rstat::config;
-use Rstat::main;
-use Rstat::mysql;
-use Rstat::net_utils;
-use strict;
-use warnings;
-
-print "Stage 1: Devices\n";
-
-my @devices = get_records_sql($dbh,"SELECT * FROM devices");
-foreach my $row (@devices) {
-next if (!$row->{device_model});
-print "Dev: $row->{id} $row->{device_name} Model: $row->{device_model} =>";
-my $model = get_record_sql($dbh,"SELECT * FROM device_models WHERE vendor_id=".$row->{vendor_id}." AND model_name='".trim($row->{device_model})."'");
-if (!$model) { print "... unknown!\n"; next; }
-print "id: $model->{id} $model->{model_name}\n";
-do_sql($dbh,"UPDATE devices SET device_model_id=".$model->{id}." WHERE id=".$row->{id});
-}
-print "Done!\n";
-
-print "Stage 2: Auth\n";
-
-@devices = get_records_sql($dbh,"SELECT * FROM User_auth WHERE host_model IS NOT NULL");
-foreach my $row (@devices) {
-next if (!$row->{host_model});
-print "Auth: $row->{id} $row->{ip} Model: $row->{host_model} =>";
-my $model = get_record_sql($dbh,"SELECT * FROM device_models WHERE model_name='".trim($row->{host_model})."'");
-if (!$model) { print "... unknown!\n"; next; }
-print "id: $model->{id} $model->{model_name}\n";
-do_sql($dbh,"UPDATE User_auth SET device_model_id=".$model->{id}." WHERE id=".$row->{id});
-}
-print "Done!\n";
-
-do_sql($dbh,"ALTER TABLE `User_auth` DROP `host_model`");
-do_sql($dbh,"ALTER TABLE `devices` DROP `device_model`");
-
-print "Done!\n";
-
-exit;

scripts/Rstat/config.pm

@@ -105,6 +105,7 @@ $vpn_networks
 $last_refresh_config
 $tftp_dir
 $tftp_server
+$cpu_count
 );
 
 BEGIN
@@ -213,6 +214,8 @@ our $sw_password=$Config->{_}->{sw_password} || 'admin';
 our $def_timeout = 90;
 our $parallel_process_count = 10;
 
+our $cpu_count = 1;
+
 ################## DB options ##################################
 
 our $save_detail;
@@ -271,5 +274,8 @@ our %switch_auth = (
 
 mkdir $LOG_DIR unless (-d $LOG_DIR);
 
+my @cpu_list = `grep ^processor /proc/cpuinfo`;
+$cpu_count = scalar @cpu_list;
+
 1;
 }

scripts/Rstat/snmp.pm

@@ -23,6 +23,7 @@ get_arp_table
 get_fdb_table
 get_mac_table
 get_vlan_at_port
+get_switch_vlans
 get_interfaces
 get_router_state
 get_bgp
@@ -244,6 +245,25 @@ sub get_vlan_at_port {
 
 #-------------------------------------------------------------------------------------
 
+sub get_switch_vlans {
+    my ($host,$community,$version) = @_;
+    my $port = 161;
+    my $timeout = 5;
+    if (!$version) { $version='2'; }
+    my $result;
+    #need for callback
+    my $vlan_table = snmp_get_oid($host,$community,$port_vlan_oid,$version);
+    if (!$vlan_table) { $vlan_table=snmp_walk_oid($host,$community,$port_vlan_oid,$version); }
+    if ($vlan_table) {
+        foreach my $vlan_oid (keys %$vlan_table) {
+            if ($vlan_oid=~/\.([0-9]*)$/) { $result->{$1} = $vlan_table->{$vlan_oid}; }
+            }
+        }
+    return $result;
+}
+
+#-------------------------------------------------------------------------------------
+
 sub get_interfaces {
     my ($host,$community,$snmp,$skip_empty) = @_;
 #    return if (!HostIsLive($host));

scripts/fetch_new_arp.pl

@@ -20,6 +20,7 @@ use Rstat::snmp;
 use Rstat::mysql;
 use NetAddr::IP;
 use Fcntl qw(:flock);
+use Parallel::ForkManager;
 
 open(SELF,"<",$0) or die "Cannot open $0 - $!";
 flock(SELF, LOCK_EX|LOCK_NB) or exit 1;
@@ -32,6 +33,10 @@ my ($sec,$min,$hour,$day,$month,$year,$zone) = localtime(time());
 $month += 1;
 $year += 1900;
 
+my $fork_count = $cpu_count*5;
+#disable fork for debug
+#if ($debug) { $fork_count = 0; }
+
 my $now_str=sprintf "%04d-%02d-%02d %02d:%02d:%02d",$year,$month,$day,$hour,$min,$sec;
 my $now_day=sprintf "%04d-%02d-%02d",$year,$month,$day;
 
@@ -44,17 +49,50 @@ db_log_verbose($dbh,'Arp discovery started.');
 if ($ARGV[0]) {
     db_log_verbose($dbh,'Active check started!');
     my $subnets=get_subnets_ref($dbh);
+    my @fping_cmd=();
     foreach my $net (keys %$subnets) {
             next if (!$net);
             next if (!$subnets->{$net}{discovery});
             my $run_cmd="$fping -g $subnets->{$net}{subnet} -B1.0 -c 1 >/dev/null 2>&1";
             db_log_debug($dbh,"Checked network $subnets->{$net}{subnet}") if ($debug);
-            do_exec($run_cmd);
+            push(@fping_cmd,$run_cmd);
             }
+    $parallel_process_count = $cpu_count*2;
+    run_in_parallel(@fping_cmd);
     }
 
 my @router_ref = get_records_sql($dbh,"SELECT * FROM devices WHERE deleted=0 AND device_type=2 AND discovery=1 AND snmp_version>0 ORDER by ip" );
 
+my @arp_array=();
+
+my $pm_arp = Parallel::ForkManager->new($fork_count);
+
+# data structure retrieval and handling
+$pm_arp -> run_on_finish (
+sub {
+    my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) = @_;
+    if (defined($data_structure_reference)) {  # children are not forced to send anything
+        my $result = ${$data_structure_reference};  # child passed a string reference
+        push(@arp_array,$result);
+        }
+    }
+);
+
+DATA_LOOP:
+foreach my $router (@router_ref) {
+my $router_ip=$router->{ip};
+my $snmp_version=$router->{snmp_version};
+my $community=$router->{community};
+$pm_arp->start() and next DATA_LOOP;
+my $arp_table=get_arp_table($router_ip,$community,$snmp_version);
+$pm_arp->finish(0, \$arp_table);
+}
+$pm_arp->wait_all_children;
+
+########################### end fork's #########################
+
+$dbh=init_db();
+
 #get userid list
 my @authlist_ref = get_records_sql($dbh,"SELECT * FROM User_auth WHERE deleted=0 ORDER by ip_int" );
 
@@ -68,36 +106,29 @@ $ip_list{$row->{id}}->{ip}=$row->{ip};
 $ip_list{$row->{id}}->{mac}=mac_splitted($row->{mac}) || '';
 }
 
-foreach my $router (@router_ref) {
-my $router_ip=$router->{ip};
-my $snmp_version=$router->{snmp_version};
-my $community=$router->{community};
-#print "Analyze $router_ip $snmp_version $community\n";
-my $arp_table=get_arp_table($router_ip,$community,$snmp_version);
-next if (!$arp_table);
-foreach my $ip (keys %$arp_table) {
-    next if (!$arp_table->{$ip});
-    my $mac=trim($arp_table->{$ip});
-    $mac=mac_splitted($mac);
-    next if (!$mac);
-    next if ($mac=~/ff:ff:ff:ff:ff:ff/i);
-    next if ($mac!~/(\S{2}):(\S{2}):(\S{2}):(\S{2}):(\S{2}):(\S{2})/);
-    my $simple_mac=mac_simplify($mac);
-    $ip=trim($ip);
-    my $ip_aton=StrToIp($ip);
-    $mac_history{$simple_mac}{changed}=0;
-    $mac_history{$simple_mac}{ip}=$ip;
-    $mac_history{$simple_mac}{auth_id}=0;
-    next if (!$office_networks->match_string($ip));
-    db_log_debug($dbh,"Analyze ip: $ip mac: $mac") if ($debug);
-    my $auth_id = $users->match_string($ip);
-    my $cur_auth_id=resurrection_auth($dbh,$ip,$mac,'arp');
-    $mac_history{$simple_mac}{auth_id}=$cur_auth_id;
-    if ($auth_id ne $cur_auth_id) {
-	$mac_history{$simple_mac}{changed}=1;
-	}
+foreach my $arp_table (@arp_array) {
+    foreach my $ip (keys %$arp_table) {
+        next if (!$arp_table->{$ip});
+        my $mac=trim($arp_table->{$ip});
+        $mac=mac_splitted($mac);
+        next if (!$mac);
+        next if ($mac=~/ff:ff:ff:ff:ff:ff/i);
+        next if ($mac!~/(\S{2}):(\S{2}):(\S{2}):(\S{2}):(\S{2}):(\S{2})/);
+        my $simple_mac=mac_simplify($mac);
+        $ip=trim($ip);
+        my $ip_aton=StrToIp($ip);
+        $mac_history{$simple_mac}{changed}=0;
+        $mac_history{$simple_mac}{ip}=$ip;
+        $mac_history{$simple_mac}{auth_id}=0;
+        next if (!$office_networks->match_string($ip));
+        db_log_debug($dbh,"Analyze ip: $ip mac: $mac") if ($debug);
+        my $auth_id = $users->match_string($ip);
+        my $cur_auth_id=resurrection_auth($dbh,$ip,$mac,'arp');
+        $mac_history{$simple_mac}{auth_id}=$cur_auth_id;
+        if ($auth_id ne $cur_auth_id) { $mac_history{$simple_mac}{changed}=1; }
     }
 }
+
 db_log_verbose($dbh,'Arp discovery stopped.');
 }
 
@@ -110,110 +141,133 @@ sleep(1);
 db_log_verbose($dbh,'Mac discovery started.');
 
 my %connections=();
-my $connections_list=do_sql($dbh,"Select id,auth_id,port_id from connections order by auth_id");
-foreach my $connection (@$connections_list) {
+my @connections_list=get_records_sql($dbh,"SELECT * FROM connections ORDER BY auth_id");
+foreach my $connection (@connections_list) {
     next if (!$connection);
-    my ($conn_id,$conn_auth_id,$conn_port_id)=@$connection;
-    $connections{$conn_auth_id}{port}=$conn_port_id;
-    $connections{$conn_auth_id}{id}=$conn_id;
+    $connections{$connection->{auth_id}}{port}=$connection->{port_id};
+    $connections{$connection->{auth_id}}{id}=$connection->{id};
     }
 
 my $auth_filter='';
-if ($arp_discovery) { $auth_filter=" and last_found >='".$now_day."' "; }
-my $auth_sql="Select id,mac from User_auth where mac is not null and deleted=0 $auth_filter order by id asc";
+if ($arp_discovery) { $auth_filter=" AND last_found >='".$now_day."' "; }
+my $auth_sql="SELECT id,mac FROM User_auth WHERE mac IS NOT NULL AND deleted=0 $auth_filter ORDER BY id ASC";
 
-my $auth_list=do_sql($dbh,$auth_sql);
+my @auth_list=get_records_sql($dbh,$auth_sql);
 
 my %auth_table;
-foreach my $auth (@$auth_list) {
+foreach my $auth (@auth_list) {
     next if (!$auth);
-    my ($auth_id,$auth_mac)=@$auth;
-    $auth_mac=mac_simplify($auth_mac);
-    $auth_table{oper_table}{$auth_mac}=$auth_id;
+    my $auth_mac=mac_simplify($auth->{mac});
+    $auth_table{oper_table}{$auth_mac}=$auth->{id};
     }
 
-
-$auth_sql="Select id,mac from User_auth where mac is not null and deleted=0 order by id asc";
-my $auth_full_list=do_sql($dbh,$auth_sql);
-
-foreach my $auth (@$auth_full_list) {
+$auth_sql="SELECT id,mac FROM User_auth WHERE mac IS NOT NULL AND deleted=0 ORDER BY id ASC";
+my @auth_full_list=get_records_sql($dbh,$auth_sql);
+foreach my $auth (@auth_full_list) {
     next if (!$auth);
-    my ($auth_id,$auth_mac)=@$auth;
-    $auth_mac=mac_simplify($auth_mac);
-    $auth_table{full_table}{$auth_mac}=$auth_id;
+    my $auth_mac=mac_simplify($auth->{mac});
+    $auth_table{full_table}{$auth_mac}=$auth->{id};
     }
 
-
-my $unknown_list=do_sql($dbh,"Select id,mac,port_id,device_id from Unknown_mac where mac !='' order by mac");
+my @unknown_list=get_records_sql($dbh,"SELECT id,mac,port_id,device_id FROM Unknown_mac WHERE mac !='' ORDER BY mac");
 my %unknown_table;
-foreach my $unknown (@$unknown_list) {
+foreach my $unknown (@unknown_list) {
     next if (!$unknown);
-    my ($unknown_id,$unknown_mac,$unknown_port_id,$unknown_device_id)=@$unknown;
-    $unknown_mac=mac_simplify($unknown_mac);
-    $unknown_table{$unknown_mac}{unknown_id}=$unknown_id;
-    $unknown_table{$unknown_mac}{port_id}=$unknown_port_id;
-    $unknown_table{$unknown_mac}{device_id}=$unknown_device_id;
+    my $unknown_mac=mac_simplify($unknown->{mac});
+    $unknown_table{$unknown_mac}{unknown_id}=$unknown->{id};
+    $unknown_table{$unknown_mac}{port_id}=$unknown->{port_id};
+    $unknown_table{$unknown_mac}{device_id}=$unknown->{device_id};
+    }
+
+my @device_list = get_records_sql($dbh,"SELECT * FROM devices WHERE deleted=0 AND discovery=1 AND snmp_version>0" );
+
+my @fdb_array=();
+
+my $pm_fdb = Parallel::ForkManager->new($fork_count);
+
+# data structure retrieval and handling
+$pm_fdb -> run_on_finish (
+sub {
+    my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) = @_;
+    if (defined($data_structure_reference)) {  # children are not forced to send anything
+        my $result = ${$data_structure_reference};  # child passed a string reference
+        push(@fdb_array,$result);
+        }
     }
+);
+
+
+FDB_LOOP:
+foreach my $device (@device_list) {
+$pm_fdb->start() and next FDB_LOOP;
+my $fdb=get_fdb_table($device->{ip},$device->{community},$device->{snmp_version});
+my $vlans = get_switch_vlans($device->{ip},$device->{community},$device->{snmp_version});
+my $result;
+$result->{id}=$device->{id};
+$result->{fdb} = $fdb;
+$result->{vlans} = $vlans;
+$pm_fdb->finish(0, \$result);
+}
+$pm_fdb->wait_all_children;
+
+my %fdb_ref;
+foreach my $fdb_table (@fdb_array){
+next if (!$fdb_table);
+$fdb_ref{$fdb_table->{id}}{fdb}=$fdb_table->{fdb};
+$fdb_ref{$fdb_table->{id}}{vlans}=$fdb_table->{vlans};
+}
+
+################################ end fork's ##############################
 
-my $device_ref = do_sql($dbh,"SELECT ip,snmp_version,community,fdb_snmp_index,device_name,id from devices WHERE deleted=0 and discovery=1 and snmp_version>0" );
-foreach my $device (@$device_ref) {
+$dbh=init_db();
 
+foreach my $device (@device_list) {
 my %port_snmp_index=();
 my %port_index=();
 my %mac_port_count=();
 my %mac_address_table=();
 my %port_links=();
 
-my ($dev_ip,$dev_snmp_ver,$dev_community,$dev_fdb_index,$dev_name,$dev_id)=@$device;
-
-next if (!$dev_id);
-
-#print "$dev_ip,$dev_snmp_ver,$dev_community,$dev_fdb_index,$dev_name,$dev_id\n" if ($debug);
-
-my $fdb=get_fdb_table($dev_ip,$dev_community,$dev_snmp_ver);
+my $dev_id = $device->{id};
+my $dev_name = $device->{device_name};
+my $fdb=$fdb_ref{$dev_id}{fdb};
+my $vlans=$fdb_ref{$dev_id}{vlans};
 
 next if (!$fdb);
 
+my @device_ports = get_records_sql($dbh,"SELECT * FROM device_ports WHERE device_id=$dev_id");
 
-my $device_ports = do_sql($dbh,"Select port,snmp_index,target_port_id,id,skip,vlan from device_ports where device_id=$dev_id");
-
-foreach my $port (@$device_ports) {
-    my ($port,$snmp_index,$target_port_id,$port_id,$skip_port,$vlan)=@$port;
-
-    my $port_index=$port;
+foreach my $port_data (@device_ports) {
+    my $vlan = $port_data->{vlan};
     if (!$vlan) { $vlan=1; }
 
-    if ($dev_fdb_index) {
-        if (!$snmp_index) { next; }
-        $port_index=$snmp_index;
-        }
-
-    my $current_vlan =  get_vlan_at_port($dev_ip,$dev_community,$dev_snmp_ver,$port_index);
-
-    if (!$current_vlan or $current_vlan=~/noSuchInstance/i or !is_integer($current_vlan)) { $current_vlan=1; }
+    if (!$port_data->{snmp_index}) { $port_data->{snmp_index} = $port_data->{port}; }
+    my $port_index=$port_data->{snmp_index};
 
+    my $current_vlan = $vlans->{$port_data->{snmp_index}};
+    if (!$current_vlan) { $current_vlan=1; }
     if ($current_vlan != $vlan) {
 	my $dev_ports;
 	$dev_ports->{vlan}=$current_vlan;
-	update_record($dbh,'device_ports',$dev_ports,"device_id=$dev_id and port=$port");
-        db_log_verbose($dbh,"Vlan changed at device $dev_name [$port] old: $vlan current: $current_vlan");
+	update_record($dbh,'device_ports',$dev_ports,"device_id=$dev_id and port=$port_data->{port}");
+        db_log_verbose($dbh,"Vlan changed at device $dev_name [$port_data->{port}] old: $vlan current: $current_vlan");
         }
 
-    next if ($skip_port);
-    $port_snmp_index{$snmp_index}=$port;
-    $port_index{$port}=$port_id;
-    $port_links{$port}=$target_port_id;
-    $mac_port_count{$port}=0;
+    next if ($port_data->{skip});
+
+    $port_snmp_index{$port_data->{snmp_index}}=$port_data->{port};
+    $port_index{$port_data->{port}}=$port_data->{id};
+    $port_links{$port_data->{port}}=$port_data->{target_port_id};
+    $mac_port_count{$port_data->{port}}=0;
     }
 
 foreach my $mac (keys %$fdb) {
+    #port from fdb table
     my $port = $fdb->{$mac};
     next if (!$port);
-    if ($dev_fdb_index) {
-        if (!exists $port_snmp_index{$port}) { next; }
-        $port=$port_snmp_index{$port};
-        }
-    if (!exists $port_index{$port}) { next; }
+    if (!exists $port_snmp_index{$port}) { next; }
+    #real port number
+    $port=$port_snmp_index{$port};
     $mac_port_count{$port}++;
     $mac_address_table{$mac}=$port;
     }