rewrite migration

docs/databases/migrate2psql.pl

@@ -24,468 +24,30 @@ use warnings;
 
 my $chunk_count = 1000;
 
-my %pg_schema = (
-    'acl' => {
-        'id' => 'SERIAL',
-        'name' => 'VARCHAR(30)',
-        'description_english' => 'VARCHAR(250)',
-        'description_russian' => 'VARCHAR(250)',
-    },
-    'ad_comp_cache' => {
-        'id' => 'SERIAL',
-        'name' => 'VARCHAR(63)',
-        'last_found' => 'TIMESTAMP',
-    },
-    'auth_rules' => {
-        'id' => 'SERIAL',
-        'user_id' => 'INTEGER',
-        'ou_id' => 'INTEGER',
-        'rule_type' => 'SMALLINT',
-        'rule' => 'VARCHAR(40)',
-        'description' => 'VARCHAR(250)',
-    },
-    'building' => {
-        'id' => 'SERIAL',
-        'name' => 'VARCHAR(50)',
-        'description' => 'VARCHAR(250)',
-    },
-    'config' => {
-        'id' => 'SERIAL',
-        'option_id' => 'INTEGER',
-        'value' => 'VARCHAR(250)',
-    },
-    'config_options' => {
-        'id' => 'SERIAL',
-        'option_name' => 'VARCHAR(50)',
-        'description_russian' => 'TEXT',
-        'description_english' => 'TEXT',
-        'draft' => 'SMALLINT',
-        'uniq' => 'SMALLINT',
-        'option_type' => 'VARCHAR(100)',
-        'default_value' => 'VARCHAR(250)',
-        'min_value' => 'INTEGER',
-        'max_value' => 'INTEGER',
-    },
-    'connections' => {
-        'id' => 'BIGSERIAL',
-        'device_id' => 'BIGINT',
-        'port_id' => 'BIGINT',
-        'auth_id' => 'BIGINT',
-        'last_found' => 'TIMESTAMP',
-    },
-    'customers' => {
-        'id' => 'SERIAL',
-        'login' => 'VARCHAR(20)',
-        'description' => 'VARCHAR(100)',
-        'password' => 'VARCHAR(255)',
-        'api_key' => 'VARCHAR(255)',
-        'rights' => 'SMALLINT',
-    },
-    'devices' => {
-        'id' => 'SERIAL',
-        'device_type' => 'INTEGER',
-        'device_model_id' => 'INTEGER',
-        'firmware' => 'VARCHAR(100)',
-        'vendor_id' => 'INTEGER',
-        'device_name' => 'VARCHAR(50)',
-        'building_id' => 'INTEGER',
-        'ip' => 'INET',
-        'ip_int' => 'BIGINT',
-        'login' => 'VARCHAR(50)',
-        'password' => 'VARCHAR(255)',
-        'protocol' => 'SMALLINT',
-        'control_port' => 'INTEGER',
-        'port_count' => 'INTEGER',
-        'sn' => 'VARCHAR(80)',
-        'description' => 'VARCHAR(255)',
-        'snmp_version' => 'SMALLINT',
-        'snmp3_auth_proto' => 'VARCHAR(10)',
-        'snmp3_priv_proto' => 'VARCHAR(10)',
-        'snmp3_user_rw' => 'VARCHAR(20)',
-        'snmp3_user_rw_password' => 'VARCHAR(20)',
-        'snmp3_user_ro' => 'VARCHAR(20)',
-        'snmp3_user_ro_password' => 'VARCHAR(20)',
-        'community' => 'VARCHAR(50)',
-        'rw_community' => 'VARCHAR(50)',
-        'fdb_snmp_index' => 'SMALLINT',
-        'discovery' => 'SMALLINT',
-        'netflow_save' => 'SMALLINT',
-        'user_acl' => 'SMALLINT',
-        'dhcp' => 'SMALLINT',
-        'nagios' => 'SMALLINT',
-        'active' => 'SMALLINT',
-        'nagios_status' => 'VARCHAR(10)',
-        'queue_enabled' => 'SMALLINT',
-        'connected_user_only' => 'SMALLINT',
-        'user_id' => 'INTEGER',
-        'deleted' => 'SMALLINT',
-        'discovery_locked' => 'SMALLINT',
-        'locked_timestamp' => 'TIMESTAMP',
-    },
-    'device_filter_instances' => {
-        'id' => 'SERIAL',
-        'instance_id' => 'INTEGER',
-        'device_id' => 'INTEGER',
-    },
-    'device_l3_interfaces' => {
-        'id' => 'SERIAL',
-        'device_id' => 'INTEGER',
-        'snmpin' => 'INTEGER',
-        'interface_type' => 'SMALLINT',
-        'name' => 'VARCHAR(100)',
-    },
-    'device_models' => {
-        'id' => 'SERIAL',
-        'model_name' => 'VARCHAR(200)',
-        'vendor_id' => 'INTEGER',
-        'poe_in' => 'SMALLINT',
-        'poe_out' => 'SMALLINT',
-        'nagios_template' => 'VARCHAR(200)',
-    },
-    'device_ports' => {
-        'id' => 'BIGSERIAL',
-        'device_id' => 'INTEGER',
-        'snmp_index' => 'INTEGER',
-        'port' => 'INTEGER',
-        'ifname' => 'VARCHAR(40)',
-        'port_name' => 'VARCHAR(40)',
-        'description' => 'VARCHAR(50)',
-        'target_port_id' => 'INTEGER',
-        'auth_id' => 'BIGINT',
-        'last_mac_count' => 'INTEGER',
-        'uplink' => 'SMALLINT',
-        'nagios' => 'SMALLINT',
-        'skip' => 'SMALLINT',
-        'vlan' => 'INTEGER',
-        'tagged_vlan' => 'VARCHAR(250)',
-        'untagged_vlan' => 'VARCHAR(250)',
-        'forbidden_vlan' => 'VARCHAR(250)',
-    },
-    'device_types' => {
-        'id' => 'SERIAL',
-        'name_russian' => 'VARCHAR(50)',
-        'name_english' => 'VARCHAR(50)',
-    },
-    'dhcp_log' => {
-        'id' => 'BIGSERIAL',
-        'mac' => 'MACADDR',
-        'ip_int' => 'BIGINT',
-        'ip' => 'INET',
-        'action' => 'VARCHAR(10)',
-        'ts' => 'TIMESTAMP',
-        'auth_id' => 'BIGINT',
-        'dhcp_hostname' => 'VARCHAR(250)',
-        'circuit_id' => 'VARCHAR(255)',
-        'remote_id' => 'VARCHAR(255)',
-        'client_id' => 'VARCHAR(250)',
-    },
-    'dhcp_queue' => {
-        'id' => 'BIGSERIAL',
-        'mac' => 'MACADDR',
-        'ip' => 'INET',
-        'action' => 'VARCHAR(10)',
-        'ts' => 'TIMESTAMP',
-        'dhcp_hostname' => 'VARCHAR(250)',
-    },
-    'dns_cache' => {
-        'id' => 'BIGSERIAL',
-        'dns' => 'VARCHAR(250)',
-        'ip' => 'BIGINT',
-        'ts' => 'TIMESTAMP',
-    },
-    'dns_queue' => {
-        'id' => 'SERIAL',
-        'auth_id' => 'INTEGER',
-        'name_type' => 'VARCHAR(10)',
-        'name' => 'VARCHAR(200)',
-        'operation_type' => 'VARCHAR(10)',
-        'value' => 'VARCHAR(100)',
-    },
-    'filter_instances' => {
-        'id' => 'SERIAL',
-        'name' => 'VARCHAR(50)',
-        'description' => 'VARCHAR(200)',
-    },
-    'filter_list' => {
-        'id' => 'SERIAL',
-        'name' => 'VARCHAR(50)',
-        'description' => 'VARCHAR(250)',
-        'proto' => 'VARCHAR(10)',
-        'dst' => 'TEXT',
-        'dstport' => 'VARCHAR(20)',
-        'srcport' => 'VARCHAR(20)',
-        'filter_type' => 'SMALLINT',
-    },
-    'gateway_subnets' => {
-        'id' => 'SERIAL',
-        'device_id' => 'INTEGER',
-        'subnet_id' => 'INTEGER',
-    },
-    'group_filters' => {
-        'id' => 'SERIAL',
-        'group_id' => 'INTEGER',
-        'filter_id' => 'INTEGER',
-        'rule_order' => 'INTEGER',
-        'action' => 'SMALLINT',
-    },
-    'group_list' => {
-        'id' => 'SERIAL',
-        'instance_id' => 'INTEGER',
-        'group_name' => 'VARCHAR(50)',
-        'description' => 'VARCHAR(250)',
-    },
-    'mac_history' => {
-        'id' => 'BIGSERIAL',
-        'mac' => 'VARCHAR(12)',
-        'ts' => 'TIMESTAMP',
-        'device_id' => 'BIGINT',
-        'port_id' => 'BIGINT',
-        'ip' => 'INET',
-        'auth_id' => 'BIGINT',
-        'dhcp_hostname' => 'VARCHAR(250)',
-    },
-    'mac_vendors' => {
-        'id' => 'SERIAL',
-        'oui' => 'VARCHAR(20)',
-        'companyname' => 'VARCHAR(255)',
-        'companyaddress' => 'VARCHAR(255)',
-    },
-    'ou' => {
-        'id' => 'SERIAL',
-        'ou_name' => 'VARCHAR(40)',
-        'description' => 'VARCHAR(250)',
-        'default_users' => 'SMALLINT',
-        'default_hotspot' => 'SMALLINT',
-        'nagios_dir' => 'VARCHAR(255)',
-        'nagios_host_use' => 'VARCHAR(50)',
-        'nagios_ping' => 'SMALLINT',
-        'nagios_default_service' => 'VARCHAR(100)',
-        'enabled' => 'SMALLINT',
-        'filter_group_id' => 'INTEGER',
-        'queue_id' => 'INTEGER',
-        'dynamic' => 'SMALLINT',
-        'life_duration' => 'DECIMAL(10,2)',
-        'parent_id' => 'INTEGER',
-    },
-    'queue_list' => {
-        'id' => 'SERIAL',
-        'queue_name' => 'VARCHAR(20)',
-        'download' => 'INTEGER',
-        'upload' => 'INTEGER',
-    },
-    'remote_syslog' => {
-        'id' => 'BIGSERIAL',
-        'ts' => 'TIMESTAMP',
-        'device_id' => 'BIGINT',
-        'ip' => 'INET',
-        'message' => 'TEXT',
-    },
-    'sessions' => {
-        'id' => 'VARCHAR(128)',
-        'data' => 'TEXT',
-        'last_accessed' => 'INTEGER',
-    },
-    'subnets' => {
-        'id' => 'SERIAL',
-        'subnet' => 'VARCHAR(18)',
-        'vlan_tag' => 'INTEGER',
-        'ip_int_start' => 'BIGINT',
-        'ip_int_stop' => 'BIGINT',
-        'dhcp_start' => 'BIGINT',
-        'dhcp_stop' => 'BIGINT',
-        'dhcp_lease_time' => 'INTEGER',
-        'gateway' => 'BIGINT',
-        'office' => 'SMALLINT',
-        'hotspot' => 'SMALLINT',
-        'vpn' => 'SMALLINT',
-        'free' => 'SMALLINT',
-        'dhcp' => 'SMALLINT',
-        'static' => 'SMALLINT',
-        'dhcp_update_hostname' => 'SMALLINT',
-        'discovery' => 'SMALLINT',
-        'notify' => 'SMALLINT',
-        'description' => 'VARCHAR(250)',
-    },
-    'traffic_detail' => {
-        'id' => 'BIGSERIAL',
-        'auth_id' => 'BIGINT',
-        'router_id' => 'INTEGER',
-        'ts' => 'TIMESTAMP',
-        'proto' => 'SMALLINT',
-        'src_ip' => 'BIGINT',
-        'dst_ip' => 'BIGINT',
-        'src_port' => 'INTEGER',
-        'dst_port' => 'INTEGER',
-        'bytes' => 'BIGINT',
-        'pkt' => 'BIGINT',
-    },
-    'unknown_mac' => {
-        'id' => 'BIGSERIAL',
-        'mac' => 'VARCHAR(12)',
-        'port_id' => 'BIGINT',
-        'device_id' => 'INTEGER',
-        'ts' => 'TIMESTAMP',
-    },
-    'user_auth' => {
-        'id' => 'SERIAL',
-        'user_id' => 'BIGINT',
-        'ou_id' => 'INTEGER',
-        'ip' => 'INET',
-        'ip_int' => 'BIGINT',
-        'save_traf' => 'SMALLINT',
-        'enabled' => 'SMALLINT',
-        'dhcp' => 'SMALLINT',
-        'filter_group_id' => 'SMALLINT',
-        'dynamic' => 'SMALLINT',
-        'end_life' => 'TIMESTAMP',
-        'deleted' => 'SMALLINT',
-        'description' => 'VARCHAR(250)',
-        'dns_name' => 'VARCHAR(253)',
-        'dns_ptr_only' => 'SMALLINT',
-        'wikiname' => 'VARCHAR(250)',
-        'dhcp_acl' => 'TEXT',
-        'queue_id' => 'INTEGER',
-        'mac' => 'VARCHAR(20)',
-        'dhcp_action' => 'VARCHAR(10)',
-        'dhcp_option_set' => 'VARCHAR(50)',
-        'dhcp_time' => 'TIMESTAMP',
-        'dhcp_hostname' => 'VARCHAR(60)',
-        'last_found' => 'TIMESTAMP',
-        'arp_found' => 'TIMESTAMP',
-        'mac_found' => 'TIMESTAMP',
-        'blocked' => 'SMALLINT',
-        'day_quota' => 'INTEGER',
-        'month_quota' => 'INTEGER',
-        'device_model_id' => 'INTEGER',
-        'firmware' => 'VARCHAR(100)',
-        'ts' => 'TIMESTAMP',
-        'client_id' => 'VARCHAR(250)',
-        'nagios' => 'SMALLINT',
-        'nagios_status' => 'VARCHAR(10)',
-        'nagios_handler' => 'VARCHAR(50)',
-        'link_check' => 'SMALLINT',
-        'changed' => 'SMALLINT',
-        'dhcp_changed' => 'SMALLINT',
-        'changed_time' => 'TIMESTAMP',
-        'created_by' => 'VARCHAR(10)',
-    },
-    'user_auth_alias' => {
-        'id' => 'SERIAL',
-        'auth_id' => 'INTEGER',
-        'alias' => 'VARCHAR(100)',
-        'description' => 'VARCHAR(100)',
-        'ts' => 'TIMESTAMP',
-    },
-    'user_list' => {
-        'id' => 'BIGSERIAL',
-        'ts' => 'TIMESTAMP',
-        'login' => 'VARCHAR(255)',
-        'description' => 'VARCHAR(255)',
-        'enabled' => 'SMALLINT',
-        'blocked' => 'SMALLINT',
-        'deleted' => 'SMALLINT',
-        'ou_id' => 'INTEGER',
-        'device_id' => 'INTEGER',
-        'filter_group_id' => 'INTEGER',
-        'queue_id' => 'INTEGER',
-        'day_quota' => 'INTEGER',
-        'month_quota' => 'INTEGER',
-        'permanent' => 'SMALLINT',
-    },
-    'user_sessions' => {
-        'id' => 'SERIAL',
-        'session_id' => 'VARCHAR(128)',
-        'user_id' => 'INTEGER',
-        'ip_address' => 'VARCHAR(45)',
-        'user_agent' => 'TEXT',
-        'created_at' => 'INTEGER',
-        'last_activity' => 'INTEGER',
-        'is_active' => 'SMALLINT',
-    },
-    'user_stats' => {
-        'id' => 'BIGSERIAL',
-        'router_id' => 'BIGINT',
-        'auth_id' => 'BIGINT',
-        'ts' => 'TIMESTAMP',
-        'byte_in' => 'BIGINT',
-        'byte_out' => 'BIGINT',
-        'pkt_in' => 'INTEGER',
-        'pkt_out' => 'INTEGER',
-        'step' => 'SMALLINT',
-    },
-    'user_stats_full' => {
-        'id' => 'BIGSERIAL',
-        'router_id' => 'BIGINT',
-        'auth_id' => 'BIGINT',
-        'ts' => 'TIMESTAMP',
-        'byte_in' => 'BIGINT',
-        'byte_out' => 'BIGINT',
-        'pkt_in' => 'INTEGER',
-        'pkt_out' => 'INTEGER',
-        'step' => 'SMALLINT',
-    },
-    'variables' => {
-        'id' => 'SERIAL',
-        'name' => 'VARCHAR(30)',
-        'value' => 'VARCHAR(255)',
-        'clear_time' => 'TIMESTAMP',
-        'created' => 'TIMESTAMP',
-    },
-    'vendors' => {
-        'id' => 'SERIAL',
-        'name' => 'VARCHAR(40)',
-    },
-    'version' => {
-        'id' => 'INTEGER',
-        'version' => 'VARCHAR(10)',
-    },
-    'wan_stats' => {
-        'id' => 'BIGSERIAL',
-        'ts' => 'TIMESTAMP',
-        'router_id' => 'INTEGER',
-        'interface_id' => 'INTEGER',
-        'bytes_in' => 'BIGINT',
-        'bytes_out' => 'BIGINT',
-        'forward_in' => 'BIGINT',
-        'forward_out' => 'BIGINT',
-    },
-    'worklog' => {
-        'id' => 'BIGSERIAL',
-        'ts' => 'TIMESTAMP',
-        'auth_id' => 'BIGINT',
-        'customer' => 'VARCHAR(50)',
-        'ip' => 'INET',
-        'message' => 'TEXT',
-        'level' => 'SMALLINT',
-    },
-);
-
-sub get_table_columns {
-    my ($db, $table) = @_;
-    my $sth = $db->column_info(undef, undef, $table, '%');
-    my @cols;
-    while (my $row = $sth->fetchrow_hashref) {
-        push @cols, $row->{COLUMN_NAME};
-    }
-    return @cols;
-}
-
 sub batch_sql_cached {
     my ($db, $sql, $data) = @_;
+    # Запоминаем исходное состояние AutoCommit
+    my $original_autocommit = $db->{AutoCommit};
     eval {
-        my $sth = $db->prepare_cached($sql) or die "Unable to prepare SQL: " . $db->errstr;
+        # Выключаем AutoCommit для транзакции
+        $db->{AutoCommit} = 0;
+        my $sth = $db->prepare_cached($sql)  or die "Unable to prepare SQL: " . $db->errstr;
         for my $params (@$data) {
             next unless @$params;
             $sth->execute(@$params) or die "Unable to execute with params [" . join(',', @$params) . "]: " . $sth->errstr;
         }
-        $db->commit() if (!$db->{AutoCommit});
+        $db->commit();
         1;
     } or do {
         my $err = $@ || 'Unknown error';
         eval { $db->rollback() };
-        print "batch_db_sql_cached failed: $err";
-	return 0;
+        warn "batch_sql_cached failed: $err";
+        # Восстанавливаем AutoCommit даже при ошибке
+        $db->{AutoCommit} = $original_autocommit;
+        return 0;
     };
+    # Восстанавливаем исходный режим AutoCommit
+    $db->{AutoCommit} = $original_autocommit;
     return 1;
 }
 
@@ -571,60 +133,62 @@ if ($opt_clear) {
 
 print "\n=== Check DB schema ===\n\n";
 
-my $mysql_schema_status = 1;
-my %mysql_tables;
+# === Сбор полной схемы из обеих БД ===
+print "Fetching schema from MySQL and PostgreSQL...\n";
 
-# --- Этап 1: Проверяем, что все таблицы и колонки MySQL есть в PG-схеме ---
-for my $idx (0 .. $#tables_to_migrate) {
-    my $table = $tables_to_migrate[$idx];
-    my $table_num = $idx + 1;
+# === Сбор схем ===
+my %schema;
+for my $table (@tables_to_migrate) {
+    next if $table =~ /(traffic_detail|sessions)/i;
+    $schema{mysql}{$table} = { get_table_columns($mysql_db, $table) };
+}
 
-    if ($table =~ /(traffic_detail|sessions)/) { next; }
+my @pg_tables = map { $_->{tablename} } get_records_sql($pg_db, "SELECT tablename FROM pg_tables WHERE schemaname = 'public'");
+for my $table (@pg_tables) {
+    next if $table =~ /(traffic_detail|sessions)/i;
+    $schema{pg}{$table} = { get_table_columns($pg_db, $table) };
+}
 
-    print "[$table_num/$total_tables] Processing table: $table\n";
+# === Флаг ошибки ===
+my $has_critical_error = 0;
 
-    if (!exists $pg_schema{$table}) {
-        print "    ❗ WARNING: Table $table not found in Postgres DB schema! Will be skip for migration.\n";
-        # Не считаем критичной ошибкой
+# === 1. Проверка: всё ли из PostgreSQL есть в MySQL? ===
+for my $table (keys %{ $schema{pg} }) {
+    if (!exists $schema{mysql}{$table}) {
+        print "❗ ERROR: Table '$table' exists in PostgreSQL but not in MySQL!\n";
+        $has_critical_error = 1;
         next;
     }
 
-    my @columns = get_table_columns($mysql_db, $table);
-    foreach my $column_name (@columns) {
-        my $col_lower = lc($column_name);  # Приводим к нижнему регистру
-        if (!exists $pg_schema{$table}->{$col_lower}) {
-            print "    ❗ WARNING: Column $column_name in table $table not in PG schema.  Will be skip for migration. \n";
-            # Не считаем критичной ошибкой
-        } else {
-            $mysql_tables{$table}->{$col_lower} = 1;
+    for my $col (keys %{ $schema{pg}{$table} }) {
+        if (!exists $schema{mysql}{$table}{$col}) {
+            print "❗ ERROR: Column '$col' in table '$table' exists in PostgreSQL but not in MySQL!\n";
+            $has_critical_error = 1;
         }
     }
 }
 
-# --- Этап 2: Проверяем, что все таблицы и колонки PG-схемы есть в MySQL ---
-for my $table (keys %pg_schema) {
-
-    if ($table =~ /(traffic_detail|sessions)/) { next; }
-
-    if (!exists $mysql_tables{$table}) {
-        print "    ❗ ERROR: Table $table from PG schema not found in source MySQL database!\n";
-        $mysql_schema_status = 0;
+# === 2. Проверка: есть ли лишнее в MySQL? ===
+for my $table (keys %{ $schema{mysql} }) {
+    if (!exists $schema{pg}{$table}) {
+        print "⚠️  WARNING: Table '$table' exists in MySQL but not in PostgreSQL — will be skipped.\n";
         next;
     }
 
-    for my $column_name (keys %{ $pg_schema{$table} }) {
-        if (!exists $mysql_tables{$table}->{$column_name}) {
-            print "    ❗ ERROR: Column $column_name in table $table missing in MySQL!\n";
-            $mysql_schema_status = 0;
+    for my $col (keys %{ $schema{mysql}{$table} }) {
+        if (!exists $schema{pg}{$table}{$col}) {
+            print "⚠️  WARNING: Column '$col' in table '$table' exists in MySQL but not in PostgreSQL — will be ignored.\n";
         }
     }
 }
 
-if (!$mysql_schema_status) {
-    print "\nSchema validation failed. Check database and try again.\n";
+if ($has_critical_error) {
+    print "\nSchema validation failed: missing required tables/columns in source MySQL database.\n";
     exit 103;
 }
 
+print "✅ Schema validation passed.\n\n";
+
 print "\n=== Starting migration of $total_tables tables ===\n\n";
 
 # === Миграция по таблицам с прогрессом ===
@@ -632,7 +196,7 @@ for my $idx (0 .. $#tables_to_migrate) {
     my $table = $tables_to_migrate[$idx];
     my $table_num = $idx + 1;
 
-    if (!exists $pg_schema{$table}) { next; }
+    if (!exists $schema{pg}->{$table}) { next; }
 
     print "[$table_num/$total_tables] Processing table: $table\n";
 
@@ -644,68 +208,113 @@ for my $idx (0 .. $#tables_to_migrate) {
         next;
     }
 
-    # === Построчное чтение ===
-    my $select_sth = $mysql_db->prepare("SELECT * FROM `$table`");
-    $select_sth->execute();
 
     my $inserted = 0;
     my $errors   = 0;
 
+    # === Построчное чтение ===
+    my $select_sth = $mysql_db->prepare("SELECT * FROM `$table`");
+    $select_sth->execute();
+
 # === Режим вставки: построчный или пакетный ===
 if ($opt_batch) {
     print "  → Using BATCH mode ($chunk_count records per chunk)\n";
 
-    # Получаем список колонок один раз
-    my @columns = get_table_columns($mysql_db, $table);
-    my $quoted_columns = '"' . join('", "', @columns) . '"';
-    my $placeholders = join(', ', ('?') x @columns);
-    my $insert_sql = "INSERT INTO \"$table\" ($quoted_columns) VALUES ($placeholders)";
+    # Берём колонки напрямую из PostgreSQL-схемы — они все есть в MySQL
+    my @valid_columns = sort keys %{ $schema{pg}{$table} };
+
+    my $quoted_columns = '"' . join('", "', @valid_columns) . '"';
+    my $placeholders   = join(', ', ('?') x @valid_columns);
+    my $insert_sql     = "INSERT INTO \"$table\" ($quoted_columns) VALUES ($placeholders)";
 
     my @batch_buffer;
     my $chunk_size = $chunk_count;
 
+    my $processed = 0;
+    my $report_every = 10_000;
+
+
     while (my $row = $select_sth->fetchrow_hashref) {
-	my @values;
-        for my $key (@columns) {
-            if (!exists $pg_schema{$table}->{lc($key)}) { next; }
-	    my $value = $row->{$key};
-    	    if (lc($key) eq 'ip') {
-        	$value = undef if !defined($value) || $value eq '';
-    		}
-            push @values, $value;
-	    }
+        my @values;
+        for my $col (@valid_columns) {
+            my $raw_value = $row->{$col};
+            my $norm_value = normalize_value($raw_value, $schema{pg}{$table}{$col});
+            push @values, $norm_value;
+        }
         push @batch_buffer, \@values;
+        $processed++;
+
         if (@batch_buffer >= $chunk_count) {
-	    my $insert_status = batch_sql_cached($pg_db, $insert_sql, \@batch_buffer);
-	    if ($insert_status) { $inserted += @batch_buffer; } else { $errors+=@batch_buffer; }
+            my $insert_status = batch_sql_cached($pg_db, $insert_sql, \@batch_buffer);
+            if ($insert_status) {
+                $inserted += @batch_buffer;
+            } else {
+                $errors += @batch_buffer;
+            }
             @batch_buffer = ();
-	    }
+
+            # Прогресс
+            if ($processed % $report_every == 0) {
+                my $pct = int($processed * 100 / $rec_count);
+                printf "  → Processed: %d / %d (%d%%)\r", $processed, $rec_count, $pct;
+                $| = 1;  # flush STDOUT
+            }
+        }
     }
+
     # Остаток
     if (@batch_buffer) {
-	my $insert_status = batch_sql_cached($pg_db, $insert_sql, \@batch_buffer);
-        if ($insert_status) { $inserted += @batch_buffer; } else { $errors+=@batch_buffer; }
-	}
+        my $insert_status = batch_sql_cached($pg_db, $insert_sql, \@batch_buffer);
+        if ($insert_status) {
+            $inserted += @batch_buffer;
+        } else {
+            $errors += @batch_buffer;
+        }
+        $processed += @batch_buffer;
+    }
+
+    # Финальная строка
+    printf "  → Processed: %d / %d (100%%)\n", $processed, $rec_count;
+
     } else {
-    # === построчный режим ===
+
+    # === Построчный режим ===
+    my $processed = 0;
+    my $report_every = 10_000;
+
     while (my $row = $select_sth->fetchrow_hashref) {
-        # === Приведение имён полей к нижнему регистру ===
         my %row_normalized;
-        while (my ($key, $value) = each %$row) {
-	    my $n_key = lc($key);
-	    if ($n_key eq 'ip') {
-		if (!defined $value || $value eq '') { $value = undef; }
-		}
-            $row_normalized{$n_key} = $value;
+        for my $col (keys %{ $schema{pg}{$table} }) {
+            my $raw_value = $row->{$col};
+            my $norm_value = normalize_value($raw_value, $schema{pg}{$table}{$col});
+            $row_normalized{$col} = $norm_value;
         }
 
         my $ret_id = insert_record($pg_db, $table, \%row_normalized);
-	if ($ret_id>0) { $inserted++; } else {
+        if ($ret_id > 0) {
+            $inserted++;
+        } else {
             $errors++;
-	    print Dumper(\%row_normalized) if ($debug);
-    	    }
+            print Dumper(\%row_normalized) if ($debug);
+        }
+
+        $processed++;
+
+        # Прогресс каждые N строк
+        if ($rec_count > 0 && $processed % $report_every == 0) {
+            my $pct = int($processed * 100 / $rec_count);
+            printf "  → Processed: %d / %d (%d%%)\r", $processed, $rec_count, $pct;
+            $| = 1;  # flush
+        }
     }
     $select_sth->finish();
+
+    # Финальная строка
+    if ($rec_count > 0) {
+        printf "  → Processed: %d / %d (100%%)\n", $processed, $rec_count;
+        } else {
+        print "  → Processed: $processed records\n";
+        }
     }
 
     # === Итог по таблице ===
@@ -720,5 +329,32 @@ if ($opt_batch) {
     print "\n";
 }
 
+print "\n=== Resetting all table sequences ===\n";
+
+# Получаем список всех таблиц из целевой схемы (PostgreSQL)
+my $tables_sql = "SELECT tablename FROM pg_tables WHERE schemaname = 'public'";
+my $sth = $pg_db->prepare($tables_sql);
+$sth->execute();
+
+while (my ($table) = $sth->fetchrow_array) {
+    # Формируем имя последовательности
+    my $seq_name = "${table}_id_seq";
+    # Проверяем, существует ли такая последовательность
+    my ($exists) = $pg_db->selectrow_array(
+        "SELECT 1 FROM pg_class WHERE relname = ? AND relkind = 'S'",
+        undef, $seq_name
+    );
+    if ($exists) {
+        # Получаем MAX(id)
+        my ($max_id) = $pg_db->selectrow_array("SELECT MAX(id) FROM \"$table\"");
+        $max_id //= 1;
+        # Сбрасываем последовательность
+        $pg_db->do("SELECT setval('$seq_name', $max_id)");
+        print "  → $table: sequence reset to $max_id\n";
+    }
+}
+
+print "✅ All sequences updated.\n";
+
 print "🎉 Migration completed! Processed $total_tables tables.\n";
 exit 0;

docs/patches/patch_seq.pl

@@ -0,0 +1,73 @@
+#!/usr/bin/perl
+
+#
+# Copyright (C) Roman Dmitriev, rnd@rajven.ru
+#
+
+use utf8;
+use open ":encoding(utf8)";
+use open ':std', ':encoding(UTF-8)';
+use Encode;
+no warnings 'utf8';
+use English;
+use FindBin '$Bin';
+use lib "/opt/Eye/scripts";
+use Getopt::Long qw(GetOptions);
+use Data::Dumper;
+use eyelib::config;
+use eyelib::main;
+use eyelib::database;
+use eyelib::common;
+use eyelib::net_utils;
+use strict;
+use warnings;
+
+# debug disable force
+$debug = 0;
+
+# === Явное указание портов ===
+my $PG_PORT    = 5432;
+
+# === Подключение к PostgreSQL (цель) ===
+my $pg_dsn = "dbi:Pg:dbname=$DBNAME;host=$DBHOST;port=$PG_PORT;";
+my $pg_db = DBI->connect($pg_dsn, $DBUSER, $DBPASS, {
+    RaiseError => 0,
+    AutoCommit => 1,
+    pg_enable_utf8 => 1,
+    pg_server_prepare => 0
+});
+if (!defined $pg_db) {
+    print "Cannot connect to PostgreSQL server: $DBI::errstr\n";
+    print "For install/configure PostgreSQL server please run migrate2psql.sh!\n";
+    exit 100;
+}
+
+
+print "\n=== Resetting all table sequences ===\n";
+
+# Получаем список всех таблиц из целевой схемы (PostgreSQL)
+my $tables_sql = "SELECT tablename FROM pg_tables WHERE schemaname = 'public'";
+my $sth = $pg_db->prepare($tables_sql);
+$sth->execute();
+
+while (my ($table) = $sth->fetchrow_array) {
+    # Формируем имя последовательности
+    my $seq_name = "${table}_id_seq";
+    # Проверяем, существует ли такая последовательность
+    my ($exists) = $pg_db->selectrow_array(
+        "SELECT 1 FROM pg_class WHERE relname = ? AND relkind = 'S'",
+        undef, $seq_name
+    );
+    if ($exists) {
+        # Получаем MAX(id)
+        my ($max_id) = $pg_db->selectrow_array("SELECT MAX(id) FROM \"$table\"");
+        $max_id //= 1;
+        # Сбрасываем последовательность
+        $pg_db->do("SELECT setval('$seq_name', $max_id)");
+        print "  → $table: sequence reset to $max_id\n";
+    }
+}
+
+print "✅ All sequences updated.\n";
+
+exit 0;

scripts/eyelib/database.pm

@@ -38,6 +38,8 @@ db_log_error
 db_log_info
 db_log_verbose
 db_log_warning
+normalize_value
+get_table_columns
 init_db
 do_sql
 _execute_param
@@ -58,6 +60,7 @@ Set_Variable
 Get_Variable
 Del_Variable
 clean_variables
+build_db_schema
 
 $add_rules
 $L_WARNING
@@ -65,6 +68,8 @@ $L_INFO
 $L_DEBUG
 $L_ERROR
 $L_VERBOSE
+
+%db_schema
 );
 
 BEGIN
@@ -109,6 +114,143 @@ our %dns_fields = (
     'alias'=>'1',
 );
 
+our %db_schema;
+
+#---------------------------------------------------------------------------------------------------------------
+
+sub build_db_schema {
+    my ($dbh) = @_;
+
+    # Определяем тип СУБД
+    my $db_type = lc($dbh->{Driver}->{Name});
+    die "Unsupported database driver: $db_type" 
+        unless $db_type eq 'mysql' || $db_type eq 'pg';
+
+    # Получаем имя базы данных
+    my $db_name;
+    if ($db_type eq 'mysql') {
+        ($db_name) = $dbh->selectrow_array("SELECT DATABASE()");
+    } elsif ($db_type eq 'pg') {
+        ($db_name) = $dbh->selectrow_array("SELECT current_database()");
+    }
+
+    my $db_info;
+    $db_info->{db_type}=$db_type;
+    $db_info->{db_name}=$db_name;
+    return $db_info if (exists $db_schema{$db_type}{$db_name});
+    # Получаем список таблиц
+    my @tables;
+    if ($db_type eq 'mysql') {
+        my $sth = $dbh->prepare("SHOW TABLES");
+        $sth->execute();
+        @tables = map { $_->[0] } @{$sth->fetchall_arrayref()};
+    } elsif ($db_type eq 'pg') {
+        my $sql = q{
+            SELECT tablename 
+            FROM pg_tables 
+            WHERE schemaname = 'public'
+        };
+        my $sth = $dbh->prepare($sql);
+        $sth->execute();
+        @tables = map { $_->[0] } @{$sth->fetchall_arrayref()};
+    }
+
+    # Собираем схему
+    for my $table (@tables) {
+        my $sth = $dbh->column_info(undef, undef, $table, '%');
+        while (my $col = $sth->fetchrow_hashref) {
+            my $col_name = lc($col->{COLUMN_NAME});
+            $db_schema{$db_type}{$db_name}{$table}{$col_name} = {
+                type     => $col->{TYPE_NAME} // '',
+                nullable => $col->{NULLABLE}  // 1,
+                default  => $col->{COLUMN_DEF} // undef,
+            };
+        }
+    }
+    return $db_info;
+}
+
+#---------------------------------------------------------------------------------------------------------------
+
+sub normalize_value {
+    my ($value, $col_info) = @_;
+
+    # Если значение пустое — обрабатываем по правилам колонки
+    if (!defined $value || $value eq '' || $value =~ /^(?:NULL|\\N)$/i) {
+        return $col_info->{nullable} ? undef : _default_for_type($col_info);
+    }
+
+    my $type = lc($col_info->{type});
+
+    # --- Числовые типы: приводим к числу, если выглядит как число ---
+    if ($type =~ /^(?:tinyint|smallint|mediumint|int|integer|bigint|serial|bigserial)$/i) {
+        # Просто конвертируем строку в число (Perl сам обрежет мусор)
+        # Например: "123abc" → 123, "abc" → 0
+        return 0 + $value;
+    }
+
+    # --- Булевы: приводим к 0/1 ---
+    if ($type =~ /^(?:bool|boolean|bit)$/i) {
+        return $value ? 1 : 0;
+    }
+
+    # --- Временные типы: оставляем как есть, но фильтруем "нулевые" даты MySQL ---
+    if ($type =~ /^(?:timestamp|datetime|date|time)$/i) {
+        # Это частая проблема при миграции — '0000-00-00' ломает PostgreSQL
+        return undef if $value =~ /^0000-00-00/;
+        return $value;
+    }
+
+    # --- Все остальные типы (строки, inet, json и т.д.) — передаём как есть ---
+    return $value;
+}
+
+# Вспомогательная: безопасное значение по умолчанию
+sub _default_for_type {
+    my ($col) = @_;
+
+    # Используем DEFAULT, только если он простой литерал (не выражение)
+    if (defined $col->{default}) {
+        my $def = $col->{default};
+        # Пропускаем выражения: nextval(), CURRENT_TIMESTAMP, NOW(), uuid() и т.п.
+        if ($def !~ /(nextval|current_timestamp|now|uuid|auto_increment|::)/i) {
+            # Убираем одинарные кавычки, если строка: 'value' → value
+            if ($def =~ /^'(.*)'$/) {
+                return $1;
+            }
+            # Если похоже на число — вернём как число
+            if ($def =~ /^[+-]?\d+$/) {
+                return 0 + $def;
+            }
+            return $def;
+        }
+    }
+
+    # Фолбэк по типу
+    my $type = lc($col->{type});
+    if ($type =~ /^(?:tinyint|smallint|int|integer|bigint)/i) { return 0; }
+    if ($type =~ /^(?:char|varchar|text)/i) { return ''; }
+    if ($type =~ /^(?:timestamp|datetime)/i) { return GetNowTime(); }
+    return undef;
+}
+
+#---------------------------------------------------------------------------------------------------------------
+
+sub get_table_columns {
+    my ($db, $table) = @_;
+    my %columns;
+    my $sth = $db->column_info(undef, undef, $table, '%');
+    while (my $row = $sth->fetchrow_hashref) {
+        my $name = lc($row->{COLUMN_NAME});  # ← приводим к нижнему регистру сразу!
+        $columns{$name} = {
+            type     => $row->{TYPE_NAME} // '',
+            nullable => $row->{NULLABLE}  // 1,
+            default  => $row->{COLUMN_DEF} // undef,
+        };
+    }
+    return %columns;  # возвращает список: key1, val1, key2, val2...
+}
+
 #---------------------------------------------------------------------------------------------------------------
 
 sub StrToIp {
@@ -732,6 +874,8 @@ unless (reconnect_db(\$db)) {
     return;
     }
 
+my $db_info= build_db_schema($db);
+
 my $dns_changed = 0;
 my $rec_id = 0;
 
@@ -749,7 +893,7 @@ my $values = '';
 my $new_str = '';
 
 foreach my $field (keys %$record) {
-    my $val = defined $record->{$field} ? $record->{$field} : undef;
+    my $val =  normalize_value($record->{$field}, $db_schema{$db_info->{db_type}}{$db_info->{db_name}}{$table}{$field});
     # Экранируем имя поля в зависимости от СУБД
     my $quoted_field = get_db_type($db) eq 'mysql'
         ? '`' . $field . '`'
@@ -819,6 +963,8 @@ unless (reconnect_db(\$db)) {
     return;
     }
 
+my $db_info = build_db_schema($db);
+
 my $select_sql = "SELECT * FROM $table WHERE $filter_sql";
 my $old_record = get_record_sql($db, $select_sql, @filter_params);
 return unless $old_record;
@@ -844,7 +990,7 @@ if ($table eq "user_auth") {
 my $diff = '';
 for my $field (keys %$record) {
         my $old_val = defined $old_record->{$field} ? $old_record->{$field} : '';
-        my $new_val = defined $record->{$field} ? $record->{$field} : '';
+        my $new_val =  normalize_value($record->{$field}, $db_schema{$db_info->{db_type}}{$db_info->{db_name}}{$table}{$field});
         if ($new_val ne $old_val) {
             $diff .= " $field => $new_val (old: $old_val),";
             $set_clause .= " $field = ?, ";