bbs/clrghouz
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Enable nodelist import test mode

Browse Source
This commit is contained in:
Deon George 2023-09-07 12:24:01 +10:00
parent 7b39dafd12
commit 6d199345ac
2 changed files with 57 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/Console/Commands/NodelistImport.php
View File

@ -18,7 +18,8 @@ class NodelistImport extends Command
.' {file : File ID | filename}' .' {file : File ID | filename}'
.' {domain? : Domain Name}' .' {domain? : Domain Name}'
.' {--D|delete : Delete old data for the date}' .' {--D|delete : Delete old data for the date}'
.' {--U|unlink : Delete file after import}'; .' {--U|unlink : Delete file after import}'
.' {--T|test : Dry run}';
/** /**
* The console command description. * The console command description.
@ -38,7 +39,8 @@ class NodelistImport extends Command
is_numeric($x=$this->argument('file')) ? File::findOrFail($x) : $x, is_numeric($x=$this->argument('file')) ? File::findOrFail($x) : $x,
$this->argument('domain'), $this->argument('domain'),
$this->option('delete'), $this->option('delete'),
$this->option('unlink') $this->option('unlink'),
$this->option('test')
); );
} }
} }

84
app/Jobs/NodelistImport.php
View File

@ -30,6 +30,7 @@ class NodelistImport implements ShouldQueue
private ?string $domain; private ?string $domain;
private bool $delete_file; private bool $delete_file;
private bool $delete_recs; private bool $delete_recs;
private bool $testmode;
/** /**
* Import Nodelist constructor. * Import Nodelist constructor.
@ -38,13 +39,15 @@ class NodelistImport implements ShouldQueue
* @param string|null $domain * @param string|null $domain
* @param bool $delete_recs * @param bool $delete_recs
* @param bool $delete_file * @param bool $delete_file
* @param bool $test
*/ */
public function __construct(File|string $file,string $domain=NULL,bool $delete_recs=FALSE,bool $delete_file=TRUE) public function __construct(File|string $file,string $domain=NULL,bool $delete_recs=FALSE,bool $delete_file=TRUE,bool $test=FALSE)
{ {
$this->file = $file; $this->file = $file;
$this->domain = $domain; $this->domain = $domain;
$this->delete_file = $delete_file; $this->delete_file = $delete_file;
$this->delete_recs = $delete_recs; $this->delete_recs = $delete_recs;
$this->testmode = $test;
$this->onQueue(self::QUEUE); $this->onQueue(self::QUEUE);
} }
@ -72,9 +75,9 @@ class NodelistImport implements ShouldQueue
// Get the file from the host // Get the file from the host
$file = $this->getFileFromHost(self::importkey,$this->file); $file = $this->getFileFromHost(self::importkey,$this->file);
Log::debug(sprintf('%s:Loading file [%s] (%s).',static::LOGKEY,$file,getcwd())); Log::debug(sprintf('%s:+ Loading file [%s] (%s).',static::LOGKEY,$file,getcwd()));
$lines = $this->getFileLines($file); $lines = $this->getFileLines($file);
Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines)); Log::debug(sprintf('%s:- Processing [%d] lines.',static::LOGKEY,$lines));
$fh = NULL; $fh = NULL;
$z = $this->openFile($file,$fh); $z = $this->openFile($file,$fh);
@ -84,7 +87,8 @@ class NodelistImport implements ShouldQueue
$matches = []; $matches = [];
if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) { if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) {
Log::error(sprintf('%s:Nodelist file [%d] is not valid?',self::LOGKEY,$this->file->id),['m'=>$matches,'l'=>$line]); Log::error(sprintf('%s:! Nodelist file [%d] is not valid?',self::LOGKEY,$this->file->id),['m'=>$matches,'l'=>$line]);
throw new \Exception('Invalid nodelist for file: '.$this->file->id); throw new \Exception('Invalid nodelist for file: '.$this->file->id);
} }
@ -93,6 +97,7 @@ class NodelistImport implements ShouldQueue
if (! $do) { if (! $do) {
Log::error(sprintf('%s:! Domain not found [%s].',static::LOGKEY,strtolower($matches[1] ?: $this->domain))); Log::error(sprintf('%s:! Domain not found [%s].',static::LOGKEY,strtolower($matches[1] ?: $this->domain)));
throw new \Exception('Nodelist Domain not found: '.$this->file->id); throw new \Exception('Nodelist Domain not found: '.$this->file->id);
} }
@ -100,10 +105,11 @@ class NodelistImport implements ShouldQueue
if ($date->dayOfYear != $matches[3]) { if ($date->dayOfYear != $matches[3]) {
Log::error(sprintf('%s:! Nodelist date doesnt match [%d] (%d:%s).',static::LOGKEY,$matches[3],$date->dayOfYear,$date->format('Y-m-d'))); Log::error(sprintf('%s:! Nodelist date doesnt match [%d] (%d:%s).',static::LOGKEY,$matches[3],$date->dayOfYear,$date->format('Y-m-d')));
throw new \Exception('Nodelist date doesnt match for file: '.$this->file->id); throw new \Exception('Nodelist date doesnt match for file: '.$this->file->id);
} }
Log::debug(sprintf('%s:Importing nodelist for [%s] dated [%s].',static::LOGKEY,$do->name,$date->format('Y-m-d'))); Log::alert(sprintf('%s:- Importing nodelist for [%s] dated [%s].',static::LOGKEY,$do->name,$date->format('Y-m-d')));
DB::beginTransaction(); DB::beginTransaction();
$no = Nodelist::firstOrCreate(['date'=>$date,'domain_id'=>$do->id]); $no = Nodelist::firstOrCreate(['date'=>$date,'domain_id'=>$do->id]);
@ -134,11 +140,13 @@ class NodelistImport implements ShouldQueue
$line = preg_replace('/^\x{feff}/u','',$line); $line = preg_replace('/^\x{feff}/u','',$line);
$c++; $c++;
Log::debug(sprintf('%s:%s',self::LOGKEY,$line));
$fields = str_getcsv(trim($line)); $fields = str_getcsv(trim($line));
// First field is either zone,region,host,hub,down,pvt (or blank) // First field is either zone,region,host,hub,down,pvt (or blank)
if ($fields[0] AND ! in_array($fields[0],Nodelist::definitions)) { if ($fields[0] AND ! in_array($fields[0],Nodelist::definitions)) {
Log::error(sprintf('%s:Invalid field zero [%s] - IGNORING record (%s)',self::LOGKEY,$fields[0],$line)); Log::error(sprintf('%s:! Invalid field zero [%s] - IGNORING record (%s)',self::LOGKEY,$fields[0],$line));
continue; continue;
} }
@ -207,12 +215,12 @@ class NodelistImport implements ShouldQueue
break; break;
default: default:
Log::error(sprintf('%s:Unhandled first field [%s]',self::LOGKEY,$fields[0])); Log::error(sprintf('%s:! Unhandled first field [%s]',self::LOGKEY,$fields[0]));
continue 2; continue 2;
} }
if (! $zone) { if (! $zone) {
Log::error(sprintf('%s:Zone NOT set, ignoring record...',self::LOGKEY)); Log::error(sprintf('%s:! Zone NOT set, ignoring record...',self::LOGKEY));
continue; continue;
} }
@ -227,14 +235,14 @@ class NodelistImport implements ShouldQueue
Address::reguard(); Address::reguard();
if ($ao->region_id && ($ao->region_id !== $region)) if ($ao->region_id && ($ao->region_id !== $region))
Log::alert(sprintf('%s:Address [%s] changing regions [%d->%d]',self::LOGKEY,$ao->ftn,$ao->region_id,$region)); Log::alert(sprintf('%s:%% Address [%s] changing regions [%d->%d]',self::LOGKEY,$ao->ftn,$ao->region_id,$region));
$ao->region_id = $region; $ao->region_id = $region;
$ao->role = $role; $ao->role = $role;
$ao->hub_id = $hub_id; $ao->hub_id = $hub_id;
if ($ao->exists) if ($ao->exists)
Log::debug(sprintf('%s:Processing existing address [%s]',self::LOGKEY,$ao->ftn)); Log::debug(sprintf('%s:- Processing existing address [%s]',self::LOGKEY,$ao->ftn));
$sysop = trim(str_replace('_',' ',$fields[4])); $sysop = trim(str_replace('_',' ',$fields[4]));
$system = trim(str_replace('_',' ',$fields[2])); $system = trim(str_replace('_',' ',$fields[2]));
@ -301,7 +309,7 @@ class NodelistImport implements ShouldQueue
break; break;
default: default:
Log::debug(sprintf('%s: - Not configured to handle flag [%s]',self::LOGKEY,$x->first())); Log::debug(sprintf('%s:! Not configured to handle flag [%s]',self::LOGKEY,$x->first()));
continue 2; continue 2;
} }
} }
@ -327,7 +335,7 @@ class NodelistImport implements ShouldQueue
->single(); ->single();
if (! $so) { if (! $so) {
Log::info(sprintf('%s:New System for ZC/RC/NC [%s] - System [%s] Sysop [%s]',self::LOGKEY,$ao->ftn,$system,$sysop)); Log::info(sprintf('%s:= New System for ZC/RC/NC [%s] - System [%s] Sysop [%s]',self::LOGKEY,$ao->ftn,$system,$sysop));
$so = new System; $so = new System;
$so->sysop = $sysop; $so->sysop = $sysop;
$so->name = $system; $so->name = $system;
@ -341,13 +349,13 @@ class NodelistImport implements ShouldQueue
// If the address exists, but it was discovered, assign it to this new host // If the address exists, but it was discovered, assign it to this new host
if ($ao->system_id && ($ao->system_id !== $so->id) && ($ao->system->name === 'Discovered System')) { if ($ao->system_id && ($ao->system_id !== $so->id) && ($ao->system->name === 'Discovered System')) {
Log::info(sprintf('%s:Re-assigning discovered address to [%s]',self::LOGKEY,$so->id)); Log::info(sprintf('%s:= Re-assigning discovered address to [%s]',self::LOGKEY,$so->id));
} elseif (! $ao->system_id) { } elseif (! $ao->system_id) {
Log::alert(sprintf('%s:[%s] new system created [%s:%s]',self::LOGKEY,$ao->ftn,$system,$sysop)); Log::alert(sprintf('%s:%% [%s] address not assigned to any system [%s:%s]',self::LOGKEY,$ao->ftn,$system,$sysop));
} elseif ($ao->system_id !== $so->id) { } elseif ($ao->system_id !== $so->id) {
Log::alert(sprintf('%s:[%s] hosted by new system [%s:%s] (was %s:%s)',self::LOGKEY,$ao->ftn,$system,$sysop,$ao->system->name,$ao->system->sysop)); Log::alert(sprintf('%s:%% [%s] hosted by new system [%s:%s] (was %s:%s)',self::LOGKEY,$ao->ftn,$system,$sysop,$ao->system->name,$ao->system->sysop));
$ao->active = FALSE; $ao->active = FALSE;
$ao->save(); $ao->save();
@ -362,21 +370,21 @@ class NodelistImport implements ShouldQueue
} }
if ($ao->system_id && ((($ao->system->sysop === $sysop) || ($ao->system->name === $system)) && (($ao->system->address === $address) || ($methods->pluck('address')->search($address) !== FALSE)))) { if ($ao->system_id && ((($ao->system->sysop === $sysop) || ($ao->system->name === $system)) && (($ao->system->address === $address) || ($methods->pluck('address')->search($address) !== FALSE)))) {
Log::debug(sprintf('%s:Matched [%s] to existing system [%s] with address [%s]',self::LOGKEY,$ao->ftn,$ao->system->name,$ao->system->address)); Log::info(sprintf('%s:= Matched [%s] to existing system [%s] with address [%s]',self::LOGKEY,$ao->ftn,$ao->system->name,$ao->system->address));
$so = $ao->system; $so = $ao->system;
// Dont change the system details if a user exists here, or its us // Dont change the system details if a user exists here, or its us
if ((! $so->users->count()) && ($so->id != $us->system_id)) { if ((! $so->users->count()) && ($so->id != $us->system_id)) {
// If the sysop name is different // If the sysop name is different
if ($so->sysop !== $sysop) { if ($so->sysop !== $sysop) {
Log::debug(sprintf('%s:Sysop Name changed for BBS [%s:%s] from [%s] to [%s]', Log::alert(sprintf('%s:! Sysop Name changed for BBS [%s:%s] from [%s] to [%s]',
self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop)); self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop));
$so->sysop = $sysop; $so->sysop = $sysop;
// We have the same name has changed (except for ZC/RC addresses) // We have the same name has changed (except for ZC/RC addresses)
} elseif (($so->name !== $system) && (! ((Address::NODE_ZC|Address::NODE_RC|Address::NODE_NC) & $ao->role))) { } elseif (($so->name !== $system) && (! ((Address::NODE_ZC|Address::NODE_RC|Address::NODE_NC) & $ao->role))) {
Log::debug(sprintf('%s:System Name changed for BBS [%s:%s] to [%s]', Log::alert(sprintf('%s:! System Name changed for BBS [%s:%s] to [%s]',
self::LOGKEY,$so->id,$so->name,$system)); self::LOGKEY,$so->id,$so->name,$system));
$so->name = $system; $so->name = $system;
@ -385,28 +393,36 @@ class NodelistImport implements ShouldQueue
// We'll search and see if we already have that system // We'll search and see if we already have that system
} else { } else {
Log::debug(sprintf('%s:Looking for existing system [%s] with address [%s]',self::LOGKEY,$system,$address)); Log::debug(sprintf('%s:- Looking for existing system [%s] with address [%s]',self::LOGKEY,$system,$address));
// If we dont have $address/port // If we dont have $address/port
$so = NULL; $so = NULL;
if ($address) if ($address) {
$so = System::select('systems.*') $so = System::select('systems.*')
->distinct()
->join('mailer_system',['mailer_system.system_id'=>'systems.id']) ->join('mailer_system',['mailer_system.system_id'=>'systems.id'])
->where(function($query) use ($address) { ->where(function($query) use ($address) {
return $query->where('systems.address',$address) return $query->where('systems.address',$address)
->orWhere('mailer_system.address',$address); ->orWhere('mailer_system.address',$address);
}) })
->single(); ->single();
}
if (! $so) {
Log::debug(sprintf('%s:- Didnt match on system address, looking for System [%s] AND Sysop [%s]',self::LOGKEY,$system,$sysop));
if (! $so)
$so = System::where('name',$system) $so = System::where('name',$system)
->where('sysop',$sysop) ->where('sysop',$sysop)
->firstOrNew(); ->firstOrNew();
} else {
Log::debug(sprintf('%s:- Matched on system [%d] address',self::LOGKEY,$so->id));
}
if ($so->exists) if ($so->exists)
Log::debug(sprintf('%s:Linking address [%d:%d/%d] to [%s:%s]',self::LOGKEY,$zo->zone_id,$ao->host_id,$ao->node_id,$so->id,$so->name)); Log::info(sprintf('%s:= Linking address [%d:%d/%d] to [%s:%s]',self::LOGKEY,$zo->zone_id,$ao->host_id,$ao->node_id,$so->id,$so->name));
else else
Log::debug(sprintf('%s:New System [%s] with FTN [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id)); Log::info(sprintf('%s:= New System [%s] with FTN [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id));
$so->name = $system; $so->name = $system;
$so->sysop = $sysop; $so->sysop = $sysop;
@ -425,12 +441,17 @@ class NodelistImport implements ShouldQueue
$so->baud = $fields[6]; $so->baud = $fields[6];
*/ */
if ($so->users->count()) {
Log::alert(sprintf('%s:! Refusing to update a system managed by a user',self::LOGKEY));
continue;
}
// Save the system record // Save the system record
try { try {
$so->save(); $so->save();
} catch (\Exception $e) { } catch (\Exception $e) {
Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); Log::error(sprintf('%s:! Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
DB::rollBack(); DB::rollBack();
throw new \Exception($e->getMessage()); throw new \Exception($e->getMessage());
@ -448,6 +469,7 @@ class NodelistImport implements ShouldQueue
} }
$ao->zone_id = $zo->id; $ao->zone_id = $zo->id;
$ao->validated = TRUE;
if ($ao->getDirty()) if ($ao->getDirty())
$p++; $p++;
@ -461,28 +483,28 @@ class NodelistImport implements ShouldQueue
$no->addresses()->attach($ao,['role'=>$ao->role]); $no->addresses()->attach($ao,['role'=>$ao->role]);
} catch (\Exception $e) { } catch (\Exception $e) {
Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); Log::error(sprintf('%s:! Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
DB::rollBack(); DB::rollBack();
throw new \Exception($e->getMessage()); throw new \Exception($e->getMessage());
} }
if (! ($c % 100)) { if (! ($c % 100)) {
Log::notice(sprintf('%s:Processed [%s] records',self::LOGKEY,$c),['memory'=>memory_get_usage(TRUE)]); Log::notice(sprintf('%s:= Processed [%s] records',self::LOGKEY,$c),['memory'=>memory_get_usage(TRUE)]);
} }
} }
// Remove addresses not recorded; // Remove addresses not recorded;
$remove = $zo->addresses->except($us->system->addresses->pluck('id')->toArray())->diff($no->addresses); $remove = $zo->addresses->except($us->system->addresses->pluck('id')->toArray())->diff($no->addresses);
Log::notice(sprintf('%s:Deleting addresses [%s]',self::LOGKEY,$remove->pluck('ftn2d')->join(','))); Log::alert(sprintf('%s:%% Deleting [%d] addresses [%s]',self::LOGKEY,$remove->count(),$remove->pluck('ftn2d')->join(',')));
Address::whereIN('id',$remove->pluck('id')->toArray())->update(['active'=>FALSE]); Address::whereIN('id',$remove->pluck('id')->toArray())->update(['active'=>FALSE]);
Address::whereIN('id',$remove->pluck('id')->toArray())->delete(); Address::whereIN('id',$remove->pluck('id')->toArray())->delete();
if (($x=crc16(substr($tocrc,0,-3))) === $file_crc) { if ((! $this->testmode) && ($x=crc16(substr($tocrc,0,-3))) === $file_crc) {
Log::info(sprintf('%s:Committing nodelist',self::LOGKEY)); Log::info(sprintf('%s:= Committing nodelist',self::LOGKEY));
DB::commit(); DB::commit();
} else { } else {
Log::error(sprintf('%s:Rolling back nodelist, CRC doesnt match [%s](%s)',self::LOGKEY,$x,$file_crc)); Log::error(sprintf('%s:! Rolling back nodelist, CRC doesnt match [%s](%s) or test mode',self::LOGKEY,$x,$file_crc));
DB::rollBack(); DB::rollBack();
} }
@ -491,6 +513,6 @@ class NodelistImport implements ShouldQueue
if ($this->delete_file and $c) if ($this->delete_file and $c)
unlink($file); unlink($file);
Log::info(sprintf('%s:Updated %d records from %d systems',self::LOGKEY,$p,$c)); Log::info(sprintf('%s:= Updated %d records from %d systems',self::LOGKEY,$p,$c));
} }
} }