From 52055c52173bd043bdc7943de29455f014bb4a5e Mon Sep 17 00:00:00 2001 From: Deon George Date: Sun, 9 Jul 2023 22:03:15 +1000 Subject: [PATCH] Update nodelist parsing now that we have mailer_system --- app/Console/Commands/NodelistImport.php | 9 +- app/Jobs/NodelistImport.php | 180 +++++++++++++----- app/Traits/Import.php | 16 +- .../2023_07_09_125321_mailer_address.php | 28 +++ 4 files changed, 173 insertions(+), 60 deletions(-) create mode 100644 database/migrations/2023_07_09_125321_mailer_address.php diff --git a/app/Console/Commands/NodelistImport.php b/app/Console/Commands/NodelistImport.php index a331903..032d70a 100644 --- a/app/Console/Commands/NodelistImport.php +++ b/app/Console/Commands/NodelistImport.php @@ -15,7 +15,7 @@ class NodelistImport extends Command * @var string */ protected $signature = 'nodelist:import' - .' {file : File ID}' + .' {file : File ID | filename}' .' {domain? : Domain Name}' .' {--D|delete : Delete old data for the date}' .' {--U|unlink : Delete file after import}'; @@ -34,6 +34,11 @@ class NodelistImport extends Command */ public function handle() { - return Job::dispatchSync(File::findOrFail($this->argument('file')),$this->argument('domain'),$this->option('delete'),$this->option('unlink')); + return Job::dispatchSync( + is_numeric($x=$this->argument('file')) ? File::findOrFail($x) : $x, + $this->argument('domain'), + $this->option('delete'), + $this->option('unlink') + ); } } \ No newline at end of file diff --git a/app/Jobs/NodelistImport.php b/app/Jobs/NodelistImport.php index 458335d..4a066a5 100644 --- a/app/Jobs/NodelistImport.php +++ b/app/Jobs/NodelistImport.php @@ -13,7 +13,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; -use App\Models\{Address,Domain,File,Nodelist,Setup,System,Zone}; +use App\Models\{Address, Domain, File, Mailer, Nodelist, Setup, System, Zone}; use App\Traits\Import as ImportTrait; class NodelistImport implements ShouldQueue @@ -24,7 +24,7 @@ class NodelistImport implements ShouldQueue protected const LOGKEY = 'JNI'; private const importkey = 'nodelist'; - private File $file; + private File|string $file; private ?string $domain; private bool $delete_file; private bool $delete_recs; @@ -32,12 +32,12 @@ class NodelistImport implements ShouldQueue /** * Import Nodelist constructor. * - * @param File $file + * @param File|string $file * @param string|null $domain * @param bool $delete_recs * @param bool $delete_file */ - public function __construct(File $file,string $domain=NULL,bool $delete_recs=FALSE,bool $delete_file=TRUE) + public function __construct(File|string $file,string $domain=NULL,bool $delete_recs=FALSE,bool $delete_file=TRUE) { $this->file = $file; $this->domain = $domain; @@ -73,7 +73,7 @@ class NodelistImport implements ShouldQueue throw new \Exception('Invalid nodelist for file: '.$this->file->id); } - $file_crc = $matches[4]; + $file_crc = (int)$matches[4]; $do = Domain::where('name',strtolower($matches[1] ?: $this->domain))->single(); if (! $do) { @@ -103,6 +103,8 @@ class NodelistImport implements ShouldQueue $hub_id = NULL; $zo = NULL; $tocrc = ''; + $mailer_binkp = Mailer::where('name','BINKP')->singleOrFail(); + $mailer_emsi = Mailer::where('name','EMSI')->singleOrFail(); while (! feof($fh)) { $line = stream_get_line($fh,0,"\r\n"); @@ -129,7 +131,7 @@ class NodelistImport implements ShouldQueue switch ($fields[0]) { case 'Zone': - $zone = $fields[1]; + $zone = (int)$fields[1]; Zone::unguard(); $zo = Zone::firstOrNew([ 'zone_id'=>$zone, @@ -146,40 +148,40 @@ class NodelistImport implements ShouldQueue break; case 'Region': - $region = $fields[1]; - $host = $fields[1]; + $region = (int)$fields[1]; + $host = (int)$fields[1]; $hub_id = NULL; $role = Address::NODE_RC; break; case 'Host': - $host = $fields[1]; + $host = (int)$fields[1]; $hub_id = NULL; $role = Address::NODE_NC; break; case 'Hub': - $node = $fields[1]; + $node = (int)$fields[1]; $role = Address::NODE_HC; break; case 'Pvt': - $node = $fields[1]; + $node = (int)$fields[1]; $role = Address::NODE_PVT; break; case 'Hold': - $node = $fields[1]; + $node = (int)$fields[1]; $role = Address::NODE_HOLD; break; case 'Down': - $node = $fields[1]; + $node = (int)$fields[1]; $role = Address::NODE_DOWN; break; @@ -202,14 +204,17 @@ class NodelistImport implements ShouldQueue Address::unguard(); $ao = Address::firstOrNew([ 'zone_id' => $zo->id, - 'region_id' => $region, 'host_id' => $host, 'node_id' => $node, 'point_id' => 0, + 'active' => TRUE, ]); Address::reguard(); - $ao->active = TRUE; + if ($ao->region_id && ($ao->region_id !== $region)) + Log::alert(sprintf('%s:Address [%s] changing regions [%d->%d]',self::LOGKEY,$ao->ftn,$ao->region_id,$region)); + + $ao->region_id = $region; $ao->role = $role; $ao->hub_id = $hub_id; @@ -220,48 +225,57 @@ class NodelistImport implements ShouldQueue $system = trim(str_replace('_',' ',$fields[2])); // Flags - $method = NULL; + $methods = collect(); $address = ''; - $port = ''; for ($i=7;$isplit('/:/'); switch ($x->first()) { - // address + // Address case 'INA': $address = $x->get(1); break; + // BINKP case 'IBN': - $method = Setup::O_BINKP; - - switch ($x->count()) { - case 1: - $port = 24554; - break; - case 2: - $port = $x->get(1); - break; - case 3: - $address = $x->get(1); - $port = $x->get(2); - } - break; - case 'ITN': - $method = Setup::O_EMSI; + if ($x->first() === 'IBN') { + $dport = 24554; + $method = $mailer_binkp->id; + + } else { + $dport = 60179; + $method = $mailer_emsi->id; + } switch ($x->count()) { case 1: - $port = 60179; + $methods->put($method,['port'=>$dport]); break; + case 2: - $port = $x->get(1); + $mp = is_numeric($x->get(1)) ? (int)$x->get(1) : $dport; + $ma = is_numeric($xx=$x->get(1)) ? NULL : $xx; + + if ($ma && ($ma !== $address)) + $methods->put($method,['address'=>$ma,'port'=>$mp]); + else + $methods->put($method,['port'=>$mp]); + break; + case 3: - $address = $x->get(1); - $port = $x->get(2); + $mp = (int)$x->get(2); + $ma = $x->get(1); + + if ($ma && ($ma !== $address)) + $methods->put($method,['address'=>$ma,'port'=>$mp]); + else + $methods->put($method,['port'=>$mp]); + + break; } + break; // Ignore @@ -278,7 +292,62 @@ class NodelistImport implements ShouldQueue } // Get the System - if ($ao->system_id && (($ao->system->sysop === $sysop) || ($ao->system->name === $system))) { + // If we are a zone/region record, then the system may change + switch ($role) { + case Address::NODE_ZC: + case Address::NODE_RC: + case Address::NODE_NC: + $so = ($x=System::distinct('systems.*') + ->join('mailer_system',['mailer_system.system_id'=>'systems.id']) + ->where('sysop',$sysop) + ->where(function($query) use ($address,$methods) { + return $query->where('systems.address',$address) + ->when($methods->pluck('address')->filter()->count(),function ($query) use ($methods) { + return $query->whereIN('mailer_system.address',$methods->pluck('address')); + }); + }) + ->when($methods->pluck('port')->filter()->count(),function ($query) use ($methods) { + return $query->whereIN('mailer_system.port',$methods->pluck('port')); + })) + ->single(); + + if (! $so) { + Log::info(sprintf('%s:New System for ZC/RC/NC [%s] - System [%s] Sysop [%s]',self::LOGKEY,$ao->ftn,$system,$sysop)); + $so = new System; + $so->sysop = $sysop; + $so->name = $system; + $so->address = $address; + $so->location = 'TBA'; + $so->notes = sprintf('Created by Nodelist Import: %d',$no->id); + $so->active = TRUE; + + $so->save(); + } + + // If the address exists, but it was discovered, assign it to this new host + if ($ao->system_id && ($ao->system_id !== $so->id) && ($ao->system->name === 'Discovered System')) { + Log::info(sprintf('%s:Re-assigning discovered address to [%s]',self::LOGKEY,$so->id)); + + } elseif (! $ao->system_id) { + Log::alert(sprintf('%s:[%s] new system created [%s:%s]',self::LOGKEY,$ao->ftn,$system,$sysop)); + + } elseif ($ao->system_id !== $so->id) { + Log::alert(sprintf('%s:[%s] hosted by new system [%s:%s] (was %s:%s)',self::LOGKEY,$ao->ftn,$system,$sysop,$ao->system->name,$ao->system->sysop)); + $ao->active = FALSE; + $ao->save(); + + $ao = $ao->replicate(); + $ao->active = TRUE; + } + + $ao->system_id = $so->id; + $ao->save(); + + $ao->load('system'); + } + + if ($ao->system_id && ((($ao->system->sysop === $sysop) || ($ao->system->name === $system)) && (($ao->system->address === $address) || ($methods->pluck('address')->search($address) !== FALSE)))) { + Log::debug(sprintf('%s:Matched [%s] to existing system [%s] with address [%s]',self::LOGKEY,$ao->ftn,$ao->system->name,$ao->system->address)); $so = $ao->system; // Dont change the system details if a user exists here, or its us @@ -290,8 +359,8 @@ class NodelistImport implements ShouldQueue $so->sysop = $sysop; - // We have the same name has changed. - } elseif ($so->name !== $system) { + // We have the same name has changed (except for ZC/RC addresses) + } elseif (($so->name !== $system) && (! ((Address::NODE_ZC|Address::NODE_RC|Address::NODE_NC) & $ao->role))) { Log::debug(sprintf('%s:System Name changed for BBS [%s:%s] to [%s]', self::LOGKEY,$so->id,$so->name,$system)); @@ -301,12 +370,17 @@ class NodelistImport implements ShouldQueue // We'll search and see if we already have that system } else { + Log::debug(sprintf('%s:Looking for existing system [%s] with address [%s]',self::LOGKEY,$system,$address)); // If we dont have $address/port $so = NULL; - if ($address && $port) - $so = System::where('mailer_address',$address) - ->where('mailer_port',$port) + if ($address) + $so = System::select('systems.*') + ->join('mailer_system',['mailer_system.system_id'=>'systems.id']) + ->where(function($query) use ($address) { + return $query->where('systems.address',$address) + ->orWhere('mailer_system.address',$address); + }) ->single(); if (! $so) @@ -317,7 +391,7 @@ class NodelistImport implements ShouldQueue if ($so->exists) Log::debug(sprintf('%s:Linking address [%d:%d/%d] to [%s:%s]',self::LOGKEY,$zo->zone_id,$ao->host_id,$ao->node_id,$so->id,$so->name)); else - Log::debug(sprintf('%s:New System [%s] with address [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id)); + Log::debug(sprintf('%s:New System [%s] with FTN [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id)); $so->name = $system; $so->sysop = $sysop; @@ -336,22 +410,22 @@ class NodelistImport implements ShouldQueue $so->baud = $fields[6]; */ - if ($method && ($ao->role != Address::NODE_PVT)) { - $so->mailer_type = $method; - $so->mailer_address = $address; - $so->mailer_port = $port; - } - // Save the system record try { $so->save(); } catch (\Exception $e) { Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); + DB::rollBack(); throw new \Exception($e->getMessage()); } + if ($methods->count() && ($ao->role != Address::NODE_PVT)) { + $methods->transform(function($item) { $item['active'] = Arr::get($item,'active',TRUE); return $item; }); + $so->mailers()->sync($methods); + } + // If our zone didnt exist, we'll create it with this system if (! $zo->exists) { $zo->system_id = $so->id; @@ -365,6 +439,7 @@ class NodelistImport implements ShouldQueue try { $so->addresses()->save($ao); + if ($ao->role === Address::NODE_HC) $hub_id = $ao->id; @@ -372,6 +447,7 @@ class NodelistImport implements ShouldQueue } catch (\Exception $e) { Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); + DB::rollBack(); throw new \Exception($e->getMessage()); } @@ -387,7 +463,7 @@ class NodelistImport implements ShouldQueue Address::whereIN('id',$remove->pluck('id')->toArray())->update(['active'=>FALSE]); Address::whereIN('id',$remove->pluck('id')->toArray())->delete(); - if ($x=crc16(substr($tocrc,0,-3)) === $file_crc) { + if (($x=crc16(substr($tocrc,0,-3))) === $file_crc) { Log::info(sprintf('%s:Committing nodelist',self::LOGKEY)); DB::commit(); } else { diff --git a/app/Traits/Import.php b/app/Traits/Import.php index 683df55..f85b29b 100644 --- a/app/Traits/Import.php +++ b/app/Traits/Import.php @@ -36,12 +36,11 @@ trait Import return $c; } - // @todo Consider merging this with File::openZipFile private function openFile(string $file,&$f): \ZipArchive { $z = new \ZipArchive; - if ($z->open($file)) { + if ($z->open($file,\ZipArchive::RDONLY) === TRUE) { if ($z->count() !== 1) throw new \Exception(sprintf('%s:File [%s] has more than 1 file (%d)', self::LOGKEY, $file, $z->count())); @@ -82,12 +81,17 @@ trait Import return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL; } - private function getFileFromHost(string $key,File $file): string + private function getFileFromHost(string $key,mixed $file): string { - $path = sprintf('import/%s.%d',$key,$file->id); + if ($file instanceof File) { + $path = sprintf('import/%s.%d',$key,$file->id); - Storage::disk('local')->put($path,Storage::get($file->full_storage_path)); + Storage::disk('local')->put($path,Storage::get($file->full_storage_path)); - return Storage::disk('local')->path($path); + return Storage::disk('local')->path($path); + + } else { + return Storage::disk('local')->path($file); + } } } \ No newline at end of file diff --git a/database/migrations/2023_07_09_125321_mailer_address.php b/database/migrations/2023_07_09_125321_mailer_address.php new file mode 100644 index 0000000..1bd80e1 --- /dev/null +++ b/database/migrations/2023_07_09_125321_mailer_address.php @@ -0,0 +1,28 @@ +string('address')->unqiue()->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('mailer_system',function (Blueprint $table) { + $table->dropColumn(['address']); + }); + } +};