bbs/clrghouz
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Update nodelist parsing now that we have mailer_system

Browse Source
This commit is contained in:
Deon George 2023-07-09 22:03:15 +10:00
parent d4439a6b58
commit 52055c5217
4 changed files with 173 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/Console/Commands/NodelistImport.php
View File

@ -15,7 +15,7 @@ class NodelistImport extends Command
* @var string * @var string
*/ */
protected $signature = 'nodelist:import' protected $signature = 'nodelist:import'
.' {file : File ID}' .' {file : File ID | filename}'
.' {domain? : Domain Name}' .' {domain? : Domain Name}'
.' {--D|delete : Delete old data for the date}' .' {--D|delete : Delete old data for the date}'
.' {--U|unlink : Delete file after import}'; .' {--U|unlink : Delete file after import}';
@ -34,6 +34,11 @@ class NodelistImport extends Command
*/ */
public function handle() public function handle()
{ {
return Job::dispatchSync(File::findOrFail($this->argument('file')),$this->argument('domain'),$this->option('delete'),$this->option('unlink')); return Job::dispatchSync(
is_numeric($x=$this->argument('file')) ? File::findOrFail($x) : $x,
$this->argument('domain'),
$this->option('delete'),
$this->option('unlink')
);
} }
} }

180
app/Jobs/NodelistImport.php
View File

@ -13,7 +13,7 @@ use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str; use Illuminate\Support\Str;
use App\Models\{Address,Domain,File,Nodelist,Setup,System,Zone}; use App\Models\{Address, Domain, File, Mailer, Nodelist, Setup, System, Zone};
use App\Traits\Import as ImportTrait; use App\Traits\Import as ImportTrait;
class NodelistImport implements ShouldQueue class NodelistImport implements ShouldQueue
@ -24,7 +24,7 @@ class NodelistImport implements ShouldQueue
protected const LOGKEY = 'JNI'; protected const LOGKEY = 'JNI';
private const importkey = 'nodelist'; private const importkey = 'nodelist';
private File $file; private File|string $file;
private ?string $domain; private ?string $domain;
private bool $delete_file; private bool $delete_file;
private bool $delete_recs; private bool $delete_recs;
@ -32,12 +32,12 @@ class NodelistImport implements ShouldQueue
/** /**
* Import Nodelist constructor. * Import Nodelist constructor.
* *
* @param File $file * @param File|string $file
* @param string|null $domain * @param string|null $domain
* @param bool $delete_recs * @param bool $delete_recs
* @param bool $delete_file * @param bool $delete_file
*/ */
public function __construct(File $file,string $domain=NULL,bool $delete_recs=FALSE,bool $delete_file=TRUE) public function __construct(File|string $file,string $domain=NULL,bool $delete_recs=FALSE,bool $delete_file=TRUE)
{ {
$this->file = $file; $this->file = $file;
$this->domain = $domain; $this->domain = $domain;
@ -73,7 +73,7 @@ class NodelistImport implements ShouldQueue
throw new \Exception('Invalid nodelist for file: '.$this->file->id); throw new \Exception('Invalid nodelist for file: '.$this->file->id);
} }
$file_crc = $matches[4]; $file_crc = (int)$matches[4];
$do = Domain::where('name',strtolower($matches[1] ?: $this->domain))->single(); $do = Domain::where('name',strtolower($matches[1] ?: $this->domain))->single();
if (! $do) { if (! $do) {
@ -103,6 +103,8 @@ class NodelistImport implements ShouldQueue
$hub_id = NULL; $hub_id = NULL;
$zo = NULL; $zo = NULL;
$tocrc = ''; $tocrc = '';
$mailer_binkp = Mailer::where('name','BINKP')->singleOrFail();
$mailer_emsi = Mailer::where('name','EMSI')->singleOrFail();
while (! feof($fh)) { while (! feof($fh)) {
$line = stream_get_line($fh,0,"\r\n"); $line = stream_get_line($fh,0,"\r\n");
@ -129,7 +131,7 @@ class NodelistImport implements ShouldQueue
switch ($fields[0]) { switch ($fields[0]) {
case 'Zone': case 'Zone':
$zone = $fields[1]; $zone = (int)$fields[1];
Zone::unguard(); Zone::unguard();
$zo = Zone::firstOrNew([ $zo = Zone::firstOrNew([
'zone_id'=>$zone, 'zone_id'=>$zone,
@ -146,40 +148,40 @@ class NodelistImport implements ShouldQueue
break; break;
case 'Region': case 'Region':
$region = $fields[1]; $region = (int)$fields[1];
$host = $fields[1]; $host = (int)$fields[1];
$hub_id = NULL; $hub_id = NULL;
$role = Address::NODE_RC; $role = Address::NODE_RC;
break; break;
case 'Host': case 'Host':
$host = $fields[1]; $host = (int)$fields[1];
$hub_id = NULL; $hub_id = NULL;
$role = Address::NODE_NC; $role = Address::NODE_NC;
break; break;
case 'Hub': case 'Hub':
$node = $fields[1]; $node = (int)$fields[1];
$role = Address::NODE_HC; $role = Address::NODE_HC;
break; break;
case 'Pvt': case 'Pvt':
$node = $fields[1]; $node = (int)$fields[1];
$role = Address::NODE_PVT; $role = Address::NODE_PVT;
break; break;
case 'Hold': case 'Hold':
$node = $fields[1]; $node = (int)$fields[1];
$role = Address::NODE_HOLD; $role = Address::NODE_HOLD;
break; break;
case 'Down': case 'Down':
$node = $fields[1]; $node = (int)$fields[1];
$role = Address::NODE_DOWN; $role = Address::NODE_DOWN;
break; break;
@ -202,14 +204,17 @@ class NodelistImport implements ShouldQueue
Address::unguard(); Address::unguard();
$ao = Address::firstOrNew([ $ao = Address::firstOrNew([
'zone_id' => $zo->id, 'zone_id' => $zo->id,
'region_id' => $region,
'host_id' => $host, 'host_id' => $host,
'node_id' => $node, 'node_id' => $node,
'point_id' => 0, 'point_id' => 0,
'active' => TRUE,
]); ]);
Address::reguard(); Address::reguard();
$ao->active = TRUE; if ($ao->region_id && ($ao->region_id !== $region))
Log::alert(sprintf('%s:Address [%s] changing regions [%d->%d]',self::LOGKEY,$ao->ftn,$ao->region_id,$region));
$ao->region_id = $region;
$ao->role = $role; $ao->role = $role;
$ao->hub_id = $hub_id; $ao->hub_id = $hub_id;
@ -220,48 +225,57 @@ class NodelistImport implements ShouldQueue
$system = trim(str_replace('_',' ',$fields[2])); $system = trim(str_replace('_',' ',$fields[2]));
// Flags // Flags
$method = NULL; $methods = collect();
$address = ''; $address = '';
$port = '';
for ($i=7;$i<count($fields);$i++) { for ($i=7;$i<count($fields);$i++) {
$x = Str::of($fields[$i])->split('/:/'); $x = Str::of($fields[$i])->split('/:/');
switch ($x->first()) { switch ($x->first()) {
// address // Address
case 'INA': case 'INA':
$address = $x->get(1); $address = $x->get(1);
break; break;
// BINKP
case 'IBN': case 'IBN':
$method = Setup::O_BINKP;
switch ($x->count()) {
case 1:
$port = 24554;
break;
case 2:
$port = $x->get(1);
break;
case 3:
$address = $x->get(1);
$port = $x->get(2);
}
break;
case 'ITN': case 'ITN':
$method = Setup::O_EMSI; if ($x->first() === 'IBN') {
$dport = 24554;
$method = $mailer_binkp->id;
} else {
$dport = 60179;
$method = $mailer_emsi->id;
}
switch ($x->count()) { switch ($x->count()) {
case 1: case 1:
$port = 60179; $methods->put($method,['port'=>$dport]);
break; break;
case 2: case 2:
$port = $x->get(1); $mp = is_numeric($x->get(1)) ? (int)$x->get(1) : $dport;
$ma = is_numeric($xx=$x->get(1)) ? NULL : $xx;
if ($ma && ($ma !== $address))
$methods->put($method,['address'=>$ma,'port'=>$mp]);
else
$methods->put($method,['port'=>$mp]);
break; break;
case 3: case 3:
$address = $x->get(1); $mp = (int)$x->get(2);
$port = $x->get(2); $ma = $x->get(1);
if ($ma && ($ma !== $address))
$methods->put($method,['address'=>$ma,'port'=>$mp]);
else
$methods->put($method,['port'=>$mp]);
break;
} }
break; break;
// Ignore // Ignore
@ -278,7 +292,62 @@ class NodelistImport implements ShouldQueue
} }
// Get the System // Get the System
if ($ao->system_id && (($ao->system->sysop === $sysop) || ($ao->system->name === $system))) { // If we are a zone/region record, then the system may change
switch ($role) {
case Address::NODE_ZC:
case Address::NODE_RC:
case Address::NODE_NC:
$so = ($x=System::distinct('systems.*')
->join('mailer_system',['mailer_system.system_id'=>'systems.id'])
->where('sysop',$sysop)
->where(function($query) use ($address,$methods) {
return $query->where('systems.address',$address)
->when($methods->pluck('address')->filter()->count(),function ($query) use ($methods) {
return $query->whereIN('mailer_system.address',$methods->pluck('address'));
});
})
->when($methods->pluck('port')->filter()->count(),function ($query) use ($methods) {
return $query->whereIN('mailer_system.port',$methods->pluck('port'));
}))
->single();
if (! $so) {
Log::info(sprintf('%s:New System for ZC/RC/NC [%s] - System [%s] Sysop [%s]',self::LOGKEY,$ao->ftn,$system,$sysop));
$so = new System;
$so->sysop = $sysop;
$so->name = $system;
$so->address = $address;
$so->location = 'TBA';
$so->notes = sprintf('Created by Nodelist Import: %d',$no->id);
$so->active = TRUE;
$so->save();
}
// If the address exists, but it was discovered, assign it to this new host
if ($ao->system_id && ($ao->system_id !== $so->id) && ($ao->system->name === 'Discovered System')) {
Log::info(sprintf('%s:Re-assigning discovered address to [%s]',self::LOGKEY,$so->id));
} elseif (! $ao->system_id) {
Log::alert(sprintf('%s:[%s] new system created [%s:%s]',self::LOGKEY,$ao->ftn,$system,$sysop));
} elseif ($ao->system_id !== $so->id) {
Log::alert(sprintf('%s:[%s] hosted by new system [%s:%s] (was %s:%s)',self::LOGKEY,$ao->ftn,$system,$sysop,$ao->system->name,$ao->system->sysop));
$ao->active = FALSE;
$ao->save();
$ao = $ao->replicate();
$ao->active = TRUE;
}
$ao->system_id = $so->id;
$ao->save();
$ao->load('system');
}
if ($ao->system_id && ((($ao->system->sysop === $sysop) || ($ao->system->name === $system)) && (($ao->system->address === $address) || ($methods->pluck('address')->search($address) !== FALSE)))) {
Log::debug(sprintf('%s:Matched [%s] to existing system [%s] with address [%s]',self::LOGKEY,$ao->ftn,$ao->system->name,$ao->system->address));
$so = $ao->system; $so = $ao->system;
// Dont change the system details if a user exists here, or its us // Dont change the system details if a user exists here, or its us
@ -290,8 +359,8 @@ class NodelistImport implements ShouldQueue
$so->sysop = $sysop; $so->sysop = $sysop;
// We have the same name has changed. // We have the same name has changed (except for ZC/RC addresses)
} elseif ($so->name !== $system) { } elseif (($so->name !== $system) && (! ((Address::NODE_ZC|Address::NODE_RC|Address::NODE_NC) & $ao->role))) {
Log::debug(sprintf('%s:System Name changed for BBS [%s:%s] to [%s]', Log::debug(sprintf('%s:System Name changed for BBS [%s:%s] to [%s]',
self::LOGKEY,$so->id,$so->name,$system)); self::LOGKEY,$so->id,$so->name,$system));
@ -301,12 +370,17 @@ class NodelistImport implements ShouldQueue
// We'll search and see if we already have that system // We'll search and see if we already have that system
} else { } else {
Log::debug(sprintf('%s:Looking for existing system [%s] with address [%s]',self::LOGKEY,$system,$address));
// If we dont have $address/port // If we dont have $address/port
$so = NULL; $so = NULL;
if ($address && $port) if ($address)
$so = System::where('mailer_address',$address) $so = System::select('systems.*')
->where('mailer_port',$port) ->join('mailer_system',['mailer_system.system_id'=>'systems.id'])
->where(function($query) use ($address) {
return $query->where('systems.address',$address)
->orWhere('mailer_system.address',$address);
})
->single(); ->single();
if (! $so) if (! $so)
@ -317,7 +391,7 @@ class NodelistImport implements ShouldQueue
if ($so->exists) if ($so->exists)
Log::debug(sprintf('%s:Linking address [%d:%d/%d] to [%s:%s]',self::LOGKEY,$zo->zone_id,$ao->host_id,$ao->node_id,$so->id,$so->name)); Log::debug(sprintf('%s:Linking address [%d:%d/%d] to [%s:%s]',self::LOGKEY,$zo->zone_id,$ao->host_id,$ao->node_id,$so->id,$so->name));
else else
Log::debug(sprintf('%s:New System [%s] with address [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id)); Log::debug(sprintf('%s:New System [%s] with FTN [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id));
$so->name = $system; $so->name = $system;
$so->sysop = $sysop; $so->sysop = $sysop;
@ -336,22 +410,22 @@ class NodelistImport implements ShouldQueue
$so->baud = $fields[6]; $so->baud = $fields[6];
*/ */
if ($method && ($ao->role != Address::NODE_PVT)) {
$so->mailer_type = $method;
$so->mailer_address = $address;
$so->mailer_port = $port;
}
// Save the system record // Save the system record
try { try {
$so->save(); $so->save();
} catch (\Exception $e) { } catch (\Exception $e) {
Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
DB::rollBack(); DB::rollBack();
throw new \Exception($e->getMessage()); throw new \Exception($e->getMessage());
} }
if ($methods->count() && ($ao->role != Address::NODE_PVT)) {
$methods->transform(function($item) { $item['active'] = Arr::get($item,'active',TRUE); return $item; });
$so->mailers()->sync($methods);
}
// If our zone didnt exist, we'll create it with this system // If our zone didnt exist, we'll create it with this system
if (! $zo->exists) { if (! $zo->exists) {
$zo->system_id = $so->id; $zo->system_id = $so->id;
@ -365,6 +439,7 @@ class NodelistImport implements ShouldQueue
try { try {
$so->addresses()->save($ao); $so->addresses()->save($ao);
if ($ao->role === Address::NODE_HC) if ($ao->role === Address::NODE_HC)
$hub_id = $ao->id; $hub_id = $ao->id;
@ -372,6 +447,7 @@ class NodelistImport implements ShouldQueue
} catch (\Exception $e) { } catch (\Exception $e) {
Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
DB::rollBack(); DB::rollBack();
throw new \Exception($e->getMessage()); throw new \Exception($e->getMessage());
} }
@ -387,7 +463,7 @@ class NodelistImport implements ShouldQueue
Address::whereIN('id',$remove->pluck('id')->toArray())->update(['active'=>FALSE]); Address::whereIN('id',$remove->pluck('id')->toArray())->update(['active'=>FALSE]);
Address::whereIN('id',$remove->pluck('id')->toArray())->delete(); Address::whereIN('id',$remove->pluck('id')->toArray())->delete();
if ($x=crc16(substr($tocrc,0,-3)) === $file_crc) { if (($x=crc16(substr($tocrc,0,-3))) === $file_crc) {
Log::info(sprintf('%s:Committing nodelist',self::LOGKEY)); Log::info(sprintf('%s:Committing nodelist',self::LOGKEY));
DB::commit(); DB::commit();
} else { } else {

10
app/Traits/Import.php
View File

@ -36,12 +36,11 @@ trait Import
return $c; return $c;
} }
// @todo Consider merging this with File::openZipFile
private function openFile(string $file,&$f): \ZipArchive private function openFile(string $file,&$f): \ZipArchive
{ {
$z = new \ZipArchive; $z = new \ZipArchive;
if ($z->open($file)) { if ($z->open($file,\ZipArchive::RDONLY) === TRUE) {
if ($z->count() !== 1) if ($z->count() !== 1)
throw new \Exception(sprintf('%s:File [%s] has more than 1 file (%d)', self::LOGKEY, $file, $z->count())); throw new \Exception(sprintf('%s:File [%s] has more than 1 file (%d)', self::LOGKEY, $file, $z->count()));
@ -82,12 +81,17 @@ trait Import
return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL; return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL;
} }
private function getFileFromHost(string $key,File $file): string private function getFileFromHost(string $key,mixed $file): string
{ {
if ($file instanceof File) {
$path = sprintf('import/%s.%d',$key,$file->id); $path = sprintf('import/%s.%d',$key,$file->id);
Storage::disk('local')->put($path,Storage::get($file->full_storage_path)); Storage::disk('local')->put($path,Storage::get($file->full_storage_path));
return Storage::disk('local')->path($path); return Storage::disk('local')->path($path);
} else {
return Storage::disk('local')->path($file);
}
} }
} }

28
database/migrations/2023_07_09_125321_mailer_address.php Normal file
View File

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('mailer_system',function (Blueprint $table) {
$table->string('address')->unqiue()->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('mailer_system',function (Blueprint $table) {
$table->dropColumn(['address']);
});
}
};