From 8c992ad0f9532e045779f9ffef3c3c20833f6099 Mon Sep 17 00:00:00 2001 From: Deon George Date: Sat, 21 Aug 2021 21:15:22 +1000 Subject: [PATCH] Enhance nodelist import --- app/Console/Commands/NodelistImport.php | 21 +-- app/Http/Controllers/SystemController.php | 3 +- app/Jobs/NodelistImport.php | 169 ++++++++++++++---- app/Models/System.php | 5 + app/Traits/CRC.php | 10 +- .../2021_08_21_134156_add_phone.php | 32 ++++ resources/views/system/form-system.blade.php | 25 ++- 7 files changed, 210 insertions(+), 55 deletions(-) create mode 100644 database/migrations/2021_08_21_134156_add_phone.php diff --git a/app/Console/Commands/NodelistImport.php b/app/Console/Commands/NodelistImport.php index a58e233..0b1984a 100644 --- a/app/Console/Commands/NodelistImport.php +++ b/app/Console/Commands/NodelistImport.php @@ -2,10 +2,8 @@ namespace App\Console\Commands; -use Carbon\Carbon; use Illuminate\Console\Command; -use App\Models\{Domain,Nodelist}; use App\Jobs\NodelistImport as Job; class NodelistImport extends Command @@ -16,9 +14,9 @@ class NodelistImport extends Command * @var string */ protected $signature = 'nodelist:import' - .' {domain : Domain Name}' .' {file : Nodelist File}' - .' {--D|delete : Delete old data for the date}'; + .' {--D|delete : Delete old data for the date}' + .' {--U|unlink : Delete file after import}'; /** * The console command description. @@ -27,16 +25,6 @@ class NodelistImport extends Command */ protected $description = 'Import Nodelist'; - /** - * Create a new command instance. - * - * @return void - */ - public function __construct() - { - parent::__construct(); - } - /** * Execute the console command. * @@ -44,9 +32,6 @@ class NodelistImport extends Command */ public function handle() { - $do = Domain::where('name',$this->argument('domain'))->singleOrFail(); - $o = Nodelist::firstOrCreate(['date'=>Carbon::now(),'domain_id'=>$do->id]); - - return Job::dispatchSync($do,$o,$this->argument('file'),$this->option('delete')); + return Job::dispatchSync($this->argument('file'),$this->option('delete'),$this->option('unlink')); } } \ No newline at end of file diff --git a/app/Http/Controllers/SystemController.php b/app/Http/Controllers/SystemController.php index 903d5eb..00152f1 100644 --- a/app/Http/Controllers/SystemController.php +++ b/app/Http/Controllers/SystemController.php @@ -257,6 +257,7 @@ class SystemController extends Controller 'name' => 'required|min:3', 'location' => 'required|min:3', 'sysop' => 'required|min:3', + 'phone' => 'nullable|regex:/^([0-9-]+)$/', 'address' => 'nullable|regex:/^(?!:\/\/)(?=.{1,255}$)((.{1,63}\.){1,127}(?![0-9]*$)[a-z0-9-]+\.?)$/i', 'port' => 'nullable|digits_between:2,5', 'method' => 'nullable|numeric', @@ -267,7 +268,7 @@ class SystemController extends Controller 'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($o->exists ? $o->id : 0), ]); - foreach (['name','location','sysop','address','port','active','method','notes','mailer_type','mailer_address','mailer_port','zt_id'] as $key) + foreach (['name','location','sysop','phone','address','port','active','method','notes','mailer_type','mailer_address','mailer_port','zt_id'] as $key) $o->{$key} = $request->post($key); $o->save(); diff --git a/app/Jobs/NodelistImport.php b/app/Jobs/NodelistImport.php index 3791121..106f01c 100644 --- a/app/Jobs/NodelistImport.php +++ b/app/Jobs/NodelistImport.php @@ -2,16 +2,18 @@ namespace App\Jobs; +use Carbon\Carbon; use Illuminate\Bus\Queueable; -use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; +use Illuminate\Support\Str; use App\Http\Controllers\DomainController; -use App\Models\{Address,Domain,Nodelist,System,Zone}; +use App\Models\{Address,Domain,Nodelist,Setup,System,Zone}; use App\Traits\Import as ImportTrait; class NodelistImport implements ShouldQueue @@ -21,27 +23,23 @@ class NodelistImport implements ShouldQueue protected const LOGKEY = 'JIN'; private const importkey = 'nodelist'; + private string $file; - private Domain $do; - private Nodelist $no; - private bool $deletefile = FALSE; + private bool $delete_file; + private bool $delete_recs; /** * Import Nodelist constructor. * - * @param Domain $do - * @param Nodelist $no * @param string $file - * @param bool $delete + * @param bool $delete_recs + * @param bool $delete_file */ - public function __construct(Domain $do,Nodelist $no,string $file,bool $delete=TRUE) + public function __construct(string $file,bool $delete_recs=FALSE,bool $delete_file=TRUE) { - $this->do = $do; - $this->no = $no; $this->file = $file; - - if ($delete) - $no->addresses()->detach(); + $this->delete_file = $delete_file; + $this->delete_recs = $delete_recs; } /** @@ -52,21 +50,56 @@ class NodelistImport implements ShouldQueue */ public function handle() { + $us = Setup::findOrFail(config('app.id')); + // Get the file from the host $file = $this->getFileFromHost('',self::importkey,$this->file); $lines = $this->getFileLines($file); Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines)); $fh = fopen($file,'r'); + + // Line 1 tells us the nodelist and the CRC + $line = stream_get_line($fh, 0, "\r\n"); + + $matches = []; + if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) + abort(500,'Nodelist invalid'); + + $file_crc = $matches[4]; + $do = Domain::where('name',strtolower($matches[1]))->single(); + + if (! $do) { + Log::error(sprintf('%s:! Domain not found [%s].',static::LOGKEY,strtolower($matches[1]))); + return; + } + + $date = Carbon::createFromFormat('D, M d, Y H:i',$matches[2].'0:00'); + + if ($date->dayOfYear != $matches[3]) { + Log::error(sprintf('%s:! Nodelist date doesnt match [%d] (%d:%s).',static::LOGKEY,$matches[3],$date->dayOfYear,$date->format('Y-m-d'))); + return; + } + + Log::debug(sprintf('%s:Importing nodelist for [%s] dated [%s] lines.',static::LOGKEY,$do->name,$date->format('Y-m-d'))); + + DB::beginTransaction(); + $no = Nodelist::firstOrCreate(['date'=>$date,'domain_id'=>$do->id]); + + if ($this->delete_recs) + $no->addresses()->detach(); + $p = $c =0; $region = NULL; $host = NULL; $hub_id = NULL; $zo = NULL; + $tocrc = ''; while (! feof($fh)) { $line = stream_get_line($fh, 0, "\r\n"); + $tocrc .= $line."\r\n"; // Lines beginning with a semicolon(;) are comments if (preg_match('/^;/',$line) OR ($line == chr(0x1a))) @@ -92,7 +125,7 @@ class NodelistImport implements ShouldQueue Zone::unguard(); $zo = Zone::firstOrNew([ 'zone_id'=>$zone, - 'domain_id'=>$this->do->id, + 'domain_id'=>$do->id, 'active'=>TRUE, ]); Zone::reguard(); @@ -159,6 +192,7 @@ class NodelistImport implements ShouldQueue Address::unguard(); $ao = Address::firstOrNew([ + 'zone_id' => $zo->id, 'region_id' => $region, 'host_id' => $host, 'node_id' => $node, @@ -177,25 +211,27 @@ class NodelistImport implements ShouldQueue $sysop = trim(str_replace('_',' ',$fields[4])); $system = trim(str_replace('_',' ',$fields[2])); - $location = trim(str_replace('_',' ',$fields[3])); // Get the System if ($ao->system_id && (($ao->system->sysop === $sysop) || ($ao->system->name === $system))) { $so = $ao->system; - // If the sysop name is different - if ($so->sysop !== $sysop) { - Log::debug(sprintf('%s:Sysop Name changed for BBS [%s:%s] from [%s] to [%s]', - self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop)); + // Dont change the system details if a user exists here, or its us + if ((! $so->users->count()) && ($so->id != $us->system_id)) { + // If the sysop name is different + if ($so->sysop !== $sysop) { + Log::debug(sprintf('%s:Sysop Name changed for BBS [%s:%s] from [%s] to [%s]', + self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop)); - $so->sysop = $sysop; + $so->sysop = $sysop; - // We have the same name has changed. - } elseif ($so->name !== $system) { - Log::debug(sprintf('%s:System Name changed for BBS [%s:%s] to [%s]', - self::LOGKEY,$so->id,$so->name,$system)); + // We have the same name has changed. + } elseif ($so->name !== $system) { + Log::debug(sprintf('%s:System Name changed for BBS [%s:%s] to [%s]', + self::LOGKEY,$so->id,$so->name,$system)); - $so->name = $system; + $so->name = $system; + } } // We'll search and see if we already have that system @@ -214,10 +250,11 @@ class NodelistImport implements ShouldQueue $so->active = TRUE; if (! $so->exists) - $so->notes = sprintf('Created by Nodelist Import: %d',$this->no->id); + $so->notes = sprintf('Created by Nodelist Import: %d',$no->id); } - $so->location = $location; + $so->phone = $fields[5] != '-Unpublished-' ? $fields[5] : NULL; + $so->location = trim(str_replace('_',' ',$fields[3])); /* if (! in_array($fields[5],['-Unpublished-'])) $so->phone = $fields[5]; @@ -225,6 +262,70 @@ class NodelistImport implements ShouldQueue $so->baud = $fields[6]; */ + // Flags + $method = NULL; + $address = ''; + $port = ''; + for ($i=7;$isplit('/:/'); + + switch ($x->first()) { + // address + case 'INA': + $address = $x->get(1); + break; + + case 'IBN': + $method = Setup::O_BINKP; + + switch ($x->count()) { + case 1: + $port = 24554; + break; + case 2: + $port = $x->get(1); + break; + case 3: + $address = $x->get(1); + $port = $x->get(2); + } + break; + + case 'ITN': + $method = Setup::O_EMSI; + + switch ($x->count()) { + case 1: + $port = 60179; + break; + case 2: + $port = $x->get(1); + break; + case 3: + $address = $x->get(1); + $port = $x->get(2); + } + break; + + // Ignore + case 'ZEC': + case 'REC': + case 'MO': + case 'CM': + break; + + default: + Log::debug(sprintf('%s: - Not configured to handle flag [%s]',self::LOGKEY,$x->first())); + continue 2; + } + + if ($method && ($ao->role != DomainController::NODE_PVT)) { + $so->mailer_type = $method; + $so->mailer_address = $address; + $so->mailer_port = $port; + } + } + // Save the system record $so->save(); @@ -244,7 +345,7 @@ class NodelistImport implements ShouldQueue if ($ao->role == DomainController::NODE_HC) $hub_id = $ao->id; - $this->no->addresses()->attach($ao,['role'=>$ao->role]); + $no->addresses()->attach($ao,['role'=>$ao->role]); } catch (\Exception $e) { Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); @@ -256,9 +357,17 @@ class NodelistImport implements ShouldQueue } } + // Remove addresses not recorded; + Address::whereIN('id',$zo->addresses->except($us->system->addresses->pluck('id')->toArray())->diff($no->addresses)->pluck('id')->toArray())->delete(); + + if (crc16(substr($tocrc,0,-3)) == $file_crc) + DB::commit(); + else + DB::rollBack(); + fclose($fh); - if ($this->deletefile and $c) + if ($this->delete_file and $c) unlink($file); Log::info(sprintf('%s:Updated %d records from %d systems',self::LOGKEY,$p,$c)); diff --git a/app/Models/System.php b/app/Models/System.php index 584f0a1..5a39045 100644 --- a/app/Models/System.php +++ b/app/Models/System.php @@ -47,6 +47,11 @@ class System extends Model return $this->hasOne(Setup::class); } + public function users() + { + return $this->belongsToMany(User::class); + } + /** * This system is the ZC for the following zones */ diff --git a/app/Traits/CRC.php b/app/Traits/CRC.php index a1ed300..1cb3104 100644 --- a/app/Traits/CRC.php +++ b/app/Traits/CRC.php @@ -4,17 +4,17 @@ namespace App\Traits; trait CRC { - private function CRC16USD_UPDATE($b,$crc): int + private function CRC16USD_UPDATE(int $b,int $crc): int { return (self::crc16usd_tab[(($crc >> 8) ^ $b) & 0xff] ^ (($crc & 0x00ff) << 8)) & 0xffff; } - private function CRC32_FINISH($crc) + private function CRC32_FINISH(int $crc) { return ~$crc & 0xffffffff; } - private function CRC32_UPDATE($b,$crc) + private function CRC32_UPDATE(int $b,int $crc) { return ((self::crc32_tab[($crc^$b) & 0xff] ^ (($crc>>8) & 0x00ffffff)) & 0xffffffff); } @@ -24,12 +24,12 @@ trait CRC return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? self::LSZ_INIT_CRC32 : self::LSZ_INIT_CRC16; } - private function LSZ_FINISH_CRC($crc) + private function LSZ_FINISH_CRC(int $crc) { return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? $this->CRC32_FINISH($crc) : $crc; } - private function LSZ_UPDATE_CRC($b,$crc) + private function LSZ_UPDATE_CRC(int $b,int $crc) { return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? $this->CRC32_UPDATE($b,$crc) : $this->CRC16USD_UPDATE($b,$crc); } diff --git a/database/migrations/2021_08_21_134156_add_phone.php b/database/migrations/2021_08_21_134156_add_phone.php new file mode 100644 index 0000000..dfb4dac --- /dev/null +++ b/database/migrations/2021_08_21_134156_add_phone.php @@ -0,0 +1,32 @@ +string('phone')->nullable(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('systems', function (Blueprint $table) { + $table->dropColumn('phone'); + }); + } +} diff --git a/resources/views/system/form-system.blade.php b/resources/views/system/form-system.blade.php index b7fbb01..13a9ac1 100644 --- a/resources/views/system/form-system.blade.php +++ b/resources/views/system/form-system.blade.php @@ -11,6 +11,7 @@

@if($o->exists) Update @else Add @endif System

+
@@ -26,6 +27,7 @@
+
@@ -39,7 +41,8 @@
-
+ +
@@ -54,6 +57,7 @@
+
@@ -69,6 +73,7 @@
+
@@ -86,6 +91,7 @@
+

Mailer Details

@@ -119,10 +125,26 @@
+ +
+
+ +
+ + + + @error('phone') + {{ $message }} + @enderror + +
+
+
+

BBS Details

@@ -161,6 +183,7 @@
+