Enhance nodelist import

This commit is contained in:
Deon George 2021-08-21 21:15:22 +10:00
parent 68a10af776
commit 8c992ad0f9
7 changed files with 210 additions and 55 deletions

View File

@ -2,10 +2,8 @@
namespace App\Console\Commands; namespace App\Console\Commands;
use Carbon\Carbon;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use App\Models\{Domain,Nodelist};
use App\Jobs\NodelistImport as Job; use App\Jobs\NodelistImport as Job;
class NodelistImport extends Command class NodelistImport extends Command
@ -16,9 +14,9 @@ class NodelistImport extends Command
* @var string * @var string
*/ */
protected $signature = 'nodelist:import' protected $signature = 'nodelist:import'
.' {domain : Domain Name}'
.' {file : Nodelist File}' .' {file : Nodelist File}'
.' {--D|delete : Delete old data for the date}'; .' {--D|delete : Delete old data for the date}'
.' {--U|unlink : Delete file after import}';
/** /**
* The console command description. * The console command description.
@ -27,16 +25,6 @@ class NodelistImport extends Command
*/ */
protected $description = 'Import Nodelist'; protected $description = 'Import Nodelist';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/** /**
* Execute the console command. * Execute the console command.
* *
@ -44,9 +32,6 @@ class NodelistImport extends Command
*/ */
public function handle() public function handle()
{ {
$do = Domain::where('name',$this->argument('domain'))->singleOrFail(); return Job::dispatchSync($this->argument('file'),$this->option('delete'),$this->option('unlink'));
$o = Nodelist::firstOrCreate(['date'=>Carbon::now(),'domain_id'=>$do->id]);
return Job::dispatchSync($do,$o,$this->argument('file'),$this->option('delete'));
} }
} }

View File

@ -257,6 +257,7 @@ class SystemController extends Controller
'name' => 'required|min:3', 'name' => 'required|min:3',
'location' => 'required|min:3', 'location' => 'required|min:3',
'sysop' => 'required|min:3', 'sysop' => 'required|min:3',
'phone' => 'nullable|regex:/^([0-9-]+)$/',
'address' => 'nullable|regex:/^(?!:\/\/)(?=.{1,255}$)((.{1,63}\.){1,127}(?![0-9]*$)[a-z0-9-]+\.?)$/i', 'address' => 'nullable|regex:/^(?!:\/\/)(?=.{1,255}$)((.{1,63}\.){1,127}(?![0-9]*$)[a-z0-9-]+\.?)$/i',
'port' => 'nullable|digits_between:2,5', 'port' => 'nullable|digits_between:2,5',
'method' => 'nullable|numeric', 'method' => 'nullable|numeric',
@ -267,7 +268,7 @@ class SystemController extends Controller
'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($o->exists ? $o->id : 0), 'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($o->exists ? $o->id : 0),
]); ]);
foreach (['name','location','sysop','address','port','active','method','notes','mailer_type','mailer_address','mailer_port','zt_id'] as $key) foreach (['name','location','sysop','phone','address','port','active','method','notes','mailer_type','mailer_address','mailer_port','zt_id'] as $key)
$o->{$key} = $request->post($key); $o->{$key} = $request->post($key);
$o->save(); $o->save();

View File

@ -2,16 +2,18 @@
namespace App\Jobs; namespace App\Jobs;
use Carbon\Carbon;
use Illuminate\Bus\Queueable; use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels; use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use App\Http\Controllers\DomainController; use App\Http\Controllers\DomainController;
use App\Models\{Address,Domain,Nodelist,System,Zone}; use App\Models\{Address,Domain,Nodelist,Setup,System,Zone};
use App\Traits\Import as ImportTrait; use App\Traits\Import as ImportTrait;
class NodelistImport implements ShouldQueue class NodelistImport implements ShouldQueue
@ -21,27 +23,23 @@ class NodelistImport implements ShouldQueue
protected const LOGKEY = 'JIN'; protected const LOGKEY = 'JIN';
private const importkey = 'nodelist'; private const importkey = 'nodelist';
private string $file; private string $file;
private Domain $do; private bool $delete_file;
private Nodelist $no; private bool $delete_recs;
private bool $deletefile = FALSE;
/** /**
* Import Nodelist constructor. * Import Nodelist constructor.
* *
* @param Domain $do
* @param Nodelist $no
* @param string $file * @param string $file
* @param bool $delete * @param bool $delete_recs
* @param bool $delete_file
*/ */
public function __construct(Domain $do,Nodelist $no,string $file,bool $delete=TRUE) public function __construct(string $file,bool $delete_recs=FALSE,bool $delete_file=TRUE)
{ {
$this->do = $do;
$this->no = $no;
$this->file = $file; $this->file = $file;
$this->delete_file = $delete_file;
if ($delete) $this->delete_recs = $delete_recs;
$no->addresses()->detach();
} }
/** /**
@ -52,21 +50,56 @@ class NodelistImport implements ShouldQueue
*/ */
public function handle() public function handle()
{ {
$us = Setup::findOrFail(config('app.id'));
// Get the file from the host // Get the file from the host
$file = $this->getFileFromHost('',self::importkey,$this->file); $file = $this->getFileFromHost('',self::importkey,$this->file);
$lines = $this->getFileLines($file); $lines = $this->getFileLines($file);
Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines)); Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines));
$fh = fopen($file,'r'); $fh = fopen($file,'r');
// Line 1 tells us the nodelist and the CRC
$line = stream_get_line($fh, 0, "\r\n");
$matches = [];
if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches)))
abort(500,'Nodelist invalid');
$file_crc = $matches[4];
$do = Domain::where('name',strtolower($matches[1]))->single();
if (! $do) {
Log::error(sprintf('%s:! Domain not found [%s].',static::LOGKEY,strtolower($matches[1])));
return;
}
$date = Carbon::createFromFormat('D, M d, Y H:i',$matches[2].'0:00');
if ($date->dayOfYear != $matches[3]) {
Log::error(sprintf('%s:! Nodelist date doesnt match [%d] (%d:%s).',static::LOGKEY,$matches[3],$date->dayOfYear,$date->format('Y-m-d')));
return;
}
Log::debug(sprintf('%s:Importing nodelist for [%s] dated [%s] lines.',static::LOGKEY,$do->name,$date->format('Y-m-d')));
DB::beginTransaction();
$no = Nodelist::firstOrCreate(['date'=>$date,'domain_id'=>$do->id]);
if ($this->delete_recs)
$no->addresses()->detach();
$p = $c =0; $p = $c =0;
$region = NULL; $region = NULL;
$host = NULL; $host = NULL;
$hub_id = NULL; $hub_id = NULL;
$zo = NULL; $zo = NULL;
$tocrc = '';
while (! feof($fh)) { while (! feof($fh)) {
$line = stream_get_line($fh, 0, "\r\n"); $line = stream_get_line($fh, 0, "\r\n");
$tocrc .= $line."\r\n";
// Lines beginning with a semicolon(;) are comments // Lines beginning with a semicolon(;) are comments
if (preg_match('/^;/',$line) OR ($line == chr(0x1a))) if (preg_match('/^;/',$line) OR ($line == chr(0x1a)))
@ -92,7 +125,7 @@ class NodelistImport implements ShouldQueue
Zone::unguard(); Zone::unguard();
$zo = Zone::firstOrNew([ $zo = Zone::firstOrNew([
'zone_id'=>$zone, 'zone_id'=>$zone,
'domain_id'=>$this->do->id, 'domain_id'=>$do->id,
'active'=>TRUE, 'active'=>TRUE,
]); ]);
Zone::reguard(); Zone::reguard();
@ -159,6 +192,7 @@ class NodelistImport implements ShouldQueue
Address::unguard(); Address::unguard();
$ao = Address::firstOrNew([ $ao = Address::firstOrNew([
'zone_id' => $zo->id,
'region_id' => $region, 'region_id' => $region,
'host_id' => $host, 'host_id' => $host,
'node_id' => $node, 'node_id' => $node,
@ -177,12 +211,13 @@ class NodelistImport implements ShouldQueue
$sysop = trim(str_replace('_',' ',$fields[4])); $sysop = trim(str_replace('_',' ',$fields[4]));
$system = trim(str_replace('_',' ',$fields[2])); $system = trim(str_replace('_',' ',$fields[2]));
$location = trim(str_replace('_',' ',$fields[3]));
// Get the System // Get the System
if ($ao->system_id && (($ao->system->sysop === $sysop) || ($ao->system->name === $system))) { if ($ao->system_id && (($ao->system->sysop === $sysop) || ($ao->system->name === $system))) {
$so = $ao->system; $so = $ao->system;
// Dont change the system details if a user exists here, or its us
if ((! $so->users->count()) && ($so->id != $us->system_id)) {
// If the sysop name is different // If the sysop name is different
if ($so->sysop !== $sysop) { if ($so->sysop !== $sysop) {
Log::debug(sprintf('%s:Sysop Name changed for BBS [%s:%s] from [%s] to [%s]', Log::debug(sprintf('%s:Sysop Name changed for BBS [%s:%s] from [%s] to [%s]',
@ -197,6 +232,7 @@ class NodelistImport implements ShouldQueue
$so->name = $system; $so->name = $system;
} }
}
// We'll search and see if we already have that system // We'll search and see if we already have that system
} else { } else {
@ -214,10 +250,11 @@ class NodelistImport implements ShouldQueue
$so->active = TRUE; $so->active = TRUE;
if (! $so->exists) if (! $so->exists)
$so->notes = sprintf('Created by Nodelist Import: %d',$this->no->id); $so->notes = sprintf('Created by Nodelist Import: %d',$no->id);
} }
$so->location = $location; $so->phone = $fields[5] != '-Unpublished-' ? $fields[5] : NULL;
$so->location = trim(str_replace('_',' ',$fields[3]));
/* /*
if (! in_array($fields[5],['-Unpublished-'])) if (! in_array($fields[5],['-Unpublished-']))
$so->phone = $fields[5]; $so->phone = $fields[5];
@ -225,6 +262,70 @@ class NodelistImport implements ShouldQueue
$so->baud = $fields[6]; $so->baud = $fields[6];
*/ */
// Flags
$method = NULL;
$address = '';
$port = '';
for ($i=7;$i<count($fields);$i++) {
$x = Str::of($fields[$i])->split('/:/');
switch ($x->first()) {
// address
case 'INA':
$address = $x->get(1);
break;
case 'IBN':
$method = Setup::O_BINKP;
switch ($x->count()) {
case 1:
$port = 24554;
break;
case 2:
$port = $x->get(1);
break;
case 3:
$address = $x->get(1);
$port = $x->get(2);
}
break;
case 'ITN':
$method = Setup::O_EMSI;
switch ($x->count()) {
case 1:
$port = 60179;
break;
case 2:
$port = $x->get(1);
break;
case 3:
$address = $x->get(1);
$port = $x->get(2);
}
break;
// Ignore
case 'ZEC':
case 'REC':
case 'MO':
case 'CM':
break;
default:
Log::debug(sprintf('%s: - Not configured to handle flag [%s]',self::LOGKEY,$x->first()));
continue 2;
}
if ($method && ($ao->role != DomainController::NODE_PVT)) {
$so->mailer_type = $method;
$so->mailer_address = $address;
$so->mailer_port = $port;
}
}
// Save the system record // Save the system record
$so->save(); $so->save();
@ -244,7 +345,7 @@ class NodelistImport implements ShouldQueue
if ($ao->role == DomainController::NODE_HC) if ($ao->role == DomainController::NODE_HC)
$hub_id = $ao->id; $hub_id = $ao->id;
$this->no->addresses()->attach($ao,['role'=>$ao->role]); $no->addresses()->attach($ao,['role'=>$ao->role]);
} catch (\Exception $e) { } catch (\Exception $e) {
Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
@ -256,9 +357,17 @@ class NodelistImport implements ShouldQueue
} }
} }
// Remove addresses not recorded;
Address::whereIN('id',$zo->addresses->except($us->system->addresses->pluck('id')->toArray())->diff($no->addresses)->pluck('id')->toArray())->delete();
if (crc16(substr($tocrc,0,-3)) == $file_crc)
DB::commit();
else
DB::rollBack();
fclose($fh); fclose($fh);
if ($this->deletefile and $c) if ($this->delete_file and $c)
unlink($file); unlink($file);
Log::info(sprintf('%s:Updated %d records from %d systems',self::LOGKEY,$p,$c)); Log::info(sprintf('%s:Updated %d records from %d systems',self::LOGKEY,$p,$c));

View File

@ -47,6 +47,11 @@ class System extends Model
return $this->hasOne(Setup::class); return $this->hasOne(Setup::class);
} }
public function users()
{
return $this->belongsToMany(User::class);
}
/** /**
* This system is the ZC for the following zones * This system is the ZC for the following zones
*/ */

View File

@ -4,17 +4,17 @@ namespace App\Traits;
trait CRC trait CRC
{ {
private function CRC16USD_UPDATE($b,$crc): int private function CRC16USD_UPDATE(int $b,int $crc): int
{ {
return (self::crc16usd_tab[(($crc >> 8) ^ $b) & 0xff] ^ (($crc & 0x00ff) << 8)) & 0xffff; return (self::crc16usd_tab[(($crc >> 8) ^ $b) & 0xff] ^ (($crc & 0x00ff) << 8)) & 0xffff;
} }
private function CRC32_FINISH($crc) private function CRC32_FINISH(int $crc)
{ {
return ~$crc & 0xffffffff; return ~$crc & 0xffffffff;
} }
private function CRC32_UPDATE($b,$crc) private function CRC32_UPDATE(int $b,int $crc)
{ {
return ((self::crc32_tab[($crc^$b) & 0xff] ^ (($crc>>8) & 0x00ffffff)) & 0xffffffff); return ((self::crc32_tab[($crc^$b) & 0xff] ^ (($crc>>8) & 0x00ffffff)) & 0xffffffff);
} }
@ -24,12 +24,12 @@ trait CRC
return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? self::LSZ_INIT_CRC32 : self::LSZ_INIT_CRC16; return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? self::LSZ_INIT_CRC32 : self::LSZ_INIT_CRC16;
} }
private function LSZ_FINISH_CRC($crc) private function LSZ_FINISH_CRC(int $crc)
{ {
return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? $this->CRC32_FINISH($crc) : $crc; return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? $this->CRC32_FINISH($crc) : $crc;
} }
private function LSZ_UPDATE_CRC($b,$crc) private function LSZ_UPDATE_CRC(int $b,int $crc)
{ {
return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? $this->CRC32_UPDATE($b,$crc) : $this->CRC16USD_UPDATE($b,$crc); return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? $this->CRC32_UPDATE($b,$crc) : $this->CRC16USD_UPDATE($b,$crc);
} }

View File

@ -0,0 +1,32 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class AddPhone extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::table('systems', function (Blueprint $table) {
$table->string('phone')->nullable();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('systems', function (Blueprint $table) {
$table->dropColumn('phone');
});
}
}

View File

@ -11,6 +11,7 @@
<h2 class="cap">@if($o->exists) Update @else Add @endif System</h2> <h2 class="cap">@if($o->exists) Update @else Add @endif System</h2>
<div class="row"> <div class="row">
<!-- Name -->
<div class="col-4"> <div class="col-4">
<label for="name" class="form-label">Name</label> <label for="name" class="form-label">Name</label>
<div class="input-group has-validation"> <div class="input-group has-validation">
@ -26,6 +27,7 @@
</div> </div>
</div> </div>
<!-- Active -->
<div class="col-2"> <div class="col-2">
<label for="active" class="form-label">Active</label> <label for="active" class="form-label">Active</label>
<div class="input-group"> <div class="input-group">
@ -39,7 +41,8 @@
</div> </div>
</div> </div>
<div class="col-3"> <!-- ZeroTier ID -->
<div class="offset-3 col-3">
<label for="zt_id" class="form-label">ZeroTier ID</label> <label for="zt_id" class="form-label">ZeroTier ID</label>
<div class="input-group has-validation"> <div class="input-group has-validation">
<span class="input-group-text"><i class="bi bi-shield-lock-fill"></i></span> <span class="input-group-text"><i class="bi bi-shield-lock-fill"></i></span>
@ -54,6 +57,7 @@
</div> </div>
<div class="row"> <div class="row">
<!-- Sysop -->
<div class="col-4"> <div class="col-4">
<label for="sysop" class="form-label">Sysop</label> <label for="sysop" class="form-label">Sysop</label>
<div class="input-group has-validation"> <div class="input-group has-validation">
@ -69,6 +73,7 @@
</div> </div>
</div> </div>
<!-- Location -->
<div class="col-8"> <div class="col-8">
<label for="location" class="form-label">Location</label> <label for="location" class="form-label">Location</label>
<div class="input-group has-validation"> <div class="input-group has-validation">
@ -86,6 +91,7 @@
</div> </div>
<div class="row"> <div class="row">
<!-- Mailer Details -->
<div class="col-12"> <div class="col-12">
<h4>Mailer Details</h4> <h4>Mailer Details</h4>
@ -119,10 +125,26 @@
</div> </div>
</div> </div>
</div> </div>
<div class="pt-0 row">
<div class="col-4">
<label for="phone" class="form-label">Phone</label>
<div class="input-group has-validation">
<span class="input-group-text"><i class="bi bi-telephone-fill"></i></span>
<input type="text" class="form-control @error('phone') is-invalid @enderror" id="phone" placeholder="Phone" name="phone" value="{{ old('phone',$o->phone) }}" @cannot('admin',$o)disabled @endcannot>
<span class="invalid-feedback" role="alert">
@error('phone')
{{ $message }}
@enderror
</span>
</div>
</div>
</div>
</div> </div>
</div> </div>
<div class="row"> <div class="row">
<!-- BBS Details -->
<div class="col-12"> <div class="col-12">
<h4>BBS Details</h4> <h4>BBS Details</h4>
@ -161,6 +183,7 @@
</div> </div>
<div class="row"> <div class="row">
<!-- Notes -->
<div class="col-12"> <div class="col-12">
<label for="notes" class="form-label">Notes</label> <label for="notes" class="form-label">Notes</label>
<textarea class="form-control" rows=3 name="notes" placeholder="Notes..." @cannot('admin',$o)disabled @endcannot>{{ old('notes',$o->notes) }}</textarea> <textarea class="form-control" rows=3 name="notes" placeholder="Notes..." @cannot('admin',$o)disabled @endcannot>{{ old('notes',$o->notes) }}</textarea>