562 lines
16 KiB
PHP
562 lines
16 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use Carbon\Carbon;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Arr;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Illuminate\Support\Str;
|
|
|
|
use App\Models\{Address,Domain,File,Mailer,Nodelist,Setup,System,SystemZone,User,Zone};
|
|
use App\Traits\Import as ImportTrait;
|
|
|
|
class NodelistImport implements ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
use ImportTrait;
|
|
|
|
protected const LOGKEY = 'JNI';
|
|
private const importkey = 'nodelist';
|
|
|
|
public const QUEUE = 'nodelist';
|
|
|
|
private File|string $file;
|
|
private ?string $domain;
|
|
private bool $delete_file;
|
|
private bool $delete_recs;
|
|
private bool $testmode;
|
|
private bool $ignore_crc;
|
|
|
|
/**
|
|
* Import Nodelist constructor.
|
|
*
|
|
* @param File|string $file
|
|
* @param string|null $domain
|
|
* @param bool $delete_recs
|
|
* @param bool $delete_file
|
|
* @param bool $test
|
|
* @param bool $ignore_crc
|
|
*/
|
|
public function __construct(File|string $file,string $domain=NULL,bool $delete_recs=FALSE,bool $delete_file=TRUE,bool $test=FALSE,bool $ignore_crc = FALSE)
|
|
{
|
|
$this->file = $file;
|
|
$this->domain = $domain;
|
|
$this->delete_file = $delete_file;
|
|
$this->delete_recs = $delete_recs;
|
|
$this->testmode = $test;
|
|
$this->ignore_crc = $ignore_crc;
|
|
|
|
$this->onQueue(self::QUEUE);
|
|
}
|
|
|
|
public function __get($key): mixed
|
|
{
|
|
switch ($key) {
|
|
case 'jobname':
|
|
return sprintf('%s-%s',$this->domain?->name,is_object($this->file) ? $this->file->name : $this->file);
|
|
|
|
default:
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Execute the job.
|
|
*
|
|
* @return void
|
|
* @throws \Exception
|
|
*/
|
|
public function handle()
|
|
{
|
|
$our_systems = SystemZone::select('system_id')
|
|
->get()
|
|
->pluck('system')
|
|
->flatten()
|
|
->pluck('addresses')
|
|
->flatten()
|
|
->filter(function($item) { return $item->active; })
|
|
->pluck('id');
|
|
|
|
$our_users = User::with(['systems.addresses'])
|
|
->get()
|
|
->pluck('systems')
|
|
->flatten()
|
|
->pluck('addresses')
|
|
->flatten()
|
|
->filter(function($item) { return $item->active; })
|
|
->pluck('id')
|
|
->diff($our_systems);
|
|
|
|
$our_addresses = our_address()->pluck('id');
|
|
|
|
// Get the file from the host
|
|
$file = $this->getFileFromHost(self::importkey,$this->file);
|
|
Log::debug(sprintf('%s:+ Loading file [%s] (%s).',static::LOGKEY,$file,getcwd()));
|
|
$lines = $this->getFileLines($file);
|
|
Log::debug(sprintf('%s:- Processing [%d] lines.',static::LOGKEY,$lines));
|
|
|
|
$fh = NULL;
|
|
$z = $this->openFile($file,$fh);
|
|
|
|
// Line 1 tells us the nodelist and the CRC
|
|
$line = stream_get_line($fh,0,"\r\n");
|
|
|
|
$matches = [];
|
|
if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) {
|
|
Log::error(sprintf('%s:! Nodelist file [%d] is not valid?',self::LOGKEY,$this->file->id),['m'=>$matches,'l'=>$line]);
|
|
|
|
throw new \Exception('Invalid nodelist for file: '.$this->file->id);
|
|
}
|
|
|
|
$file_crc = (int)$matches[4];
|
|
$do = Domain::where('name',strtolower($matches[1] ?: $this->domain))->single();
|
|
|
|
if (! $do) {
|
|
Log::error(sprintf('%s:! Domain not found [%s].',static::LOGKEY,strtolower($matches[1] ?: $this->domain)));
|
|
|
|
throw new \Exception('Nodelist Domain not found: '.$this->file);
|
|
}
|
|
|
|
$date = Carbon::createFromFormat('D, M d, Y H:i',$matches[2].'0:00');
|
|
|
|
if ($date->dayOfYear != $matches[3]) {
|
|
Log::error(sprintf('%s:! Nodelist date doesnt match [%d] (%d:%s).',static::LOGKEY,$matches[3],$date->dayOfYear,$date->format('Y-m-d')));
|
|
|
|
throw new \Exception('Nodelist date doesnt match for file: '.$this->file->id);
|
|
}
|
|
|
|
Log::alert(sprintf('%s:- Importing nodelist for [%s] dated [%s].',static::LOGKEY,$do->name,$date->format('Y-m-d')));
|
|
|
|
DB::beginTransaction();
|
|
$no = Nodelist::firstOrCreate(['date'=>$date,'domain_id'=>$do->id]);
|
|
|
|
if ($this->delete_recs)
|
|
$no->addresses()->detach();
|
|
elseif ($no->addresses->count()) {
|
|
Log::error($x=sprintf('%s:! Nodelist [%s] for [%s] has existing records [%d]',self::LOGKEY,$date,$do->name,$no->addresses->count()));
|
|
|
|
throw new \Exception($x);
|
|
}
|
|
|
|
$p = $c = 0;
|
|
|
|
$region = NULL;
|
|
$host = NULL;
|
|
$hub_id = NULL;
|
|
$zo = NULL;
|
|
$tocrc = '';
|
|
$mailer_binkp = Mailer::where('name','BINKP')->singleOrFail();
|
|
$mailer_emsi = Mailer::where('name','EMSI')->singleOrFail();
|
|
|
|
while (! feof($fh)) {
|
|
$line = stream_get_line($fh,0,"\r\n");
|
|
$tocrc .= $line."\r\n";
|
|
|
|
// Lines beginning with a semicolon(;) are comments
|
|
if ((! $line) OR preg_match('/^;/',$line) OR ($line === chr(0x1a)))
|
|
continue;
|
|
|
|
// Remove any embedded CR and BOM
|
|
$line = str_replace("\r",'',$line);
|
|
$line = preg_replace('/^\x{feff}/u','',$line);
|
|
$c++;
|
|
|
|
Log::debug(sprintf('%s:%s',self::LOGKEY,$line));
|
|
|
|
$fields = str_getcsv(trim($line));
|
|
|
|
// First field is either zone,region,host,hub,down,pvt (or blank)
|
|
if ($fields[0] AND ! in_array($fields[0],Nodelist::definitions)) {
|
|
Log::error(sprintf('%s:! Invalid field zero [%s] - IGNORING record (%s)',self::LOGKEY,$fields[0],$line));
|
|
continue;
|
|
}
|
|
|
|
$node = 0;
|
|
$role = NULL;
|
|
|
|
switch ($fields[0]) {
|
|
case 'Zone':
|
|
$zone = (int)$fields[1];
|
|
Zone::unguard();
|
|
$zo = Zone::firstOrNew([
|
|
'zone_id'=>$zone,
|
|
'domain_id'=>$do->id,
|
|
'active'=>TRUE,
|
|
]);
|
|
Zone::reguard();
|
|
|
|
$region = 0;
|
|
$host = 0;
|
|
$hub_id = NULL;
|
|
|
|
break;
|
|
|
|
case 'Region':
|
|
$region = (int)$fields[1];
|
|
$host = (int)$fields[1];
|
|
$hub_id = NULL;
|
|
|
|
break;
|
|
|
|
case 'Host':
|
|
$host = (int)$fields[1];
|
|
$hub_id = NULL;
|
|
|
|
break;
|
|
|
|
case 'Hub':
|
|
$node = (int)$fields[1];
|
|
$role = Address::NODE_HC;
|
|
|
|
break;
|
|
|
|
case 'Pvt':
|
|
$node = (int)$fields[1];
|
|
$role = Address::NODE_PVT;
|
|
|
|
break;
|
|
|
|
case 'Hold':
|
|
$node = (int)$fields[1];
|
|
$role = Address::NODE_HOLD;
|
|
|
|
break;
|
|
|
|
case 'Down':
|
|
$node = (int)$fields[1];
|
|
$role = Address::NODE_DOWN;
|
|
|
|
break;
|
|
|
|
case '':
|
|
$node = $fields[1];
|
|
break;
|
|
|
|
default:
|
|
Log::error(sprintf('%s:! Unhandled first field [%s]',self::LOGKEY,$fields[0]));
|
|
continue 2;
|
|
}
|
|
|
|
if (! $zone) {
|
|
Log::error(sprintf('%s:! Zone NOT set, ignoring record...',self::LOGKEY));
|
|
continue;
|
|
}
|
|
|
|
Address::unguard();
|
|
$ao = Address::firstOrNew([
|
|
'zone_id' => $zo->id,
|
|
'host_id' => $host,
|
|
'node_id' => $node,
|
|
'point_id' => 0,
|
|
'active' => TRUE,
|
|
]);
|
|
Address::reguard();
|
|
|
|
if ($ao->region_id && ($ao->region_id !== $region))
|
|
Log::alert(sprintf('%s:%% Address [%s] changing regions [%d->%d]',self::LOGKEY,$ao->ftn,$ao->region_id,$region));
|
|
|
|
$ao->region_id = $region;
|
|
$ao->role = $role;
|
|
$ao->hub_id = $hub_id;
|
|
|
|
if ($ao->exists) {
|
|
Log::info(sprintf('%s:- Processing existing address [%s] (%d)',self::LOGKEY,$ao->ftn,$ao->id));
|
|
|
|
// If the address is linked to a user's system, or our system, we'll not process it any further
|
|
$skip = FALSE;
|
|
if ($our_addresses->contains($ao->id)) {
|
|
Log::info(sprintf('%s:! Ignoring updating an address belonging to me',self::LOGKEY));
|
|
$skip = TRUE;
|
|
}
|
|
|
|
if ($our_systems->contains($ao->id)) {
|
|
Log::info(sprintf('%s:! Ignoring a system managed by this site',self::LOGKEY));
|
|
$skip = TRUE;
|
|
}
|
|
|
|
if ($our_users->contains($ao->id)) {
|
|
Log::info(sprintf('%s:! Ignoring a system managed by a user',self::LOGKEY));
|
|
$skip = TRUE;
|
|
}
|
|
|
|
if ($skip) {
|
|
$no->addresses()->attach($ao,['role'=>$ao->role]);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
$sysop = trim(str_replace('_',' ',$fields[4]));
|
|
$system = trim(str_replace('_',' ',$fields[2]));
|
|
|
|
// Flags
|
|
$methods = collect();
|
|
$address = '';
|
|
for ($i=7;$i<count($fields);$i++) {
|
|
$x = Str::of($fields[$i])->split('/:/');
|
|
|
|
switch ($x->first()) {
|
|
// Address
|
|
case 'INA':
|
|
$address = $x->get(1);
|
|
break;
|
|
|
|
// BINKP
|
|
case 'IBN':
|
|
case 'ITN':
|
|
if ($x->first() === 'IBN') {
|
|
$dport = 24554;
|
|
$method = $mailer_binkp->id;
|
|
|
|
} else {
|
|
$dport = 60179;
|
|
$method = $mailer_emsi->id;
|
|
}
|
|
|
|
switch ($x->count()) {
|
|
case 1:
|
|
$methods->put($method,['port'=>$dport]);
|
|
break;
|
|
|
|
case 2:
|
|
$mp = is_numeric($x->get(1)) ? (int)$x->get(1) : $dport;
|
|
$ma = is_numeric($xx=$x->get(1)) ? NULL : $xx;
|
|
|
|
if ($ma && ($ma !== $address))
|
|
$methods->put($method,['address'=>$ma,'port'=>$mp]);
|
|
else
|
|
$methods->put($method,['port'=>$mp]);
|
|
|
|
break;
|
|
|
|
case 3:
|
|
$mp = (int)$x->get(2);
|
|
$ma = $x->get(1);
|
|
|
|
if ($ma && ($ma !== $address))
|
|
$methods->put($method,['address'=>$ma,'port'=>$mp]);
|
|
else
|
|
$methods->put($method,['port'=>$mp]);
|
|
|
|
break;
|
|
}
|
|
|
|
break;
|
|
|
|
// Ignore
|
|
case 'ZEC':
|
|
case 'REC':
|
|
case 'MO':
|
|
case 'CM':
|
|
break;
|
|
|
|
default:
|
|
Log::debug(sprintf('%s:! Not configured to handle flag [%s]',self::LOGKEY,$x->first()));
|
|
continue 2;
|
|
}
|
|
}
|
|
|
|
// Get the System
|
|
// If we are a zone/region record, then the system may change
|
|
switch ($role) {
|
|
case Address::NODE_ZC:
|
|
case Address::NODE_RC:
|
|
case Address::NODE_NC:
|
|
$so = ($x=System::distinct('systems.*')
|
|
->join('mailer_system',['mailer_system.system_id'=>'systems.id'])
|
|
->where('sysop',$sysop)
|
|
->where(function($query) use ($address,$methods) {
|
|
return $query->where('systems.address',$address)
|
|
->when($methods->pluck('address')->filter()->count(),function ($query) use ($methods) {
|
|
return $query->whereIN('mailer_system.address',$methods->pluck('address'));
|
|
});
|
|
})
|
|
->when($methods->pluck('port')->filter()->count(),function ($query) use ($methods) {
|
|
return $query->whereIN('mailer_system.port',$methods->pluck('port'));
|
|
}))
|
|
->single();
|
|
|
|
if (! $so) {
|
|
Log::info(sprintf('%s:= New System for ZC/RC/NC [%s] - System [%s] Sysop [%s]',self::LOGKEY,$ao->ftn,$system,$sysop));
|
|
$so = new System;
|
|
$so->sysop = $sysop;
|
|
$so->name = $system;
|
|
$so->address = $address;
|
|
$so->location = 'TBA';
|
|
$so->notes = sprintf('Created by Nodelist Import: %d',$no->id);
|
|
$so->active = TRUE;
|
|
|
|
$so->save();
|
|
}
|
|
|
|
// If the address exists, but it was discovered, assign it to this new host
|
|
if ($ao->system_id && ($ao->system_id !== $so->id) && ($ao->system->name === System::default)) {
|
|
Log::info(sprintf('%s:= Re-assigning discovered address to [%s]',self::LOGKEY,$so->id));
|
|
|
|
} elseif (! $ao->system_id) {
|
|
Log::alert(sprintf('%s:%% [%s] address not assigned to any system [%s:%s]',self::LOGKEY,$ao->ftn,$system,$sysop));
|
|
|
|
} elseif ($ao->system_id !== $so->id) {
|
|
Log::alert(sprintf('%s:%% [%s] hosted by new system [%s:%s] (was %s:%s)',self::LOGKEY,$ao->ftn,$system,$sysop,$ao->system->name,$ao->system->sysop));
|
|
$ao->active = FALSE;
|
|
$ao->save();
|
|
|
|
$ao = $ao->replicate();
|
|
$ao->active = TRUE;
|
|
}
|
|
|
|
$ao->system_id = $so->id;
|
|
$ao->save();
|
|
|
|
$ao->load('system');
|
|
}
|
|
|
|
if ($ao->system_id && ((($ao->system->sysop === $sysop) || ($ao->system->name === $system)) && (($ao->system->address === $address) || ($methods->pluck('address')->search($address) !== FALSE)))) {
|
|
Log::info(sprintf('%s:= Matched [%s] to existing system [%s] with address [%s]',self::LOGKEY,$ao->ftn,$ao->system->name,$ao->system->address));
|
|
$so = $ao->system;
|
|
|
|
// If the sysop name is different
|
|
if ($so->sysop !== $sysop) {
|
|
Log::alert(sprintf('%s:! Sysop Name changed for BBS [%s:%s] from [%s] to [%s]',
|
|
self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop));
|
|
|
|
$so->sysop = $sysop;
|
|
|
|
// We have the same name has changed (except for ZC/RC addresses)
|
|
} elseif (($so->name !== $system) && (! ((Address::NODE_ZC|Address::NODE_RC|Address::NODE_NC) & $ao->role_id))) {
|
|
Log::alert(sprintf('%s:! System Name changed for BBS [%s:%s] to [%s]',
|
|
self::LOGKEY,$so->id,$so->name,$system));
|
|
|
|
$so->name = $system;
|
|
}
|
|
|
|
// We'll search and see if we already have that system
|
|
} else {
|
|
Log::debug(sprintf('%s:- Looking for existing system [%s] with address [%s]',self::LOGKEY,$system,$address));
|
|
// If we dont have $address/port
|
|
$so = NULL;
|
|
|
|
if ($address) {
|
|
$so = System::select('systems.*')
|
|
->distinct()
|
|
->join('mailer_system',['mailer_system.system_id'=>'systems.id'])
|
|
->where(function($query) use ($address) {
|
|
return $query->where('systems.address',$address)
|
|
->orWhere('mailer_system.address',$address);
|
|
})
|
|
->single();
|
|
}
|
|
|
|
if (! $so) {
|
|
Log::debug(sprintf('%s:- Didnt match on system address, looking for System [%s] AND Sysop [%s]',self::LOGKEY,$system,$sysop));
|
|
|
|
$so = System::where('name',$system)
|
|
->where('sysop',$sysop)
|
|
->firstOrNew();
|
|
|
|
} else {
|
|
Log::debug(sprintf('%s:- Matched on system [%d] address',self::LOGKEY,$so->id));
|
|
}
|
|
|
|
if ($so->exists)
|
|
Log::info(sprintf('%s:= Linking address [%d:%d/%d] to [%s:%s]',self::LOGKEY,$zo->zone_id,$ao->host_id,$ao->node_id,$so->id,$so->name));
|
|
else
|
|
Log::info(sprintf('%s:= New System [%s] with FTN [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id));
|
|
|
|
$so->name = $system;
|
|
$so->sysop = $sysop;
|
|
$so->active = TRUE;
|
|
|
|
if (! $so->exists)
|
|
$so->notes = sprintf('Created by Nodelist Import: %d',$no->id);
|
|
}
|
|
|
|
$so->phone = $fields[5] != '-Unpublished-' ? $fields[5] : NULL;
|
|
$so->location = trim(str_replace('_',' ',$fields[3]));
|
|
/*
|
|
if (! in_array($fields[5],['-Unpublished-']))
|
|
$so->phone = $fields[5];
|
|
|
|
$so->baud = $fields[6];
|
|
*/
|
|
|
|
// Save the system record
|
|
try {
|
|
$so->save();
|
|
|
|
} catch (\Exception $e) {
|
|
Log::error(sprintf('%s:! Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
|
|
|
|
DB::rollBack();
|
|
throw new \Exception($e->getMessage());
|
|
}
|
|
|
|
// @todo This should be a bit test ($ao->rule & Address::NODE_PVT)?
|
|
if ($methods->count() && ($ao->role != Address::NODE_PVT)) {
|
|
$methods->transform(function($item) { $item['active'] = Arr::get($item,'active',TRUE); return $item; });
|
|
$so->mailers()->sync($methods);
|
|
}
|
|
|
|
// If our zone didnt exist, we'll create it with this system
|
|
if (! $zo->exists) {
|
|
$zo->system_id = $so->id;
|
|
$zo->save();
|
|
}
|
|
|
|
$ao->zone_id = $zo->id;
|
|
$ao->validated = TRUE;
|
|
|
|
if ($ao->getDirty())
|
|
$p++;
|
|
|
|
try {
|
|
$so->addresses()->save($ao);
|
|
|
|
if ($ao->role_id === Address::NODE_HC)
|
|
$hub_id = $ao->id;
|
|
|
|
$no->addresses()->attach($ao,['role'=>$ao->role]);
|
|
|
|
} catch (\Exception $e) {
|
|
Log::error(sprintf('%s:! Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
|
|
|
|
DB::rollBack();
|
|
throw new \Exception($e->getMessage());
|
|
}
|
|
|
|
if (! ($c % 100)) {
|
|
Log::notice(sprintf('%s:= Processed [%s] records',self::LOGKEY,$c),['memory'=>memory_get_usage(TRUE)]);
|
|
}
|
|
}
|
|
|
|
// Remove addresses not recorded;
|
|
$no->load('addresses');
|
|
$remove = $zo->addresses->diff($no->addresses)->except($our_systems->toArray())->except($our_users->toArray());
|
|
Log::alert(sprintf('%s:%% Deleting [%d] addresses [%s]',self::LOGKEY,$remove->count(),$remove->pluck('ftn2d')->join(',')));
|
|
Address::whereIN('id',$remove->pluck('id')->toArray())->update(['active'=>FALSE]);
|
|
Address::whereIN('id',$remove->pluck('id')->toArray())->delete();
|
|
|
|
$crc = crc16(substr($tocrc,0,-3));
|
|
|
|
if ((! $this->testmode) && ($this->ignore_crc || ($crc === $file_crc))) {
|
|
Log::info(sprintf('%s:= Committing nodelist',self::LOGKEY));
|
|
DB::commit();
|
|
|
|
} else {
|
|
Log::error(sprintf('%s:! Rolling back nodelist, CRC doesnt match [%s](%s) or test mode',self::LOGKEY,$crc,$file_crc));
|
|
DB::rollBack();
|
|
}
|
|
|
|
fclose($fh);
|
|
|
|
if ($this->delete_file and $c)
|
|
unlink($file);
|
|
|
|
Log::info(sprintf('%s:= Updated %d records from %d systems',self::LOGKEY,$p,$c));
|
|
}
|
|
} |