Import nodelists

This commit is contained in:
Deon George 2021-06-25 21:31:57 +10:00
parent 1f04f8374e
commit 64215ebcea
11 changed files with 446 additions and 149 deletions

View File

@ -2,8 +2,11 @@
namespace App\Console\Commands;
use Carbon\Carbon;
use Illuminate\Console\Command;
use App\Models\{Flag,Node,Zone};
use App\Models\{Domain,Nodelist};
use App\Jobs\ImportNodelist as Job;
class ImportNodelist extends Command
{
@ -12,7 +15,10 @@ class ImportNodelist extends Command
*
* @var string
*/
protected $signature = 'import:nodelist {file : Nodelist File}';
protected $signature = 'import:nodelist'
.' {domain : Domain Name}'
.' {file : Nodelist File}'
.' {--D|delete : Delete old data for the date}';
/**
* The console command description.
@ -38,135 +44,9 @@ class ImportNodelist extends Command
*/
public function handle()
{
$file = $this->argument('file');
$lines = intval(exec("wc -l '$file'"));
$do = Domain::where('name',$this->argument('domain'))->singleOrFail();
$o = Nodelist::firstOrCreate(['date'=>Carbon::now(),'domain_id'=>$do->id]);
$file = fopen($file,"r");
$bar = $this->output->createProgressBar($lines);
$bar->setFormat("%current%/%max% [%bar%] %percent:3s%% (%memory%) (%remaining%) ");
$bar->setRedrawFrequency(100);
$bar->start();
$zone = $region = $host = NULL;
while (! feof($file)) {
$line = stream_get_line($file, 0, "\r\n");
// Lines beginning with a semicolon(;) are comments
if (preg_match('/^;/',$line) OR ($line == chr(0x1A)))
continue;
$fields = explode(',',$line);
$o = new Node();
// First field is either zone,region,host,hub,down,pvt (or blank)
if ($fields[0] AND ! in_array($fields[0],['Zone','Region','Host','Hub','Pvt','Down']))
{
$this->error(sprintf('Invalid field zero [%s] - IGNORING record (%s)',$fields[0],$line));
$bar->advance();
continue;
}
$status = NULL;
$node = 0;
switch ($fields[0])
{
case 'Zone': $zone = $fields[1];
$zo = Zone::firstOrCreate([
'id'=>$zone
]);
break;
case 'Host': $host = $fields[1]; break;
case 'Region': $region = $fields[1]; break;
case 'Down':
case 'Pvt': $status = $fields[0];
case '':
$node = $fields[1];
break;
default:
$this->error(sprintf('Unhandled first field [%s]',$fields[0]));
$bar->advance();
continue 2;
}
if (! $zone)
{
$this->error('Zone NOT set, ignoring record...');
$bar->advance();
continue;
}
if (! $host)
{
$host = $zone;
}
if (! $region)
{
$region = 0 ;
}
$o = Node::firstOrNew([
'zone_id'=>$zo->id,
'host_id'=>$host,
'node_id'=>$node,
]);
$o->region_id = $region;
$o->system = $fields[2];
$o->location = $fields[3];
$o->sysop = $fields[4];
$o->active = TRUE;
$o->status = $status;
if (! in_array($fields[5],['-Unpublished-']))
$o->phone = $fields[5];
$o->baud = $fields[6];
$o->save();
// Fields 7 onwards are flags
$flags = collect();
for ($i=7;$i<count($fields);$i++)
{
$value = NULL;
if (preg_match('/^[A-Z]+:/',$fields[$i]))
{
list($flag,$value) = explode(':',$fields[$i],2);
} else {
$flag = $fields[$i];
}
$fo = Flag::firstOrNew(
['flag'=>$flag]
);
if (! $flag)
{
$this->warn('Ignoring blank flag on: '.$line);
continue;
}
if (! $fo->exists)
{
$fo->description = 'Auto Created on Import';
$fo->save();
}
$flags->put($fo->id,['arguments'=>$value]);
}
$o->flags()->sync($flags);
$bar->advance();
}
fclose($file);
$bar->finish();
echo "\n";
return Job::dispatchSync($do,$o,$this->argument('file'),$this->option('delete'));
}
}

231
app/Jobs/ImportNodelist.php Normal file
View File

@ -0,0 +1,231 @@
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use App\Http\Controllers\DomainController;
use App\Models\{Address,Domain,Nodelist,System,Zone};
use App\Traits\Import as ImportTrait;
class ImportNodelist implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
use ImportTrait;
protected const LOGKEY = 'JIN';
private const importkey = 'nodelist';
private string $file;
private Domain $do;
private Nodelist $no;
private bool $deletefile = FALSE;
/**
* Import Nodelist constructor.
*
* @param Domain $do
* @param Nodelist $no
* @param string $file
* @param bool $delete
*/
public function __construct(Domain $do,Nodelist $no,string $file,bool $delete=TRUE)
{
$this->do = $do;
$this->no = $no;
$this->file = $file;
if ($delete)
$no->addresses()->detach();
}
/**
* Execute the job.
*
* @return void
* @throws \Exception
*/
public function handle()
{
// Get the file from the host
$file = $this->getFileFromHost('',self::importkey,$this->file);
$lines = $this->getFileLines($file);
Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines));
$fh = fopen($file,'r');
$c =0;
$region = NULL;
$host = NULL;
$hub_id = NULL;
while (! feof($fh)) {
$line = stream_get_line($fh, 0, "\r\n");
// Lines beginning with a semicolon(;) are comments
if (preg_match('/^;/',$line) OR ($line == chr(0x1a)))
continue;
// Remove any embedded CR and BOM
$line = str_replace("\r",'',$line);
$line = preg_replace('/^\x{feff}/u','',$line);
$c++;
$fields = str_getcsv(trim($line));
// First field is either zone,region,host,hub,down,pvt (or blank)
if ($fields[0] AND ! in_array($fields[0],Nodelist::definitions)) {
Log::error(sprintf('%s:Invalid field zero [%s] - IGNORING record (%s)',self::LOGKEY,$fields[0],$line));
continue;
}
$node = 0;
switch ($fields[0]) {
case 'Zone': $zone = $fields[1];
// We ignore the Zone entries, since they are dynamically created.
$zo = Zone::firstOrCreate([
'zone_id'=>$zone,
'domain_id'=>$this->do->id,
]);
$region = 0;
$host = 0;
$hub_id = NULL;
$role = DomainController::NODE_ZC;
continue 2;
case 'Region':
$region = $fields[1];
$role = DomainController::NODE_RC;
$host = 0;
$hub_id = NULL;
break;
case 'Host':
$host = $fields[1];
$hub_id = NULL;
$role = DomainController::NODE_NC;
// We ignore the Host entries, since they are dynamically created.
continue 2;
case 'Hub':
$role = DomainController::NODE_HC;
break;
case 'Pvt':
$node = $fields[1];
$role = DomainController::NODE_PVT;
break;
case 'Down':
$node = $fields[1];
$role = DomainController::NODE_DOWN;
break;
case '':
$node = $fields[1];
break;
default:
Log::error(sprintf('%s:Unhandled first field [%s]',self::LOGKEY,$fields[0]));
continue 2;
}
if (! $zone) {
Log::error(sprintf('%s:Zone NOT set, ignoring record...',self::LOGKEY));
continue;
}
Address::unguard();
$ao = Address::firstOrNew([
'zone_id' => $zo->id,
'region_id' => $region,
'host_id' => $host,
'node_id' => $node,
'point_id' => 0,
]);
Address::reguard();
$ao->active = TRUE;
$ao->role = $role;
$ao->hub_id = $hub_id;
$role = NULL;
// Get the System
if ($ao->system_id && (($ao->system->sysop === str_replace('_',' ',$fields[4])) || ($ao->system->name !== str_replace('_',' ',$fields[2])))) {
$so = $ao->system;
// If the sysop name is different
if ($so->sysop !== str_replace('_',' ',$fields[4])) {
$so->sysop = str_replace('_',' ',$fields[4]);
$so->location = str_replace('_',' ',$fields[3]);
// We have the same name has changed.
} else {
$so->name = str_replace('_',' ',$fields[2]);
$so->location = str_replace('_',' ',$fields[3]);
}
// We'll search and see if we already have that system
} else {
$so = System::where('name',str_replace('_',' ',$fields[2]))
->where('sysop',str_replace('_',' ',$fields[4]))
->firstOrNew();
$so->name = str_replace('_',' ',$fields[2]);
$so->sysop = str_replace('_',' ',$fields[4]);
$so->location = str_replace('_',' ',$fields[3]);
$so->active = TRUE;
if (! $so->exists)
$so->notes = sprintf('Created by Nodelist Import: %d',$this->no->id);
}
/*
if (! in_array($fields[5],['-Unpublished-']))
$so->phone = $fields[5];
$so->baud = $fields[6];
*/
// Save the system record
$so->save();
try {
$so->addresses()->save($ao);
if ($role == DomainController::NODE_HC)
$hub_id = $ao->id;
$this->no->addresses()->attach($ao,['role'=>$role]);
} catch (\Exception $e) {
Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
throw new \Exception($e->getMessage());
}
if (! ($c % 100)) {
Log::notice(sprintf('%s:Processed [%s] records',self::LOGKEY,$c),['memory'=>memory_get_usage(TRUE)]);
}
}
fclose($fh);
if ($this->deletefile and $c)
unlink($file);
Log::info(sprintf('%s:Records Updated: %d',self::LOGKEY,$c));
}
}

View File

@ -59,6 +59,10 @@ class Address extends Model
return 'Host';
case DomainController::NODE_HC;
return 'Hub';
case DomainController::NODE_PVT;
return 'PVT';
case DomainController::NODE_DOWN;
return 'DOWN';
case NULL:
return 'Node';
default:

20
app/Models/Nodelist.php Normal file
View File

@ -0,0 +1,20 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
class Nodelist extends Model
{
protected $dates = ['date'];
protected $fillable = ['date','domain_id'];
public const definitions = ['Zone','Region','Host','Hub','Pvt','Down'];
/* RELATIONS */
public function addresses()
{
return $this->belongsToMany(Address::class);
}
}

View File

@ -31,7 +31,7 @@ class System extends Model
* @param Address $o
* @return string
*/
public function name(Address $o): string
public function full_name(Address $o): string
{
switch ($o->attributes['role']) {
case DomainController::NODE_ZC;

82
app/Traits/Import.php Normal file
View File

@ -0,0 +1,82 @@
<?php
/**
* Import from files
*/
namespace App\Traits;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
trait Import
{
protected Collection $_columns; // Columns in the Import File
/**
* Count the lines in a file
*/
private function getFileLines(string $file): int
{
$f = fopen($file,'r');
$c = 0;
while (! feof($f)) {
fgets($f);
$c++;
}
fclose($f);
return $c;
}
/**
* Return the columns from the file that we'll work with
*
* @param string $line
* @return Collection
*/
private function getColumns(string $line): Collection
{
$this->_columns = collect(explode(',',strtoupper($line)))->filter();
return $this->_columns->intersect($this->columns);
}
/**
* Get the index for the column in the file
*
* @param string $key
* @return int|null
*/
private function getColumnKey(string $key): ?int
{
return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL;
}
private function getFileFromHost(string $host,string $key,string $file): string
{
$path = 'import/'.$key;
// If we are doing it locally, we dont need to retrieve it via curl
if (! $host) {
return preg_match('/^data/',$file) ? $file : sprintf('storage/app/public/%s/%s',$path,basename($file));
}
// Get the file from the host
$srcfile = sprintf('http://%s%s',$host,$file);
$dstfile = '/tmp/'.basename($host);
Log::debug(sprintf('%s:Retrieving [%s] from [%s]',static::LOGKEY,$host,$file));
$src = fopen($srcfile,'r');
$dst = fopen($dstfile,'w');
stream_copy_to_stream($src,$dst);
fclose($src);
fclose($dst);
// Store the file for later processing
Storage::putFileAs($path,$dstfile,basename($file));
return Storage::path($path.'/'.basename($file));
}
}

View File

@ -0,0 +1,48 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateNodelist extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('nodelists', function (Blueprint $table) {
$table->id();
$table->timestamps();
$table->date('date');
$table->integer('domain_id');
$table->foreign('domain_id')->references('id')->on('domains');
$table->unique(['date','domain_id']);
});
Schema::create('address_nodelist', function (Blueprint $table) {
$table->integer('role')->nullable();
$table->integer('address_id');
$table->foreign('address_id')->references('id')->on('addresses');
$table->integer('nodelist_id');
$table->foreign('nodelist_id')->references('id')->on('nodelists');
$table->unique(['address_id','nodelist_id']);
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('address_nodelist');
Schema::dropIfExists('nodelists');
}
}

View File

@ -341,13 +341,13 @@ ul#navlist-desktop {
#content ul.ind {
margin-right:2ch
}
#content ul li {
#content ul:not(.pagination) li {
margin:0 1ch;
text-indent:-3ch;
padding-left:3ch;
display:block
}
#content ul li:before {
#content ul:not(.pagination) li:before {
content:"\2666\a0\A0";
color:#0a0
}

View File

@ -79,10 +79,20 @@
<td>-</td>
</tr>
<!-- Other Nodes -->
@foreach ($oz->addresses()->active()->FTNorder()->whereNull('hub_id')->with(['system','zone.domain'])->get() as $ao)
@if ($ao->role == 'Host')
<tr>
<td>{{ sprintf('NC-%s-%05d',$oz->domain->name,$ao->host_id) }}</td>
<td>{{ $ao->system->sysop }}</td>
<td>{{ $ao->system->location }}</td>
<td>{{ $ao->role }}</td>
<td>{{ $oz->zone_id }}:{{ $ao->host_id }}/0.0<span>@</span>{{ $oz->domain->name }}</td>
<td>-</td>
</tr>
@endif
<tr>
<td>{{ $ao->system->name($ao) }}</td>
<td>{{ $ao->system->full_name($ao) }}</td>
<td>{{ $ao->system->sysop }}</td>
<td>{{ $ao->system->location }}</td>
<td>{{ $ao->role }}</td>
@ -93,7 +103,7 @@
<!-- If this node is a hub -->
@foreach ($oz->addresses()->active()->FTNorder()->where('hub_id',$ao->id)->with(['system','zone.domain'])->get() as $aoo)
<tr>
<td>{{ $aoo->system->name($aoo) }}</td>
<td>{{ $aoo->system->full_name($aoo) }}</td>
<td>{{ $aoo->system->sysop }}</td>
<td>{{ $ao->system->location }}</td>
<td>{{ $aoo->role }}</td>

View File

@ -9,6 +9,10 @@
@endsection
@section('content')
@if ($o->exists)
<h1>{{ $o->name }}</h1>
@endif
<div class="accordion accordion-flush" id="accordion_homepage">
@if ($o->exists)

View File

@ -12,7 +12,7 @@
</div>
<div class="row">
<div class="col-10">
<div class="col-12">
<p>This system is aware of the following systems:</p>
@if (\App\Models\System::count() == 0)
@ -22,33 +22,30 @@
<p class="pad">There are no systems - you need to ask an admin to create one for you.</p>
@endcan
@else
<table class="table monotable">
@can('admin',(new \App\Models\System))
<p>You can <a href="{{ url('ftn/system/addedit') }}">Add New System</a>.</p>
@endcan
<table class="table monotable" id="systems">
<thead>
<tr>
<th>ID</th>
<th>System</th>
<th>Sysop</th>
<th>Location</th>
<th>Active</th>
<th>ZeroTier ID</th>
<th>Connect</th>
<th>Address</th>
<th>ZeroTier ID</th>
</tr>
</thead>
<tbody>
@can('admin',(new \App\Models\System))
<tr>
<td colspan="7"><a href="{{ url('ftn/system/addedit') }}">Add New System</a></td>
</tr>
@endcan
@foreach (\App\Models\System::orderBy('name')->cursor() as $oo)
@foreach (\App\Models\System::active()->orderBy('name')->cursor() as $oo)
<tr>
<td><a href="{{ url('ftn/system/addedit',[$oo->id]) }}">{{ $oo->id }}</a></td>
<td>{{ $oo->name }}</td>
<td>{{ $oo->sysop }}</td>
<td>{{ $oo->location }}</td>
<td>{{ $oo->active ? 'YES' : 'NO' }}</td>
<td>-</td>
<td>
@switch($oo->method)
@case(23)<a href="telnet://{{ $oo->address }}:{{ $oo->port }}">Telnet</a>@break
@ -57,6 +54,8 @@
@default No details
@endswitch
</td>
<td>{{ join(',',$oo->addresses->pluck('ftn')->toArray()) }}</td>
<td>{{ $oo->zt_id }}</td>
</tr>
@endforeach
</tbody>
@ -67,6 +66,13 @@
@endsection
@section('page-scripts')
<link type="text/css" rel="stylesheet" href="https://cdn.datatables.net/1.10.25/css/dataTables.bootstrap5.min.css" media="screen" >
<link type="text/css" rel="stylesheet" href="{{ asset('plugin/dataTables/dataTables.bootstrap5.css') }}" media="screen">
<script type="text/javascript" src="https://cdn.datatables.net/1.10.25/js/jquery.dataTables.min.js"></script>
<script type="text/javascript" src="https://cdn.datatables.net/rowgroup/1.1.2/js/dataTables.rowGroup.min.js"></script>
<script type="text/javascript" src="https://cdn.datatables.net/1.10.25/js/dataTables.bootstrap5.min.js"></script>
<script type="text/javascript">
$('table tr').click(function() {
var href = $(this).find('a').attr('href');
@ -74,5 +80,17 @@
if (href)
window.location = href;
});
$('#systems').DataTable({
paging: true,
pageLength: 25,
searching: true,
columnDefs: [
{
targets: [5,6],
visible: false,
}
],
});
</script>
@append