Support nodelist archives with more than 1 file in it

This commit is contained in:
Deon George 2024-06-15 15:12:09 +10:00
parent 941117b342
commit 180c620168
4 changed files with 31 additions and 31 deletions

View File

@ -10,12 +10,10 @@ use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use App\Models\{Domain,Echoarea}; use App\Models\{Domain,Echoarea};
use App\Traits\Import as ImportTrait;
class EchoareaImport implements ShouldQueue class EchoareaImport implements ShouldQueue
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
use ImportTrait;
protected const LOGKEY = 'JEI'; protected const LOGKEY = 'JEI';
private const importkey = 'echoarea'; private const importkey = 'echoarea';

View File

@ -10,12 +10,10 @@ use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use App\Models\{Domain,Filearea}; use App\Models\{Domain,Filearea};
use App\Traits\Import as ImportTrait;
class FileareaImport implements ShouldQueue class FileareaImport implements ShouldQueue
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
use ImportTrait;
protected const LOGKEY = 'JFI'; protected const LOGKEY = 'JFI';
private const importkey = 'filearea'; private const importkey = 'filearea';

View File

@ -80,11 +80,27 @@ class NodelistImport implements ShouldQueue
// Get the file from the host // Get the file from the host
$file = $this->getFileFromHost(self::importkey,$this->file); $file = $this->getFileFromHost(self::importkey,$this->file);
Log::debug(sprintf('%s:+ Loading file [%s].',static::LOGKEY,$file)); Log::debug(sprintf('%s:+ Loading file [%s].',static::LOGKEY,$file));
$lines = $this->getFileLines($file); $z = $this->openFile($file);
Log::debug(sprintf('%s:- Processing [%d] lines.',static::LOGKEY,$lines));
$c = 0;
$fh = NULL; $fh = NULL;
$z = $this->openFile($file,$fh); while ($c < $z->count()) {
// Nodelist files have an extension of numbers, between 1-365
if (preg_match('/^.+\.[0-3][0-9][0-9]$/',$z->getNameIndex($c))) {
$fh = $z->getStreamIndex($c);
break;
}
$c++;
}
if (is_null($fh))
throw new \Exception('Couldnt find nodelist in file');
$lines = $this->getFileLines($fh);
Log::debug(sprintf('%s:- Processing [%d] lines.',static::LOGKEY,$lines));
// Rewind
$fh = $z->getStreamIndex($c);
// Line 1 tells us the nodelist and the CRC // Line 1 tells us the nodelist and the CRC
$line = stream_get_line($fh,0,"\r\n"); $line = stream_get_line($fh,0,"\r\n");
@ -123,9 +139,9 @@ class NodelistImport implements ShouldQueue
$no->addresses()->detach(); $no->addresses()->detach();
elseif ($no->addresses->count()) { elseif ($no->addresses->count()) {
Log::error($x=sprintf('%s:! Nodelist [%s] for [%s] has existing records [%d]',self::LOGKEY,$date,$do->name,$no->addresses->count())); Log::error(sprintf('%s:! Nodelist [%s] for [%s] has existing records [%d]',self::LOGKEY,$date,$do->name,$no->addresses->count()));
throw new \Exception($x); return;
} }
$mailer_binkp = Mailer::where('name','BINKP')->singleOrFail(); $mailer_binkp = Mailer::where('name','BINKP')->singleOrFail();
@ -179,6 +195,7 @@ class NodelistImport implements ShouldQueue
$region = 0; $region = 0;
$host = 0; $host = 0;
$ishub = FALSE; $ishub = FALSE;
$ho = NULL;
break; break;
@ -186,18 +203,21 @@ class NodelistImport implements ShouldQueue
$region = (int)$fields[1]; $region = (int)$fields[1];
$host = (int)$fields[1]; $host = (int)$fields[1];
$ishub = FALSE; $ishub = FALSE;
$ho = NULL;
break; break;
case 'Host': case 'Host':
$host = (int)$fields[1]; $host = (int)$fields[1];
$ishub = FALSE; $ishub = FALSE;
$ho = NULL;
break; break;
case 'Hub': case 'Hub':
$node = (int)$fields[1]; $node = (int)$fields[1];
$ishub = TRUE; $ishub = TRUE;
$ho = NULL;
break; break;
@ -543,7 +563,7 @@ class NodelistImport implements ShouldQueue
->diff(our_nodes($do)->pluck('id')); ->diff(our_nodes($do)->pluck('id'));
$remove = Address::whereIn('id',$remove)->get(); $remove = Address::whereIn('id',$remove)->get();
Log::alert(sprintf('%s:%% Deleting [%d] addresses [%s]',self::LOGKEY,$remove->count(),$remove->pluck('ftn4d')->join(','))); Log::alert(sprintf('%s:%% Deleting [%d] addresses [%s]',self::LOGKEY,$remove->count(),$remove->pluck('ftn2d')->join(',')));
Address::whereIN('id',$remove->pluck('id'))->update(['active'=>FALSE]); Address::whereIN('id',$remove->pluck('id'))->update(['active'=>FALSE]);
Address::whereIN('id',$remove->pluck('id'))->delete(); Address::whereIN('id',$remove->pluck('id'))->delete();

View File

@ -6,7 +6,6 @@
namespace App\Traits; namespace App\Traits;
use Illuminate\Support\Collection; use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Storage;
use App\Models\File; use App\Models\File;
@ -19,38 +18,23 @@ trait Import
* Count the lines in a file * Count the lines in a file
* Assumes $file is compressed with ZIP * Assumes $file is compressed with ZIP
*/ */
private function getFileLines(string $file): int private function getFileLines(mixed $f): int
{ {
$c = 0; $c = 0;
$f = NULL;
$z = $this->openFile($file,$f);
while (! feof($f)) { while (! feof($f)) {
fgets($f); fgets($f);
$c++; $c++;
} }
fclose($f);
return $c; return $c;
} }
private function openFile(string $file,&$f): \ZipArchive private function openFile(string $file): \ZipArchive
{ {
$z = new \ZipArchive; $z = new \ZipArchive;
if ($z->open($file,\ZipArchive::RDONLY) === TRUE) { if ($z->open($file,\ZipArchive::RDONLY) === TRUE) {
if ($z->count() !== 1)
throw new \Exception(sprintf('%s:! File [%s] has more than 1 file (%d)', self::LOGKEY, $file, $z->count()));
$zipfile = $z->statIndex(0, \ZipArchive::FL_UNCHANGED);
Log::debug(sprintf('%s:- Looking at [%s] in archive [%s]', self::LOGKEY,$zipfile['name'],$file));
$f = $z->getStream($zipfile['name']);
if (! $f)
throw new \Exception(sprintf('%s:! Failed getting ZipArchive::stream (%s)',self::LOGKEY,$z->getStatusString()));
else
return $z; return $z;
} else { } else {