From b45e2c6dd88014bc8c13a947f9e1326164d5ccf6 Mon Sep 17 00:00:00 2001 From: Deon George Date: Fri, 4 Nov 2022 22:26:08 +1100 Subject: [PATCH] Support processing nodelists from file repository and assume nodelists are zip fiels --- app/Console/Commands/NodelistImport.php | 5 ++- app/Jobs/NodelistImport.php | 23 ++++++----- app/Traits/Import.php | 54 +++++++++++++++---------- 3 files changed, 48 insertions(+), 34 deletions(-) diff --git a/app/Console/Commands/NodelistImport.php b/app/Console/Commands/NodelistImport.php index 0b1984a..68a2a97 100644 --- a/app/Console/Commands/NodelistImport.php +++ b/app/Console/Commands/NodelistImport.php @@ -5,6 +5,7 @@ namespace App\Console\Commands; use Illuminate\Console\Command; use App\Jobs\NodelistImport as Job; +use App\Models\File; class NodelistImport extends Command { @@ -14,7 +15,7 @@ class NodelistImport extends Command * @var string */ protected $signature = 'nodelist:import' - .' {file : Nodelist File}' + .' {file : File ID}' .' {--D|delete : Delete old data for the date}' .' {--U|unlink : Delete file after import}'; @@ -32,6 +33,6 @@ class NodelistImport extends Command */ public function handle() { - return Job::dispatchSync($this->argument('file'),$this->option('delete'),$this->option('unlink')); + return Job::dispatchSync(File::findOrFail($this->argument('file')),$this->option('delete'),$this->option('unlink')); } } \ No newline at end of file diff --git a/app/Jobs/NodelistImport.php b/app/Jobs/NodelistImport.php index 9d2c150..49b55ce 100644 --- a/app/Jobs/NodelistImport.php +++ b/app/Jobs/NodelistImport.php @@ -12,7 +12,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Str; -use App\Models\{Address,Domain,Nodelist,Setup,System,Zone}; +use App\Models\{Address,Domain,File,Nodelist,Setup,System,Zone}; use App\Traits\Import as ImportTrait; class NodelistImport implements ShouldQueue @@ -23,18 +23,18 @@ class NodelistImport implements ShouldQueue protected const LOGKEY = 'JNI'; private const importkey = 'nodelist'; - private string $file; + private File $file; private bool $delete_file; private bool $delete_recs; /** * Import Nodelist constructor. * - * @param string $file + * @param File $file * @param bool $delete_recs * @param bool $delete_file */ - public function __construct(string $file,bool $delete_recs=FALSE,bool $delete_file=TRUE) + public function __construct(File $file,bool $delete_recs=FALSE,bool $delete_file=TRUE) { $this->file = $file; $this->delete_file = $delete_file; @@ -52,19 +52,22 @@ class NodelistImport implements ShouldQueue $us = Setup::findOrFail(config('app.id')); // Get the file from the host - $file = $this->getFileFromHost('',self::importkey,$this->file); + $file = $this->getFileFromHost(self::importkey,$this->file); Log::debug(sprintf('%s:Loading file [%s] (%s).',static::LOGKEY,$file,getcwd())); $lines = $this->getFileLines($file); Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines)); - $fh = fopen($file,'r'); + $fh = NULL; + $z = $this->openFile($file,$fh); // Line 1 tells us the nodelist and the CRC - $line = stream_get_line($fh, 0, "\r\n"); + $line = stream_get_line($fh,0,"\r\n"); $matches = []; - if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) - abort(500,'Nodelist invalid'); + if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) { + Log::error(sprintf('Nodelist file [%d] is not valid?',$this->file->id),['m'=>$matches,'l'=>$line]); + throw new \Exception('Invalid nodelist for file '.$this->file->id); + } $file_crc = $matches[4]; $do = Domain::where('name',strtolower($matches[1]))->single(); @@ -98,7 +101,7 @@ class NodelistImport implements ShouldQueue $tocrc = ''; while (! feof($fh)) { - $line = stream_get_line($fh, 0, "\r\n"); + $line = stream_get_line($fh,0,"\r\n"); $tocrc .= $line."\r\n"; // Lines beginning with a semicolon(;) are comments diff --git a/app/Traits/Import.php b/app/Traits/Import.php index 526d71b..d5ce711 100644 --- a/app/Traits/Import.php +++ b/app/Traits/Import.php @@ -9,18 +9,23 @@ use Illuminate\Support\Collection; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; +use App\Models\File; + trait Import { protected Collection $_columns; // Columns in the Import File /** * Count the lines in a file + * Assumes $file is compressed with ZIP */ private function getFileLines(string $file): int { - $f = fopen($file,'r'); $c = 0; + $f = NULL; + $z = $this->openFile($file,$f); + while (! feof($f)) { fgets($f); $c++; @@ -31,6 +36,28 @@ trait Import return $c; } + private function openFile(string $file,&$f): \ZipArchive + { + $z = new \ZipArchive; + + if ($z->open($file)) { + if ($z->count() !== 1) + throw new \Exception(sprintf('%s:File [%s] has more than 1 file (%d)', self::LOGKEY, $file, $z->count())); + + $zipfile = $z->statIndex(0, \ZipArchive::FL_UNCHANGED); + Log::debug(sprintf('%s:Looking at [%s] in archive [%s]', self::LOGKEY,$zipfile['name'],$file)); + + $f = $z->getStream($zipfile['name']); + if (! $f) + throw new \Exception(sprintf('%s:Failed getting ZipArchive::stream (%s)',self::LOGKEY,$z->getStatusString())); + else + return $z; + + } else { + throw new \Exception(sprintf('%s:Failed opening ZipArchive [%s] (%s)',self::LOGKEY,$file,$z->getStatusString())); + } + } + /** * Return the columns from the file that we'll work with * @@ -54,29 +81,12 @@ trait Import return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL; } - private function getFileFromHost(string $host,string $key,string $file): string + private function getFileFromHost(string $key,File $file): string { - $path = 'import/'.$key; + $path = sprintf('import/%s.%d',$key,$file->id); - // If we are doing it locally, we dont need to retrieve it via curl - if (! $host) { - return preg_match('/^data/',$file) ? $file : sprintf('storage/app/public/%s/%s',$path,basename($file)); - } + Storage::disk('local')->put($path,Storage::get($file->full_storage_path)); - // Get the file from the host - $srcfile = sprintf('http://%s%s',$host,$file); - $dstfile = '/tmp/'.basename($host); - Log::debug(sprintf('%s:Retrieving [%s] from [%s]',static::LOGKEY,$host,$file)); - - $src = fopen($srcfile,'r'); - $dst = fopen($dstfile,'w'); - stream_copy_to_stream($src,$dst); - fclose($src); - fclose($dst); - - // Store the file for later processing - Storage::putFileAs($path,$dstfile,basename($file)); - - return Storage::path($path.'/'.basename($file)); + return Storage::disk('local')->path($path); } } \ No newline at end of file