Support processing nodelists from file repository and assume nodelists are zip fiels

app/Console/Commands/NodelistImport.php

@@ -5,6 +5,7 @@ namespace App\Console\Commands;
 use Illuminate\Console\Command;
 
 use App\Jobs\NodelistImport as Job;
+use App\Models\File;
 
 class NodelistImport extends Command
 {
@@ -14,7 +15,7 @@ class NodelistImport extends Command
 	 * @var string
 	 */
 	protected $signature = 'nodelist:import'
-		.' {file : Nodelist File}'
+		.' {file : File ID}'
 		.' {--D|delete : Delete old data for the date}'
 		.' {--U|unlink : Delete file after import}';
 
@@ -32,6 +33,6 @@ class NodelistImport extends Command
 	 */
 	public function handle()
 	{
-		return Job::dispatchSync($this->argument('file'),$this->option('delete'),$this->option('unlink'));
+		return Job::dispatchSync(File::findOrFail($this->argument('file')),$this->option('delete'),$this->option('unlink'));
 	}
 }

app/Jobs/NodelistImport.php

@@ -12,7 +12,7 @@ use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
 
-use App\Models\{Address,Domain,Nodelist,Setup,System,Zone};
+use App\Models\{Address,Domain,File,Nodelist,Setup,System,Zone};
 use App\Traits\Import as ImportTrait;
 
 class NodelistImport implements ShouldQueue
@@ -23,18 +23,18 @@ class NodelistImport implements ShouldQueue
 	protected const LOGKEY = 'JNI';
 	private const importkey = 'nodelist';
 
-	private string $file;
+	private File $file;
 	private bool $delete_file;
 	private bool $delete_recs;
 
 	/**
 	 * Import Nodelist constructor.
 	 *
-	 * @param string $file
+	 * @param File $file
 	 * @param bool $delete_recs
 	 * @param bool $delete_file
 	 */
-	public function __construct(string $file,bool $delete_recs=FALSE,bool $delete_file=TRUE)
+	public function __construct(File $file,bool $delete_recs=FALSE,bool $delete_file=TRUE)
 	{
 		$this->file = $file;
 		$this->delete_file = $delete_file;
@@ -52,19 +52,22 @@ class NodelistImport implements ShouldQueue
 		$us = Setup::findOrFail(config('app.id'));
 
 		// Get the file from the host
-		$file = $this->getFileFromHost('',self::importkey,$this->file);
+		$file = $this->getFileFromHost(self::importkey,$this->file);
 		Log::debug(sprintf('%s:Loading file [%s] (%s).',static::LOGKEY,$file,getcwd()));
 		$lines = $this->getFileLines($file);
 		Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines));
 
-		$fh = fopen($file,'r');
+		$fh = NULL;
+		$z = $this->openFile($file,$fh);
 
 		// Line 1 tells us the nodelist and the CRC
-		$line = stream_get_line($fh, 0, "\r\n");
+		$line = stream_get_line($fh,0,"\r\n");
 
 		$matches = [];
-		if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches)))
+		if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) {
-			abort(500,'Nodelist invalid');
+			Log::error(sprintf('Nodelist file [%d] is not valid?',$this->file->id),['m'=>$matches,'l'=>$line]);
+			throw new \Exception('Invalid nodelist for file '.$this->file->id);
+		}
 
 		$file_crc = $matches[4];
 		$do = Domain::where('name',strtolower($matches[1]))->single();
@@ -98,7 +101,7 @@ class NodelistImport implements ShouldQueue
 		$tocrc = '';
 
 		while (! feof($fh)) {
-			$line = stream_get_line($fh, 0, "\r\n");
+			$line = stream_get_line($fh,0,"\r\n");
 			$tocrc .= $line."\r\n";
 
 			// Lines beginning with a semicolon(;) are comments

app/Traits/Import.php

@@ -9,18 +9,23 @@ use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
 
+use App\Models\File;
+
 trait Import
 {
 	protected Collection $_columns; // Columns in the Import File
 
 	/**
 	 * Count the lines in a file
+	 * Assumes $file is compressed with ZIP
 	 */
 	private function getFileLines(string $file): int
 	{
-		$f = fopen($file,'r');
 		$c = 0;
 
+		$f = NULL;
+		$z = $this->openFile($file,$f);
+
 		while (! feof($f)) {
 			fgets($f);
 			$c++;
@@ -31,6 +36,28 @@ trait Import
 		return $c;
 	}
 
+	private function openFile(string $file,&$f): \ZipArchive
+	{
+		$z = new \ZipArchive;
+
+		if ($z->open($file)) {
+			if ($z->count() !== 1)
+				throw new \Exception(sprintf('%s:File [%s] has more than 1 file (%d)', self::LOGKEY, $file, $z->count()));
+
+			$zipfile = $z->statIndex(0, \ZipArchive::FL_UNCHANGED);
+			Log::debug(sprintf('%s:Looking at [%s] in archive [%s]', self::LOGKEY,$zipfile['name'],$file));
+
+			$f = $z->getStream($zipfile['name']);
+			if (! $f)
+				throw new \Exception(sprintf('%s:Failed getting ZipArchive::stream (%s)',self::LOGKEY,$z->getStatusString()));
+			else
+				return $z;
+
+		} else {
+			throw new \Exception(sprintf('%s:Failed opening ZipArchive [%s] (%s)',self::LOGKEY,$file,$z->getStatusString()));
+		}
+	}
+
 	/**
 	 * Return the columns from the file that we'll work with
 	 *
@@ -54,29 +81,12 @@ trait Import
 		return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL;
 	}
 
-	private function getFileFromHost(string $host,string $key,string $file): string
+	private function getFileFromHost(string $key,File $file): string
 	{
-		$path = 'import/'.$key;
+		$path = sprintf('import/%s.%d',$key,$file->id);
 
-		// If we are doing it locally, we dont need to retrieve it via curl
+		Storage::disk('local')->put($path,Storage::get($file->full_storage_path));
-		if (! $host) {
-			return preg_match('/^data/',$file) ? $file : sprintf('storage/app/public/%s/%s',$path,basename($file));
-		}
 
-		// Get the file from the host
+		return Storage::disk('local')->path($path);
-		$srcfile = sprintf('http://%s%s',$host,$file);
-		$dstfile = '/tmp/'.basename($host);
-		Log::debug(sprintf('%s:Retrieving [%s] from [%s]',static::LOGKEY,$host,$file));
-
-		$src = fopen($srcfile,'r');
-		$dst = fopen($dstfile,'w');
-		stream_copy_to_stream($src,$dst);
-		fclose($src);
-		fclose($dst);
-
-		// Store the file for later processing
-		Storage::putFileAs($path,$dstfile,basename($file));
-
-		return Storage::path($path.'/'.basename($file));
 	}
 }