bbs
/
clrghouz
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Support processing nodelists from file repository and assume nodelists are zip fiels

This commit is contained in:
Deon George 2022-11-04 22:26:08 +11:00
parent 2790381a30
commit b45e2c6dd8
3 changed files with 48 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/Console/Commands/NodelistImport.php
View File

@ -5,6 +5,7 @@ namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Jobs\NodelistImport as Job;
use App\Models\File;
class NodelistImport extends Command
{
@ -14,7 +15,7 @@ class NodelistImport extends Command
* @var string
*/
protected $signature = 'nodelist:import'
.' {file : Nodelist File}'
.' {file : File ID}'
.' {--D|delete : Delete old data for the date}'
.' {--U|unlink : Delete file after import}';
@ -32,6 +33,6 @@ class NodelistImport extends Command
*/
public function handle()
{
return Job::dispatchSync($this->argument('file'),$this->option('delete'),$this->option('unlink'));
return Job::dispatchSync(File::findOrFail($this->argument('file')),$this->option('delete'),$this->option('unlink'));
}
}

23
app/Jobs/NodelistImport.php
View File

@ -12,7 +12,7 @@ use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use App\Models\{Address,Domain,Nodelist,Setup,System,Zone};
use App\Models\{Address,Domain,File,Nodelist,Setup,System,Zone};
use App\Traits\Import as ImportTrait;
class NodelistImport implements ShouldQueue
@ -23,18 +23,18 @@ class NodelistImport implements ShouldQueue
protected const LOGKEY = 'JNI';
private const importkey = 'nodelist';
private string $file;
private File $file;
private bool $delete_file;
private bool $delete_recs;
/**
* Import Nodelist constructor.
*
* @param string $file
* @param File $file
* @param bool $delete_recs
* @param bool $delete_file
*/
public function __construct(string $file,bool $delete_recs=FALSE,bool $delete_file=TRUE)
public function __construct(File $file,bool $delete_recs=FALSE,bool $delete_file=TRUE)
{
$this->file = $file;
$this->delete_file = $delete_file;
@ -52,19 +52,22 @@ class NodelistImport implements ShouldQueue
$us = Setup::findOrFail(config('app.id'));
// Get the file from the host
$file = $this->getFileFromHost('',self::importkey,$this->file);
$file = $this->getFileFromHost(self::importkey,$this->file);
Log::debug(sprintf('%s:Loading file [%s] (%s).',static::LOGKEY,$file,getcwd()));
$lines = $this->getFileLines($file);
Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines));
$fh = fopen($file,'r');
$fh = NULL;
$z = $this->openFile($file,$fh);
// Line 1 tells us the nodelist and the CRC
$line = stream_get_line($fh, 0, "\r\n");
$line = stream_get_line($fh,0,"\r\n");
$matches = [];
if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches)))
abort(500,'Nodelist invalid');
if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) {
Log::error(sprintf('Nodelist file [%d] is not valid?',$this->file->id),['m'=>$matches,'l'=>$line]);
throw new \Exception('Invalid nodelist for file '.$this->file->id);
}
$file_crc = $matches[4];
$do = Domain::where('name',strtolower($matches[1]))->single();
@ -98,7 +101,7 @@ class NodelistImport implements ShouldQueue
$tocrc = '';
while (! feof($fh)) {
$line = stream_get_line($fh, 0, "\r\n");
$line = stream_get_line($fh,0,"\r\n");
$tocrc .= $line."\r\n";
// Lines beginning with a semicolon(;) are comments

54
app/Traits/Import.php
View File

@ -9,18 +9,23 @@ use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use App\Models\File;
trait Import
{
protected Collection $_columns; // Columns in the Import File
/**
* Count the lines in a file
* Assumes $file is compressed with ZIP
*/
private function getFileLines(string $file): int
{
$f = fopen($file,'r');
$c = 0;
$f = NULL;
$z = $this->openFile($file,$f);
while (! feof($f)) {
fgets($f);
$c++;
@ -31,6 +36,28 @@ trait Import
return $c;
}
private function openFile(string $file,&$f): \ZipArchive
{
$z = new \ZipArchive;
if ($z->open($file)) {
if ($z->count() !== 1)
throw new \Exception(sprintf('%s:File [%s] has more than 1 file (%d)', self::LOGKEY, $file, $z->count()));
$zipfile = $z->statIndex(0, \ZipArchive::FL_UNCHANGED);
Log::debug(sprintf('%s:Looking at [%s] in archive [%s]', self::LOGKEY,$zipfile['name'],$file));
$f = $z->getStream($zipfile['name']);
if (! $f)
throw new \Exception(sprintf('%s:Failed getting ZipArchive::stream (%s)',self::LOGKEY,$z->getStatusString()));
else
return $z;
} else {
throw new \Exception(sprintf('%s:Failed opening ZipArchive [%s] (%s)',self::LOGKEY,$file,$z->getStatusString()));
}
}
/**
* Return the columns from the file that we'll work with
*
@ -54,29 +81,12 @@ trait Import
return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL;
}
private function getFileFromHost(string $host,string $key,string $file): string
private function getFileFromHost(string $key,File $file): string
{
$path = 'import/'.$key;
$path = sprintf('import/%s.%d',$key,$file->id);
// If we are doing it locally, we dont need to retrieve it via curl
if (! $host) {
return preg_match('/^data/',$file) ? $file : sprintf('storage/app/public/%s/%s',$path,basename($file));
}
Storage::disk('local')->put($path,Storage::get($file->full_storage_path));
// Get the file from the host
$srcfile = sprintf('http://%s%s',$host,$file);
$dstfile = '/tmp/'.basename($host);
Log::debug(sprintf('%s:Retrieving [%s] from [%s]',static::LOGKEY,$host,$file));
$src = fopen($srcfile,'r');
$dst = fopen($dstfile,'w');
stream_copy_to_stream($src,$dst);
fclose($src);
fclose($dst);
// Store the file for later processing
Storage::putFileAs($path,$dstfile,basename($file));
return Storage::path($path.'/'.basename($file));
return Storage::disk('local')->path($path);
}
}