From ff04de52b5bb07f5efbb4cd62d522cb918bbfcfb Mon Sep 17 00:00:00 2001 From: Deon George Date: Sat, 23 Sep 2023 22:01:18 +1000 Subject: [PATCH] Rework TIC processing to use Storage::disk(). Implemented handling of replaces and files that already exist --- app/Classes/FTN/Tic.php | 96 +++++++------ app/Classes/File/File.php | 2 +- app/Classes/File/Item.php | 17 +-- app/Classes/File/Receive.php | 2 +- app/Jobs/PacketProcess.php | 2 +- app/Jobs/TicProcess.php | 10 +- app/Models/File.php | 132 +++++++++++------- app/Traits/Import.php | 6 +- config/fido.php | 3 + .../2023_09_22_233947_file_tweeks.php | 34 +++++ 10 files changed, 199 insertions(+), 105 deletions(-) create mode 100644 database/migrations/2023_09_22_233947_file_tweeks.php diff --git a/app/Classes/FTN/Tic.php b/app/Classes/FTN/Tic.php index 32e99b2..c3f2b2b 100644 --- a/app/Classes/FTN/Tic.php +++ b/app/Classes/FTN/Tic.php @@ -4,10 +4,11 @@ namespace App\Classes\FTN; use Carbon\Carbon; use Illuminate\Contracts\Filesystem\FileNotFoundException; +use Illuminate\Support\Arr; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; -use League\Flysystem\UnableToWriteFile; +use League\Flysystem\UnableToReadFile; use App\Classes\FTN as FTNBase; use App\Models\{Address,File,Filearea,Setup,System}; @@ -104,6 +105,8 @@ class Tic extends FTNBase $result->put('SIZE',$fo->size); if ($fo->description) $result->put('DESC',$fo->description); + if ($fo->replaces) + $result->put('REPLACES',$fo->replaces); $result->put('AREA',$fo->filearea->name); $result->put('AREADESC',$fo->filearea->description); if ($x=$ao->session('ticpass')) @@ -117,9 +120,14 @@ class Tic extends FTNBase foreach ($fo->path as $o) $out .= sprintf("PATH %s %s %s\r\n",$o->ftn3d,$o->pivot->datetime,$o->pivot->extra); + // Add ourself to the path: + $out .= sprintf("PATH %s %s\r\n",$sysaddress->ftn3d,Carbon::now()); + foreach ($fo->seenby as $o) $out .= sprintf("SEENBY %s\r\n",$o->ftn3d); + $out .= sprintf("SEENBY %s\r\n",$sysaddress->ftn3d); + return $out; } @@ -137,33 +145,38 @@ class Tic extends FTNBase /** * Load a TIC file from an existing filename * - * @param string $filename + * @param string $sp + * @param string $filename Relative to filesystem * @return void * @throws FileNotFoundException */ public function load(string $filename): void { Log::info(sprintf('%s:+ Processing TIC file [%s]',self::LOGKEY,$filename)); + $fs = Storage::disk(config('fido.local_disk')); if (str_contains($filename,'-')) { list($hex,$name) = explode('-',$filename); $hex = basename($hex); + } else { $hex = ''; } - if (! file_exists($filename)) + if (! $fs->exists($filename)) throw new FileNotFoundException(sprintf('File [%s] doesnt exist',$filename)); - if (! is_readable($filename)) - throw new UnableToWriteFile(sprintf('File [%s] is not readable',realpath($filename))); + if (! is_readable($fs->path($filename))) + throw new UnableToReadFile(sprintf('File [%s] is not readable',realpath($filename))); - $f = fopen($filename,'rb'); + $f = $fs->readStream($filename); if (! $f) { - Log::error(sprintf('%s:! Unable to open file [%s] for writing',self::LOGKEY,$filename)); + Log::error(sprintf('%s:! Unable to open file [%s] for reading',self::LOGKEY,$filename)); return; } + $ldesc = ''; + while (! feof($f)) { $line = chop(fgets($f)); $matches = []; @@ -171,12 +184,13 @@ class Tic extends FTNBase if (! $line) continue; - preg_match('/([a-zA-Z]+)\ (.*)/',$line,$matches); + preg_match('/([a-zA-Z]+)\ ?(.*)?/',$line,$matches); - if (in_array(strtolower($matches[1]),$this->_kludge)) { + if (in_array(strtolower(Arr::get($matches,1,'-')),$this->_kludge)) { switch ($k=strtolower($matches[1])) { case 'area': $this->{$k} = Filearea::singleOrNew(['name'=>strtoupper($matches[2])]); + break; case 'origin': @@ -184,19 +198,26 @@ class Tic extends FTNBase case 'to': $this->{$k} = Address::findFTN($matches[2]); - // @todo If $this->{$k} is null, we have discovered the system and it should be created + if (! $this->{$k}) + Log::alert(sprintf('%s:! Unable to find an FTN for [%s] for the (%s)',self::LOGKEY,$matches[2],$k)); + break; case 'file': - if (! Storage::disk('local')->exists($x=sprintf('%s/%s-%s',config('app.fido'),$hex,$matches[2]))) - throw new FileNotFoundException(sprintf('File not found? [%s]',$x)); - $this->fo->name = $matches[2]; - $this->fo->fullname = $x; + $this->fo->prefix = $hex; + + if (! $fs->exists($this->fo->recvd_rel_name)) { + // @todo Fail this, so that it is rescheduled to try again in 1-24hrs. + + throw new FileNotFoundException(sprintf('File not found? [%s]',$fs->path($this->fo->recvd_rel_name))); + } + break; case 'areadesc': $areadesc = $matches[2]; + break; case 'created': @@ -205,10 +226,12 @@ class Tic extends FTNBase case 'pw': $pw = $matches[2]; + break; case 'lfile': $this->fo->lname = $matches[2]; + break; case 'desc': @@ -216,46 +239,36 @@ class Tic extends FTNBase case 'replaces': case 'size': $this->fo->{$k} = $matches[2]; + break; case 'fullname': $this->fo->lfile = $matches[2]; + break; case 'date': - $this->fo->datetime = Carbon::create($matches[2]); + $this->fo->datetime = Carbon::createFromTimestamp($matches[2]); + break; case 'ldesc': - $this->fo->{$k} .= $matches[2]; + $ldesc .= ($ldesc ? "\r" : '').$matches[2]; + break; case 'crc': $this->fo->{$k} = hexdec($matches[2]); + break; case 'path': - $x = []; - preg_match(sprintf('#^[Pp]ath (%s)\ ?([0-9]+)\ ?(.*)$#',Address::ftn_regex),$line,$x); - $ao = Address::findFTN($x[1]); - - if (! $ao) - $ao = Address::createFTN($x[1],System::createUnknownSystem()); - - $this->fo->set_path->push(['address'=>$ao,'datetime'=>Carbon::createFromTimestamp($x[8]),'extra'=>$x[9]]); + $this->fo->set_path->push($matches[2]); break; case 'seenby': - $ao = Address::findFTN($matches[2]); - - if (! $ao) - $ao = Address::createFTN($x[1],System::createUnknownSystem()); - - if (! $ao) - $this->fo->rogue_seenby->push($matches[2]); - else - $this->fo->set_seenby->push($ao->id); + $this->fo->set_seenby->push($matches[2]); break; } @@ -265,19 +278,24 @@ class Tic extends FTNBase } } - fclose($f); + if ($ldesc) + $this->fo->ldesc = $ldesc; - $f = fopen($x=Storage::disk('local')->path($this->fo->fullname),'rb'); - $stat = fstat($f); fclose($f); // @todo Add notifictions back to the system + if ($this->fo->replaces && (! preg_match('/^'.$this->fo->replaces.'$/',$this->fo->name))) { + Log::alert(sprintf('%s:! Regex [%s] doesnt match file name [%s]',self::LOGKEY,$this->fo->replaces,$this->fo->name)); + + $this->fo->replaces = NULL; + } + // Validate Size - if ($this->fo->size !== ($y=$stat['size'])) + if ($this->fo->size !== ($y=$fs->size($this->fo->recvd_rel_name))) throw new \Exception(sprintf('TIC file size [%d] doesnt match file [%s] (%d)',$this->fo->size,$this->fo->fullname,$y)); // Validate CRC - if (sprintf('%08x',$this->fo->crc) !== ($y=hash_file('crc32b',$x))) + if (sprintf('%08x',$this->fo->crc) !== ($y=$fs->checksum($this->fo->recvd_rel_name,['checksum_algo'=>'crc32b']))) throw new \Exception(sprintf('TIC file CRC [%08x] doesnt match file [%s] (%s)',$this->fo->crc,$this->fo->fullname,$y)); // Validate Password @@ -303,7 +321,7 @@ class Tic extends FTNBase // If the file create time is blank, we'll take the files if (! $this->fo->datetime) - $this->fo->datetime = Carbon::createFromTimestamp($stat['ctime']); + $this->fo->datetime = Carbon::createFromTimestamp($fs->lastModified($this->fo->recvd_rel_name)); $this->fo->save(); } diff --git a/app/Classes/File/File.php b/app/Classes/File/File.php index 3ef28bc..cbe06a6 100644 --- a/app/Classes/File/File.php +++ b/app/Classes/File/File.php @@ -86,7 +86,7 @@ final class File extends Send { // If sending file is a File::class, then our file is s3 if ($this->nameas && $this->f instanceof FileModel) { - $this->fd = Storage::readStream($this->f->full_storage_path); + $this->fd = Storage::readStream($this->f->rel_name); } else { $this->fd = fopen($this->full_name,'rb'); diff --git a/app/Classes/File/Item.php b/app/Classes/File/Item.php index 64ae20b..896e9c7 100644 --- a/app/Classes/File/Item.php +++ b/app/Classes/File/Item.php @@ -2,19 +2,19 @@ namespace App\Classes\File; +use Illuminate\Contracts\Filesystem\Filesystem; use Illuminate\Support\Facades\Storage; use App\Models\Address; final class Item extends Receive { - private const LOCATION = 'local'; - /** @var Address The address that sent us this item */ private Address $ao; private string $recvas; private int $recvmtime; private int $recvsize; + private Filesystem $fs; /** * @throws \Exception @@ -29,19 +29,20 @@ final class Item extends Receive $this->recvsize = $size; $this->ftype = self::IS_FILE; + $this->fs = Storage::disk(config('fido.local_disk')); } public function __get($key) { switch ($key) { case 'exists': - return Storage::disk(self::LOCATION)->exists($this->rel_name); + return $this->fs->exists($this->rel_name); - case 'stor_name': + case 'pref_name': return sprintf('%04X-%s',$this->ao->id,$this->recvas); case 'rel_name': - return sprintf('%s/%s',config('fido.dir'),$this->stor_name); + return sprintf('%s/%s',config('fido.dir'),$this->pref_name); case 'full_name': - return Storage::disk(self::LOCATION)->path($this->rel_name); + return $this->fs->path($this->rel_name); case 'match_mtime': return $this->mtime === $this->recvmtime; @@ -59,10 +60,10 @@ final class Item extends Receive return sprintf('%s %lu %lu',$this->recvas,$this->recvsize,$this->recvmtime); case 'mtime': - return Storage::disk(self::LOCATION)->lastModified($this->rel_name); + return $this->fs->lastModified($this->rel_name); case 'size': - return Storage::disk(self::LOCATION)->size($this->rel_name); + return $this->fs->size($this->rel_name); default: return parent::__get($key); diff --git a/app/Classes/File/Receive.php b/app/Classes/File/Receive.php index d51f9b4..14e9967 100644 --- a/app/Classes/File/Receive.php +++ b/app/Classes/File/Receive.php @@ -60,7 +60,7 @@ class Receive extends Base case 'nameas': case 'size': case 'name_size_time': - case 'stor_name': + case 'pref_name': return $this->receiving->{$key}; case 'pos': diff --git a/app/Jobs/PacketProcess.php b/app/Jobs/PacketProcess.php index f5f8286..b7f01d4 100644 --- a/app/Jobs/PacketProcess.php +++ b/app/Jobs/PacketProcess.php @@ -137,7 +137,7 @@ class PacketProcess implements ShouldQueue try { if ($fs->makeDirectory($dir)) { - $fs->move($this->file->rel_name,$x=sprintf('%s/%s',$dir,$this->file->stor_name)); + $fs->move($this->file->rel_name,$x=sprintf('%s/%s',$dir,$this->file->pref_name)); Log::info(sprintf('%s:- Moved processed packet [%s] to [%s]',self::LOGKEY,$this->file->rel_name,$x)); } else diff --git a/app/Jobs/TicProcess.php b/app/Jobs/TicProcess.php index 3fdcc5f..80ef812 100644 --- a/app/Jobs/TicProcess.php +++ b/app/Jobs/TicProcess.php @@ -10,6 +10,7 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Log; +use Illuminate\Support\Facades\Storage; use App\Classes\FTN\Tic; use App\Models\Domain; @@ -27,7 +28,7 @@ class TicProcess implements ShouldQueue /** * Create a new job instance. * - * @param string $file + * @param string $file Relative to Storage::disk('local'), ie: storage/app * @param string|null $domain */ public function __construct(private string $file,private ?string $domain=NULL) @@ -56,12 +57,15 @@ class TicProcess implements ShouldQueue */ public function handle() { + $fs = Storage::disk(config('fido.local_disk')); + $to = new Tic; - $to->load(storage_path('app').'/'.$this->file); + $to->load($this->file); Log::info(sprintf('%s:= Processed [%s] storing [%s] as id [%d]',self::LOGKEY,$this->file,$to->fo->name,$to->fo->id)); - unlink(storage_path('app').'/'.$this->file); + if (! $fs->delete($this->file)) + Log::alert(sprintf('%s:! Failed to delete [%s]',self::LOGKEY,$this->file)); if ($to->isNodelist()) NodelistImport::dispatch($to->fo,$this->domain); diff --git a/app/Models/File.php b/app/Models/File.php index 5371f82..e573925 100644 --- a/app/Models/File.php +++ b/app/Models/File.php @@ -5,6 +5,8 @@ namespace App\Models; use Carbon\Carbon; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; +use Illuminate\Support\Arr; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; @@ -20,7 +22,8 @@ class File extends Model private const LOGKEY = 'MF-'; private bool $no_export = FALSE; public string $prefix = ''; - public string $replaces = ''; + public Collection $set_path; + public Collection $set_seenby; protected $casts = [ 'kludges' => CollectionOrNull::class, @@ -37,82 +40,108 @@ class File extends Model 'ldesc', ]; - public function __set($key,$value) - { - switch ($key) { - case 'fullname': - case 'replaces': - case 'no_export': - case 'set_path': - case 'set_seenby': - $this->{$key} = $value; - break; - - default: - parent::__set($key,$value); - } - } - public static function boot() { parent::boot(); static::creating(function($model) { - Log::info(sprintf('%s:- Storing file [%s] in [%s]',self::LOGKEY,$model->fullname,$model->full_storage_path)); - - // Store file - if (Storage::put($model->full_storage_path,Storage::disk('local')->get($model->fullname),'public')) { - unlink(Storage::disk('local')->path($model->fullname)); - } else { - throw new \Exception(sprintf('Unable to move file [%s] to [%s]',$model->fullname,$model->full_storage_path)); + if (! $model->filearea_id) { + Log::alert(sprintf('%s:- File has no filearea, not processing creating [%s]',self::LOGKEY,$model->name)); + return; } + Log::info(sprintf('%s:- Storing file [%s] in [%s]',self::LOGKEY,$model->recvd_rel_name,$model->rel_name)); + + $srcfs = Storage::disk(config('fido.local_disk')); + $tgtfs = Storage::disk(config('fido.file_disk')); + // Delete anything being replaced - // @todo implement replace + foreach (self::where('name',$model->name)->where('filearea_id',$model->filearea_id)->get() as $fo) { + Log::info(sprintf('%s:%% Deleting old file record [%d] for file [%s]',self::LOGKEY,$fo->id,$fo->rel_name)); + + $tgtfs->move($fo->rel_name,$fo->relname.'.'.$fo->id); + $fo->delete(); + } + + // Store file + if ($tgtfs->put($model->rel_name,$srcfs->get($model->recvd_rel_name),'public')) { + $srcfs->delete($model->recvd_rel_name); + + } else { + throw new \Exception(sprintf('Unable to move file [%s] to [%s]',$model->recvd_rel_name,$model->rel_name)); + } }); // @todo if the file is updated with new SEEN-BY's from another route, we'll delete the pending export for systems (if there is one) static::created(function($model) { if (! $model->filearea_id) { - Log::alert(sprintf('%s:- File has no filearea, not exporting',self::LOGKEY,$model->id)); + Log::alert(sprintf('%s:- File has no filearea, not exporting [%d]',self::LOGKEY,$model->id)); return; } + $rogue = collect(); + $seenby = collect(); + $path = collect(); + $so = Setup::findOrFail(config('app.id')); + $zone = $model->fftn->zone; - // Our address - $ftns = $so - ->system - ->match($model->fftn->zone); + // Parse PATH + foreach ($model->set_path as $line) { + $matches = []; + preg_match(sprintf('#^(%s)\ ?([0-9]+)\ ?(.*)$#',Address::ftn_regex),$line,$matches); - // Add our address to the seenby; - $model->set_seenby = $model->set_seenby->merge($ftns->pluck('id'))->unique(); - $model->set_path = $model->set_path->merge([[ - 'address'=>$ftns->first(), - 'datetime'=>$x=Carbon::now(), - 'extra'=>sprintf('%s %s (%s)',$x->toRfc7231String(),$so::PRODUCT_NAME,$so->version), - ]]); + if ($x=Arr::get($matches,1)) { + $ftn = Address::parseFTN($x); - // Make sure all the path is in the seenby - $model->set_seenby = $model->set_seenby->merge($model->set_path->pluck('address.id'))->unique()->filter(); + // If domain should be flattened, look for node regardless of zone (within the list of zones for the domain) + $ao = ($zone->domain->flatten) + ? Address::findZone($zone->domain,$ftn['n'],$ftn['f'],0) + : Address::findFTN($x); - // Save the seenby - $model->seenby()->sync($model->set_seenby); + if (! $ao) + $ao = Address::createFTN($x,System::createUnknownSystem()); + + $path->push(['address'=>$ao,'datetime'=>Carbon::createFromTimestamp($matches[9]),'extra'=>$matches[10]]); + } + } // Save the Path $ppoid = NULL; - foreach ($model->set_path as $path) { + foreach ($path as $item) { $po = DB::select('INSERT INTO file_path (file_id,address_id,parent_id,datetime,extra) VALUES (?,?,?,?,?) RETURNING id',[ $model->id, - $path['address']->id, + $item['address']->id, $ppoid, - Carbon::createFromTimestamp($path['datetime']), - $path['extra'], + $item['datetime'], + $item['extra'], ]); $ppoid = $po[0]->id; } + // Make sure all the path is in the seenby + // Add zone to seenby + $model->set_seenby = $model->set_seenby->merge($path->pluck('address.ftn3d'))->unique()->filter(); + + foreach ($model->set_seenby as $sb) { + $ftn = Address::parseFTN($x); + + $ao = ($zone->domain->flatten) + ? Address::findZone($zone->domain,$ftn['n'],$ftn['f'],0) + : Address::findFTN($x); + + if ($ao) + $seenby->push($ao->id); + else + $rogue->push($sb); + } + + $model->rogue_seenby = $rogue; + + $model->seenby()->sync($seenby); + $model->save(); + // See if we need to export this file. if ($model->filearea->sec_read) { $exportto = $model @@ -169,7 +198,7 @@ class File extends Model * * @return string */ - public function getFullStoragePathAttribute(): string + public function getRelNameAttribute(): string { return sprintf('%04X/%s',$this->filearea_id,$this->name); } @@ -179,12 +208,17 @@ class File extends Model * * @return string */ - public function getRelNameAttribute(): string + public function getRecvdRelNameAttribute(): string { - return sprintf('%s/%s',config('fido.dir'),$this->prefix_name); + return sprintf('%s/%s',config('fido.dir'),$this->recvd_pref_name); } - public function getPrefixNameAttribute(): string + /** + * This is the name of the file, with the sender prefix + * + * @return string + */ + public function getRecvdPrefNameAttribute(): string { return sprintf('%s%s',$this->prefix ? $this->prefix.'-' : '',$this->name); } diff --git a/app/Traits/Import.php b/app/Traits/Import.php index f85b29b..b95fe84 100644 --- a/app/Traits/Import.php +++ b/app/Traits/Import.php @@ -86,12 +86,12 @@ trait Import if ($file instanceof File) { $path = sprintf('import/%s.%d',$key,$file->id); - Storage::disk('local')->put($path,Storage::get($file->full_storage_path)); + Storage::disk(config('fido.local_disk'))->put($path,Storage::get($file->rel_name)); - return Storage::disk('local')->path($path); + return Storage::disk(config('fido.local_disk'))->path($path); } else { - return Storage::disk('local')->path($file); + return Storage::disk(config('fido.local_disk'))->path($file); } } } \ No newline at end of file diff --git a/config/fido.php b/config/fido.php index 557d490..b8138a4 100644 --- a/config/fido.php +++ b/config/fido.php @@ -10,6 +10,9 @@ return [ // Directory to use for any data we want to store locally 'dir' => env('FIDO_DIR', 'fido'), + // Our Storage::disk() for files storage + 'file_disk' => env('FIDO_DIR_FILES', 's3'), + // Our Storage::disk() for locally stored data 'local_disk' => env('FIDO_LOCAL_DIR','local'), diff --git a/database/migrations/2023_09_22_233947_file_tweeks.php b/database/migrations/2023_09_22_233947_file_tweeks.php new file mode 100644 index 0000000..01e45af --- /dev/null +++ b/database/migrations/2023_09_22_233947_file_tweeks.php @@ -0,0 +1,34 @@ +dropUnique(['filearea_id','name']); + $table->string('replaces')->nullable(); + }); + + DB::statement("CREATE UNIQUE INDEX files_active ON files (filearea_id, name) WHERE deleted_at IS NULL"); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + DB::statement("DROP INDEX files_active"); + + Schema::table('files', function (Blueprint $table) { + $table->unique(['filearea_id','name']); + $table->dropColumn('replaces'); + }); + } +};