bbs/clrghouz
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Rework TIC processing to use Storage::disk(). Implemented handling of replaces and files that already exist

Browse Source
This commit is contained in:
Deon George 2023-09-23 22:01:18 +10:00
parent 56544b89e1
commit ff04de52b5
10 changed files with 199 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
app/Classes/FTN/Tic.php
View File

@ -4,10 +4,11 @@ namespace App\Classes\FTN;
use Carbon\Carbon;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use League\Flysystem\UnableToWriteFile;
use League\Flysystem\UnableToReadFile;
use App\Classes\FTN as FTNBase;
use App\Models\{Address,File,Filearea,Setup,System};
@ -104,6 +105,8 @@ class Tic extends FTNBase
$result->put('SIZE',$fo->size);
if ($fo->description)
$result->put('DESC',$fo->description);
if ($fo->replaces)
$result->put('REPLACES',$fo->replaces);
$result->put('AREA',$fo->filearea->name);
$result->put('AREADESC',$fo->filearea->description);
if ($x=$ao->session('ticpass'))
@ -117,9 +120,14 @@ class Tic extends FTNBase
foreach ($fo->path as $o)
$out .= sprintf("PATH %s %s %s\r\n",$o->ftn3d,$o->pivot->datetime,$o->pivot->extra);
// Add ourself to the path:
$out .= sprintf("PATH %s %s\r\n",$sysaddress->ftn3d,Carbon::now());
foreach ($fo->seenby as $o)
$out .= sprintf("SEENBY %s\r\n",$o->ftn3d);
$out .= sprintf("SEENBY %s\r\n",$sysaddress->ftn3d);
return $out;
}
@ -137,33 +145,38 @@ class Tic extends FTNBase
/**
* Load a TIC file from an existing filename
*
* @param string $filename
* @param string $sp
* @param string $filename Relative to filesystem
* @return void
* @throws FileNotFoundException
*/
public function load(string $filename): void
{
Log::info(sprintf('%s:+ Processing TIC file [%s]',self::LOGKEY,$filename));
$fs = Storage::disk(config('fido.local_disk'));
if (str_contains($filename,'-')) {
list($hex,$name) = explode('-',$filename);
$hex = basename($hex);
} else {
$hex = '';
}
if (! file_exists($filename))
if (! $fs->exists($filename))
throw new FileNotFoundException(sprintf('File [%s] doesnt exist',$filename));
if (! is_readable($filename))
throw new UnableToWriteFile(sprintf('File [%s] is not readable',realpath($filename)));
if (! is_readable($fs->path($filename)))
throw new UnableToReadFile(sprintf('File [%s] is not readable',realpath($filename)));
$f = fopen($filename,'rb');
$f = $fs->readStream($filename);
if (! $f) {
Log::error(sprintf('%s:! Unable to open file [%s] for writing',self::LOGKEY,$filename));
Log::error(sprintf('%s:! Unable to open file [%s] for reading',self::LOGKEY,$filename));
return;
}
$ldesc = '';
while (! feof($f)) {
$line = chop(fgets($f));
$matches = [];
@ -171,12 +184,13 @@ class Tic extends FTNBase
if (! $line)
continue;
preg_match('/([a-zA-Z]+)\ (.*)/',$line,$matches);
preg_match('/([a-zA-Z]+)\ ?(.*)?/',$line,$matches);
if (in_array(strtolower($matches[1]),$this->_kludge)) {
if (in_array(strtolower(Arr::get($matches,1,'-')),$this->_kludge)) {
switch ($k=strtolower($matches[1])) {
case 'area':
$this->{$k} = Filearea::singleOrNew(['name'=>strtoupper($matches[2])]);
break;
case 'origin':
@ -184,19 +198,26 @@ class Tic extends FTNBase
case 'to':
$this->{$k} = Address::findFTN($matches[2]);
// @todo If $this->{$k} is null, we have discovered the system and it should be created
if (! $this->{$k})
Log::alert(sprintf('%s:! Unable to find an FTN for [%s] for the (%s)',self::LOGKEY,$matches[2],$k));
break;
case 'file':
if (! Storage::disk('local')->exists($x=sprintf('%s/%s-%s',config('app.fido'),$hex,$matches[2])))
throw new FileNotFoundException(sprintf('File not found? [%s]',$x));
$this->fo->name = $matches[2];
$this->fo->fullname = $x;
$this->fo->prefix = $hex;
if (! $fs->exists($this->fo->recvd_rel_name)) {
// @todo Fail this, so that it is rescheduled to try again in 1-24hrs.
throw new FileNotFoundException(sprintf('File not found? [%s]',$fs->path($this->fo->recvd_rel_name)));
}
break;
case 'areadesc':
$areadesc = $matches[2];
break;
case 'created':
@ -205,10 +226,12 @@ class Tic extends FTNBase
case 'pw':
$pw = $matches[2];
break;
case 'lfile':
$this->fo->lname = $matches[2];
break;
case 'desc':
@ -216,46 +239,36 @@ class Tic extends FTNBase
case 'replaces':
case 'size':
$this->fo->{$k} = $matches[2];
break;
case 'fullname':
$this->fo->lfile = $matches[2];
break;
case 'date':
$this->fo->datetime = Carbon::create($matches[2]);
$this->fo->datetime = Carbon::createFromTimestamp($matches[2]);
break;
case 'ldesc':
$this->fo->{$k} .= $matches[2];
$ldesc .= ($ldesc ? "\r" : '').$matches[2];
break;
case 'crc':
$this->fo->{$k} = hexdec($matches[2]);
break;
case 'path':
$x = [];
preg_match(sprintf('#^[Pp]ath (%s)\ ?([0-9]+)\ ?(.*)$#',Address::ftn_regex),$line,$x);
$ao = Address::findFTN($x[1]);
if (! $ao)
$ao = Address::createFTN($x[1],System::createUnknownSystem());
$this->fo->set_path->push(['address'=>$ao,'datetime'=>Carbon::createFromTimestamp($x[8]),'extra'=>$x[9]]);
$this->fo->set_path->push($matches[2]);
break;
case 'seenby':
$ao = Address::findFTN($matches[2]);
if (! $ao)
$ao = Address::createFTN($x[1],System::createUnknownSystem());
if (! $ao)
$this->fo->rogue_seenby->push($matches[2]);
else
$this->fo->set_seenby->push($ao->id);
$this->fo->set_seenby->push($matches[2]);
break;
}
@ -265,19 +278,24 @@ class Tic extends FTNBase
}
}
fclose($f);
if ($ldesc)
$this->fo->ldesc = $ldesc;
$f = fopen($x=Storage::disk('local')->path($this->fo->fullname),'rb');
$stat = fstat($f);
fclose($f);
// @todo Add notifictions back to the system
if ($this->fo->replaces && (! preg_match('/^'.$this->fo->replaces.'$/',$this->fo->name))) {
Log::alert(sprintf('%s:! Regex [%s] doesnt match file name [%s]',self::LOGKEY,$this->fo->replaces,$this->fo->name));
$this->fo->replaces = NULL;
}
// Validate Size
if ($this->fo->size !== ($y=$stat['size']))
if ($this->fo->size !== ($y=$fs->size($this->fo->recvd_rel_name)))
throw new \Exception(sprintf('TIC file size [%d] doesnt match file [%s] (%d)',$this->fo->size,$this->fo->fullname,$y));
// Validate CRC
if (sprintf('%08x',$this->fo->crc) !== ($y=hash_file('crc32b',$x)))
if (sprintf('%08x',$this->fo->crc) !== ($y=$fs->checksum($this->fo->recvd_rel_name,['checksum_algo'=>'crc32b'])))
throw new \Exception(sprintf('TIC file CRC [%08x] doesnt match file [%s] (%s)',$this->fo->crc,$this->fo->fullname,$y));
// Validate Password
@ -303,7 +321,7 @@ class Tic extends FTNBase
// If the file create time is blank, we'll take the files
if (! $this->fo->datetime)
$this->fo->datetime = Carbon::createFromTimestamp($stat['ctime']);
$this->fo->datetime = Carbon::createFromTimestamp($fs->lastModified($this->fo->recvd_rel_name));
$this->fo->save();
}

2
app/Classes/File/File.php
View File

@ -86,7 +86,7 @@ final class File extends Send
{
// If sending file is a File::class, then our file is s3
if ($this->nameas && $this->f instanceof FileModel) {
$this->fd = Storage::readStream($this->f->full_storage_path);
$this->fd = Storage::readStream($this->f->rel_name);
} else {
$this->fd = fopen($this->full_name,'rb');

17
app/Classes/File/Item.php
View File

@ -2,19 +2,19 @@
namespace App\Classes\File;
use Illuminate\Contracts\Filesystem\Filesystem;
use Illuminate\Support\Facades\Storage;
use App\Models\Address;
final class Item extends Receive
{
private const LOCATION = 'local';
/** @var Address The address that sent us this item */
private Address $ao;
private string $recvas;
private int $recvmtime;
private int $recvsize;
private Filesystem $fs;
/**
* @throws \Exception
@ -29,19 +29,20 @@ final class Item extends Receive
$this->recvsize = $size;
$this->ftype = self::IS_FILE;
$this->fs = Storage::disk(config('fido.local_disk'));
}
public function __get($key) {
switch ($key) {
case 'exists':
return Storage::disk(self::LOCATION)->exists($this->rel_name);
return $this->fs->exists($this->rel_name);
case 'stor_name':
case 'pref_name':
return sprintf('%04X-%s',$this->ao->id,$this->recvas);
case 'rel_name':
return sprintf('%s/%s',config('fido.dir'),$this->stor_name);
return sprintf('%s/%s',config('fido.dir'),$this->pref_name);
case 'full_name':
return Storage::disk(self::LOCATION)->path($this->rel_name);
return $this->fs->path($this->rel_name);
case 'match_mtime':
return $this->mtime === $this->recvmtime;
@ -59,10 +60,10 @@ final class Item extends Receive
return sprintf('%s %lu %lu',$this->recvas,$this->recvsize,$this->recvmtime);
case 'mtime':
return Storage::disk(self::LOCATION)->lastModified($this->rel_name);
return $this->fs->lastModified($this->rel_name);
case 'size':
return Storage::disk(self::LOCATION)->size($this->rel_name);
return $this->fs->size($this->rel_name);
default:
return parent::__get($key);

2
app/Classes/File/Receive.php
View File

@ -60,7 +60,7 @@ class Receive extends Base
case 'nameas':
case 'size':
case 'name_size_time':
case 'stor_name':
case 'pref_name':
return $this->receiving->{$key};
case 'pos':

2
app/Jobs/PacketProcess.php
View File

@ -137,7 +137,7 @@ class PacketProcess implements ShouldQueue
try {
if ($fs->makeDirectory($dir)) {
$fs->move($this->file->rel_name,$x=sprintf('%s/%s',$dir,$this->file->stor_name));
$fs->move($this->file->rel_name,$x=sprintf('%s/%s',$dir,$this->file->pref_name));
Log::info(sprintf('%s:- Moved processed packet [%s] to [%s]',self::LOGKEY,$this->file->rel_name,$x));
} else

10
app/Jobs/TicProcess.php
View File

@ -10,6 +10,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use App\Classes\FTN\Tic;
use App\Models\Domain;
@ -27,7 +28,7 @@ class TicProcess implements ShouldQueue
/**
* Create a new job instance.
*
* @param string $file
* @param string $file Relative to Storage::disk('local'), ie: storage/app
* @param string|null $domain
*/
public function __construct(private string $file,private ?string $domain=NULL)
@ -56,12 +57,15 @@ class TicProcess implements ShouldQueue
*/
public function handle()
{
$fs = Storage::disk(config('fido.local_disk'));
$to = new Tic;
$to->load(storage_path('app').'/'.$this->file);
$to->load($this->file);
Log::info(sprintf('%s:= Processed [%s] storing [%s] as id [%d]',self::LOGKEY,$this->file,$to->fo->name,$to->fo->id));
unlink(storage_path('app').'/'.$this->file);
if (! $fs->delete($this->file))
Log::alert(sprintf('%s:! Failed to delete [%s]',self::LOGKEY,$this->file));
if ($to->isNodelist())
NodelistImport::dispatch($to->fo,$this->domain);

132
app/Models/File.php
View File

@ -5,6 +5,8 @@ namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
@ -20,7 +22,8 @@ class File extends Model
private const LOGKEY = 'MF-';
private bool $no_export = FALSE;
public string $prefix = '';
public string $replaces = '';
public Collection $set_path;
public Collection $set_seenby;
protected $casts = [
'kludges' => CollectionOrNull::class,
@ -37,82 +40,108 @@ class File extends Model
'ldesc',
];
public function __set($key,$value)
{
switch ($key) {
case 'fullname':
case 'replaces':
case 'no_export':
case 'set_path':
case 'set_seenby':
$this->{$key} = $value;
break;
default:
parent::__set($key,$value);
}
}
public static function boot()
{
parent::boot();
static::creating(function($model) {
Log::info(sprintf('%s:- Storing file [%s] in [%s]',self::LOGKEY,$model->fullname,$model->full_storage_path));
// Store file
if (Storage::put($model->full_storage_path,Storage::disk('local')->get($model->fullname),'public')) {
unlink(Storage::disk('local')->path($model->fullname));
} else {
throw new \Exception(sprintf('Unable to move file [%s] to [%s]',$model->fullname,$model->full_storage_path));
if (! $model->filearea_id) {
Log::alert(sprintf('%s:- File has no filearea, not processing creating [%s]',self::LOGKEY,$model->name));
return;
}
Log::info(sprintf('%s:- Storing file [%s] in [%s]',self::LOGKEY,$model->recvd_rel_name,$model->rel_name));
$srcfs = Storage::disk(config('fido.local_disk'));
$tgtfs = Storage::disk(config('fido.file_disk'));
// Delete anything being replaced
// @todo implement replace
foreach (self::where('name',$model->name)->where('filearea_id',$model->filearea_id)->get() as $fo) {
Log::info(sprintf('%s:%% Deleting old file record [%d] for file [%s]',self::LOGKEY,$fo->id,$fo->rel_name));
$tgtfs->move($fo->rel_name,$fo->relname.'.'.$fo->id);
$fo->delete();
}
// Store file
if ($tgtfs->put($model->rel_name,$srcfs->get($model->recvd_rel_name),'public')) {
$srcfs->delete($model->recvd_rel_name);
} else {
throw new \Exception(sprintf('Unable to move file [%s] to [%s]',$model->recvd_rel_name,$model->rel_name));
}
});
// @todo if the file is updated with new SEEN-BY's from another route, we'll delete the pending export for systems (if there is one)
static::created(function($model) {
if (! $model->filearea_id) {
Log::alert(sprintf('%s:- File has no filearea, not exporting',self::LOGKEY,$model->id));
Log::alert(sprintf('%s:- File has no filearea, not exporting [%d]',self::LOGKEY,$model->id));
return;
}
$rogue = collect();
$seenby = collect();
$path = collect();
$so = Setup::findOrFail(config('app.id'));
$zone = $model->fftn->zone;
// Our address
$ftns = $so
->system
->match($model->fftn->zone);
// Parse PATH
foreach ($model->set_path as $line) {
$matches = [];
preg_match(sprintf('#^(%s)\ ?([0-9]+)\ ?(.*)$#',Address::ftn_regex),$line,$matches);
// Add our address to the seenby;
$model->set_seenby = $model->set_seenby->merge($ftns->pluck('id'))->unique();
$model->set_path = $model->set_path->merge([[
'address'=>$ftns->first(),
'datetime'=>$x=Carbon::now(),
'extra'=>sprintf('%s %s (%s)',$x->toRfc7231String(),$so::PRODUCT_NAME,$so->version),
]]);
if ($x=Arr::get($matches,1)) {
$ftn = Address::parseFTN($x);
// Make sure all the path is in the seenby
$model->set_seenby = $model->set_seenby->merge($model->set_path->pluck('address.id'))->unique()->filter();
// If domain should be flattened, look for node regardless of zone (within the list of zones for the domain)
$ao = ($zone->domain->flatten)
? Address::findZone($zone->domain,$ftn['n'],$ftn['f'],0)
: Address::findFTN($x);
// Save the seenby
$model->seenby()->sync($model->set_seenby);
if (! $ao)
$ao = Address::createFTN($x,System::createUnknownSystem());
$path->push(['address'=>$ao,'datetime'=>Carbon::createFromTimestamp($matches[9]),'extra'=>$matches[10]]);
}
}
// Save the Path
$ppoid = NULL;
foreach ($model->set_path as $path) {
foreach ($path as $item) {
$po = DB::select('INSERT INTO file_path (file_id,address_id,parent_id,datetime,extra) VALUES (?,?,?,?,?) RETURNING id',[
$model->id,
$path['address']->id,
$item['address']->id,
$ppoid,
Carbon::createFromTimestamp($path['datetime']),
$path['extra'],
$item['datetime'],
$item['extra'],
]);
$ppoid = $po[0]->id;
}
// Make sure all the path is in the seenby
// Add zone to seenby
$model->set_seenby = $model->set_seenby->merge($path->pluck('address.ftn3d'))->unique()->filter();
foreach ($model->set_seenby as $sb) {
$ftn = Address::parseFTN($x);
$ao = ($zone->domain->flatten)
? Address::findZone($zone->domain,$ftn['n'],$ftn['f'],0)
: Address::findFTN($x);
if ($ao)
$seenby->push($ao->id);
else
$rogue->push($sb);
}
$model->rogue_seenby = $rogue;
$model->seenby()->sync($seenby);
$model->save();
// See if we need to export this file.
if ($model->filearea->sec_read) {
$exportto = $model
@ -169,7 +198,7 @@ class File extends Model
*
* @return string
*/
public function getFullStoragePathAttribute(): string
public function getRelNameAttribute(): string
{
return sprintf('%04X/%s',$this->filearea_id,$this->name);
}
@ -179,12 +208,17 @@ class File extends Model
*
* @return string
*/
public function getRelNameAttribute(): string
public function getRecvdRelNameAttribute(): string
{
return sprintf('%s/%s',config('fido.dir'),$this->prefix_name);
return sprintf('%s/%s',config('fido.dir'),$this->recvd_pref_name);
}
public function getPrefixNameAttribute(): string
/**
* This is the name of the file, with the sender prefix
*
* @return string
*/
public function getRecvdPrefNameAttribute(): string
{
return sprintf('%s%s',$this->prefix ? $this->prefix.'-' : '',$this->name);
}

6
app/Traits/Import.php
View File

@ -86,12 +86,12 @@ trait Import
if ($file instanceof File) {
$path = sprintf('import/%s.%d',$key,$file->id);
Storage::disk('local')->put($path,Storage::get($file->full_storage_path));
Storage::disk(config('fido.local_disk'))->put($path,Storage::get($file->rel_name));
return Storage::disk('local')->path($path);
return Storage::disk(config('fido.local_disk'))->path($path);
} else {
return Storage::disk('local')->path($file);
return Storage::disk(config('fido.local_disk'))->path($file);
}
}
}

3
config/fido.php
View File

@ -10,6 +10,9 @@ return [
// Directory to use for any data we want to store locally
'dir' => env('FIDO_DIR', 'fido'),
// Our Storage::disk() for files storage
'file_disk' => env('FIDO_DIR_FILES', 's3'),
// Our Storage::disk() for locally stored data
'local_disk' => env('FIDO_LOCAL_DIR','local'),

34
database/migrations/2023_09_22_233947_file_tweeks.php Normal file
View File

@ -0,0 +1,34 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('files', function (Blueprint $table) {
$table->dropUnique(['filearea_id','name']);
$table->string('replaces')->nullable();
});
DB::statement("CREATE UNIQUE INDEX files_active ON files (filearea_id, name) WHERE deleted_at IS NULL");
}
/**
* Reverse the migrations.
*/
public function down(): void
{
DB::statement("DROP INDEX files_active");
Schema::table('files', function (Blueprint $table) {
$table->unique(['filearea_id','name']);
$table->dropColumn('replaces');
});
}
};