Mail bundling and processing performance improvements

app/Classes/FTN/Packet.php

@@ -52,6 +52,7 @@ abstract class Packet extends FTNBase implements \Iterator, \Countable
 	protected Address $fftn_p;			// Address the packet is from (when packing messages)
 	protected Address $tftn_p;			// Address the packet is to (when packing messages)
 	protected Collection $messages;		// Messages in the Packet
+	protected string $content;			// Outgoing packet data
 	public Collection $errors;			// Messages that fail validation
 	protected int $index;				// Our array index
 	protected $pass_p = NULL;			// Overwrite the packet password (when packing messages)
@@ -67,7 +68,7 @@ abstract class Packet extends FTNBase implements \Iterator, \Countable
 	 * @return bool
 	 */
 	abstract public static function is_type(string $header): bool;
-	abstract protected function header(): string;
+	abstract protected function header(Collection $msgs): string;
 
 	/* STATIC */
 
@@ -287,20 +288,7 @@ abstract class Packet extends FTNBase implements \Iterator, \Countable
 	 */
 	public function __toString(): string
 	{
-		if (empty($this->messages))
-			throw new InvalidPacketException('Refusing to make an empty packet');
-
-		if (empty($this->tftn_p) || empty($this->fftn_p))
-			throw new InvalidPacketException('Cannot generate a packet without a destination address');
-
-		$return = $this->header();
-
-		foreach ($this->messages as $o)
-			$return .= self::PACKED_MSG_LEAD.$o->packet($this->tftn_p);
-
-		$return .= "\00\00";
-
-		return $return;
+		return $this->content;
 	}
 
 	/* INTERFACE */
@@ -360,7 +348,20 @@ abstract class Packet extends FTNBase implements \Iterator, \Countable
 
 	public function mail(Collection $msgs): self
 	{
-		$this->messages = $msgs;
+		if (! $msgs->count())
+			throw new InvalidPacketException('Refusing to make an empty packet');
+
+		if (empty($this->tftn_p) || empty($this->fftn_p))
+			throw new InvalidPacketException('Cannot generate a packet without a destination address');
+
+		$this->content = $this->header($msgs);
+
+		foreach ($msgs as $o)
+			$this->content .= self::PACKED_MSG_LEAD.$o->packet($this->tftn_p);
+
+		$this->content .= "\00\00";
+
+		$this->messages = $msgs->map(fn($item)=>$item->only(['id','date']));
 
 		return $this;
 	}
@@ -381,7 +382,6 @@ abstract class Packet extends FTNBase implements \Iterator, \Countable
 		if ($msg->errors->count()) {
 			Log::info(sprintf('%s:- Message [%s] has [%d] errors',self::LOGKEY,$msg->msgid ?: 'No ID',$msg->errors->count()));
 
-
 			// If the messages is not for the right zone, we'll ignore it
 			if ($msg->errors->has('invalid-zone')) {
 				Log::alert(sprintf('%s:! Message [%s] is from an invalid zone [%s], packet is from [%s] - ignoring it',self::LOGKEY,$msg->msgid,$msg->fftn->zone->zone_id,$this->fftn->zone->zone_id));

app/Classes/FTN/Packet/FSC39.php

@@ -3,6 +3,7 @@
 namespace App\Classes\FTN\Packet;
 
 use Illuminate\Support\Arr;
+use Illuminate\Support\Collection;
 
 use App\Classes\FTN\Packet;
 use App\Models\Setup;
@@ -62,11 +63,10 @@ final class FSC39 extends Packet
 	/**
 	 * Create our message packet header
 	 */
-	protected function header(): string
+	protected function header(Collection $msgs): string
 	{
 		$oldest = $this->messages->sortBy('datetime')->last();
 
-		try {
 		return pack(collect(self::HEADER)->pluck(1)->join(''),
 			$this->fftn_p->node_id,				// Orig Node
 			$this->tftn_p->node_id,						// Dest Node
@@ -96,10 +96,6 @@ final class FSC39 extends Packet
 			$this->tftn_p->point_id,					// Dest Point
 			strtoupper(hexstr(Setup::PRODUCT_ID)),	// ProdData
 		);
-
-		} catch (\Exception $e) {
-			return $e->getMessage();
-		}
 	}
 
 	/**

app/Classes/FTN/Packet/FSC45.php

@@ -3,6 +3,7 @@
 namespace App\Classes\FTN\Packet;
 
 use Illuminate\Support\Arr;
+use Illuminate\Support\Collection;
 
 use App\Classes\FTN\Packet;
 use App\Models\Setup;
@@ -54,9 +55,8 @@ final class FSC45 extends Packet
 	/**
 	 * Create our message packet header
 	 */
-	protected function header(): string
+	protected function header(Collection $msgs): string
 	{
-		try {
 		return pack(collect(self::HEADER)->pluck(1)->join(''),
 			$this->fftn_p->node_id,				// Orig Node
 			$this->tftn_p->node_id,						// Dest Node
@@ -76,10 +76,6 @@ final class FSC45 extends Packet
 			$this->tftn_p->zone->domain->name,			// Dest Domain
 			strtoupper(hexstr(Setup::PRODUCT_ID)),	// ProdData
 		);
-
-		} catch (\Exception $e) {
-			return $e->getMessage();
-		}
 	}
 
 	/**

app/Classes/FTN/Packet/FSC48.php

@@ -3,6 +3,7 @@
 namespace App\Classes\FTN\Packet;
 
 use Illuminate\Support\Arr;
+use Illuminate\Support\Collection;
 
 use App\Classes\FTN\Packet;
 use App\Models\Setup;
@@ -62,11 +63,10 @@ final class FSC48 extends Packet
 	/**
 	 * Create our message packet header
 	 */
-	protected function header(): string
+	protected function header(Collection $msgs): string
 	{
-		$oldest = $this->messages->sortBy('datetime')->last();
+		$oldest = $msgs->sortBy('date')->last();
 
-		try {
 		return pack(collect(self::HEADER)->pluck(1)->join(''),
 			$this->fftn_p->node_id,				// Orig Node
 			$this->tftn_p->node_id,						// Dest Node
@@ -96,10 +96,6 @@ final class FSC48 extends Packet
 			$this->tftn_p->point_id,					// Dest Point
 			strtoupper(hexstr(Setup::PRODUCT_ID)),	// ProdData
 		);
-
-		} catch (\Exception $e) {
-			return $e->getMessage();
-		}
 	}
 
 	/**

app/Classes/FTN/Packet/FTS1.php

@@ -3,6 +3,7 @@
 namespace App\Classes\FTN\Packet;
 
 use Illuminate\Support\Arr;
+use Illuminate\Support\Collection;
 
 use App\Classes\FTN\Packet;
 use App\Models\Setup;
@@ -55,11 +56,10 @@ final class FTS1 extends Packet
 	/**
 	 * Create our message packet header
 	 */
-	protected function header(): string
+	protected function header(Collection $msgs): string
 	{
 		$oldest = $this->messages->sortBy('datetime')->last();
 
-		try {
 		return pack(collect(self::HEADER)->pluck(1)->join(''),
 			$this->fftn_p->node_id,				// Orig Node
 			$this->tftn_p->node_id,						// Dest Node
@@ -80,10 +80,6 @@ final class FTS1 extends Packet
 			$this->tftn_p->zone->zone_id,				// Dest Zone
 			'',											// Reserved
 		);
-
-		} catch (\Exception $e) {
-			return $e->getMessage();
-		}
 	}
 
 	/**

app/Classes/FTN/Process.php

@@ -11,7 +11,7 @@ abstract class Process
 {
 	public static function canProcess(Echoarea $eao): bool
 	{
-		return $eao->automsgs ? TRUE : FALSE;
+		return (bool)$eao->automsgs;
 	}
 
 	/**

app/Classes/File/Mail.php

@@ -44,9 +44,6 @@ final class Mail extends Send
 			case 'mtime':
 				return $this->youngest()->timestamp;
 
-			case 'size':
-				return strlen($this->f);
-
 			case 'type':
 				return ($this->ftype&0xff00)>>8;
 
@@ -99,6 +96,7 @@ final class Mail extends Send
 	public function open(string $compress=''): bool
 	{
 		$this->content = (string)$this->f;
+		$this->size = strlen($this->content);
 		return TRUE;
 	}
 

app/Classes/File/Send.php

@@ -34,6 +34,7 @@ class Send extends Base
 	public const T_ECHOMAIL			= (1<<2);
 
 	private string $comp_data;
+	protected int $size = 0;
 
 	public function __construct()
 	{

app/Console/Commands/Debug/PacketDump.php

@@ -16,8 +16,7 @@ class PacketDump extends Command
 	protected $signature = 'debug:packet:dump'.
 		' {type : Type of packet, netmail|echomail }'.
 		' {ftn : FTN}'.
-		' {file? : filename}'.
-		' {--dump : Dump packet}';
+		' {file? : filename}';
 
 	/**
 	 * The console command description.
@@ -50,11 +49,11 @@ class PacketDump extends Command
 				throw new \Exception('Unknown type: '.$this->argument('type'));
 		}
 
-		if ($this->option('dump')) {
+		if (! $this->argument('file')) {
 			$this->info('Item Name:'.$pkt->name);
 			$this->info('Item Type:'.get_class($pkt));
 			$this->info('Dump:');
-			echo hex_dump($pkt);
+			echo hex_dump((string)$pkt);
 
 		} else {
 			$f = fopen($this->argument('file'),'w+');

app/Http/Controllers/DomainController.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Controllers;
 
+use Carbon\Carbon;
 use Illuminate\Http\Request;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Gate;
@@ -14,7 +15,7 @@ class DomainController extends Controller
 	/**
 	 * Daily stats as shown on the about page
 	 *
-	 * @param Domain $o
+	 * @param Request $request
 	 * @return Collection
 	 */
 	public function api_daily_stats(Request $request): Collection
@@ -25,7 +26,7 @@ class DomainController extends Controller
 			->sortBy('date')
 			->groupBy('date')
 			->transform(function($item,$key) { return [
-				'x'=>\Carbon\Carbon::createFromFormat('Y-m-d',$key)->timestamp,
+				'x'=>Carbon::createFromFormat('Y-m-d',$key)->timestamp*1000,
 				'y'=>$item->sum('count')]; } )
 			->values();
 	}

app/Models/Address.php

@@ -209,6 +209,7 @@ class Address extends Model
 			$o = $query
 				->where('region_id',$ftn['n'])
 				->where('host_id',$ftn['n'])
+				->with(['system:id,active'])
 				->first();
 
 		// Look for a normal address
@@ -223,15 +224,16 @@ class Address extends Model
 						})
 						->orWhere('host_id',$ftn['n']);
 				})
+				->with(['system:id,active'])
 				->first();
 
 		// Check and see if we are a flattened domain, our address might be available with a different zone.
 		// This occurs when we are parsing 2D addresses from SEEN-BY, but we have the zone
 		if (! $o && ($ftn['p'] === 0)) {
 			if ($ftn['d'])
-				$do = Domain::where(['name'=>$ftn['d']])->single();
+				$do = Domain::select('flatten')->where(['name'=>$ftn['d']])->single();
 			else {
-				$zo = Zone::where('zone_id',$ftn['z'])->where('default',TRUE)->single();
+				$zo = Zone::where('zone_id',$ftn['z'])->where('default',TRUE)->with(['domain:id,flatten'])->single();
 				$do = $zo?->domain;
 			}
 
@@ -393,8 +395,11 @@ class Address extends Model
 	 */
 	public function scopeFTN($query)
 	{
-		return $query->select(['addresses.zone_id','host_id','node_id','point_id'])
-			->with('zone.domain');
+		return $query->select(['id','addresses.zone_id','host_id','node_id','point_id'])
+			->with([
+				'zone:zones.id,domain_id,zone_id',
+				'zone.domain:domains.id,name',
+			]);
 	}
 
 	public function scopeFTNOrder($query)
@@ -597,9 +602,9 @@ class Address extends Model
 	public function nodes_hub(): HasMany
 	{
 		return $this->hasMany(Address::class,'hub_id','id')
+			->FTN()
 			->active()
-			->FTNorder()
-			->with(['zone.domain']);
+			->FTNorder();
 	}
 
 	/**
@@ -618,7 +623,7 @@ class Address extends Model
 					->whereNot('id',$this->id)
 					->active()
 					->FTNorder()
-					->with(['zone.domain']),
+					->with(['zone:id,domain_id,zone_id']),
 				$this,
 				NULL,
 				($this->role_id === self::NODE_NC) ? 'id' : NULL)
@@ -642,7 +647,7 @@ class Address extends Model
 					->whereNot('id',$this->id)
 					->active()
 					->FTNorder()
-					->with(['zone.domain']),
+					->with(['zone:id,domain_id,zone_id']),
 				$this,
 				NULL,
 				($this->role_id !== self::NODE_POINT) ? 'id' : NULL)
@@ -659,7 +664,7 @@ class Address extends Model
 					->whereNot('id',$this->id)
 					->active()
 					->FTNorder()
-					->with(['zone.domain']),
+					->with(['zone:id,domain_id,zone_id']),
 				$this,
 				NULL,
 				($this->role_id === self::NODE_RC) ? 'id' : NULL)
@@ -675,7 +680,7 @@ class Address extends Model
 					->whereNot('id',$this->id)
 					->active()
 					->FTNorder()
-					->with(['zone.domain']),
+					->with(['zone:id,domain_id,zone_id']),
 				$this,
 				NULL,
 				($this->role_id === self::NODE_ZC) ? 'id' : NULL)
@@ -722,6 +727,9 @@ class Address extends Model
 	 */
 	public function getFTNAttribute(): string
 	{
+		if (! $this->relationLoaded('zone'))
+			$this->load(['zone:id,domain_id,zone_id','zone.domain:domains.id,name']);
+
 		return sprintf('%s@%s',$this->getFTN4DAttribute(),$this->zone->domain->name);
 	}
 
@@ -732,11 +740,17 @@ class Address extends Model
 
 	public function getFTN3DAttribute(): string
 	{
+		if (! $this->relationLoaded('zone'))
+			$this->load(['zone:id,domain_id,zone_id']);
+
 		return sprintf('%d:%s',$this->zone->zone_id,$this->getFTN2DAttribute());
 	}
 
 	public function getFTN4DAttribute(): string
 	{
+		if (! $this->relationLoaded('zone'))
+			$this->load(['zone:id,domain_id,zone_id']);
+
 		return sprintf('%s.%d',$this->getFTN3DAttribute(),$this->point_id);
 	}
 
@@ -998,20 +1012,28 @@ class Address extends Model
 	/**
 	 * Echomail waiting to be sent to this system
 	 *
-	 * @param int|null $max
 	 * @return Builder
 	 */
-	public function echomailWaiting(int $max=NULL): Builder
+	public function echomailWaiting(): Builder
 	{
-		$echomails = $this
-			->UncollectedEchomail()
-			->select('echomails.id')
-			->where('addresses.id',$this->id)
-			->when($max,function($query) use ($max) { return $query->limit($max); })
-			->groupBy(['echomails.id'])
-			->get();
-
-		return Echomail::whereIn('id',$echomails->pluck('id'));
+		return Echomail::select('echomails.*')
+			->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
+			->where('address_id',$this->id)
+			->whereNull('echomails.deleted_at')
+			->whereNotNull('export_at')
+			->whereNull('sent_at')
+			->orderby('id')
+			->with([
+				'tagline:id,value',
+				'tearline:id,value',
+				'origin:id,value',
+				'echoarea:id,name,domain_id',
+				'echoarea.domain:id,name',
+				'fftn:id,zone_id,host_id,node_id,point_id',
+				'fftn.zone:id,domain_id,zone_id',
+				'fftn.zone.domain:id,name',
+			])
+			->dontCache();
 	}
 
 	/**
@@ -1076,11 +1098,11 @@ class Address extends Model
 	 */
 	public function getEchomail(): ?Packet
 	{
-		if (($num=$this->echomailWaiting())->count()) {
-			Log::info(sprintf('%s:= Got [%d] echomails for [%s] for sending',self::LOGKEY,$num->count(),$this->ftn));
+		if ($count=($num=$this->echomailWaiting())->count()) {
+			Log::info(sprintf('%s:= Got [%d] echomails for [%s] for sending',self::LOGKEY,$count,$this->ftn));
 
 			// Limit to max messages
-			if ($num->count() > $this->system->pkt_msgs)
+			if ($count > $this->system->pkt_msgs)
 				Log::notice(sprintf('%s:= Only sending [%d] echomails for [%s]',self::LOGKEY,$this->system->pkt_msgs,$this->ftn));
 
 			return $this->system->packet($this)->mail($num->take($this->system->pkt_msgs)->get());

app/Models/Domain.php

@@ -62,7 +62,8 @@ class Domain extends Model
 
 	public function zones()
 	{
-		return $this->hasMany(Zone::class);
+		return $this->hasMany(Zone::class)
+			->select(['id','zone_id','domain_id']);
 	}
 
 	/* ATTRIBUTES */

app/Models/Echomail.php

@@ -280,12 +280,15 @@ final class Echomail extends Model implements Packet
 
 	public function echoarea()
 	{
-		return $this->belongsTo(Echoarea::class);
+		return $this->belongsTo(Echoarea::class)
+			->select('id','active','name','domain_id','security','automsgs')
+			->with(['domain:id,name']);
 	}
 
 	public function seenby()
 	{
 		return $this->belongsToMany(Address::class,'echomail_seenby')
+			->select(['id','zone_id','host_id','node_id'])
 			->withPivot(['export_at','sent_at','sent_pkt'])
 			->FTN2DOrder();
 	}
@@ -293,6 +296,7 @@ final class Echomail extends Model implements Packet
 	public function path()
 	{
 		return $this->belongsToMany(Address::class,'echomail_path')
+			->select(['addresses.id','zone_id','host_id','node_id'])
 			->withPivot(['id','parent_id','recv_pkt','recv_at'])
 			->orderBy('id','DESC');
 	}

app/Models/System.php

@@ -110,6 +110,7 @@ class System extends Model
 	public function sessions()
 	{
 		return $this->belongsToMany(Zone::class)
+			->select(['id','zones.zone_id','domain_id','active'])
 			->withPivot(['sespass','pktpass','ticpass','fixpass','zt_ipv4','zt_ipv6','default'])
 			->dontCache();
 	}

app/Traits/MessageAttributes.php

@@ -160,7 +160,7 @@ trait MessageAttributes
 	 */
 	public function packet(Address $ao): Message
 	{
-		Log::debug(sprintf('%s:+ Bundling [%s] for [%s]',self::LOGKEY,$this->id,$ao->ftn),['type'=>get_class($this)]);
+		Log::debug(sprintf('%s:+ Bundling [%s] for [%s]',self::LOGKEY,$this->id,$ao->ftn3d),['type'=>get_class($this)]);
 
 		// For echomail, our tftn is this address
 		if ($this instanceof Echomail)

app/helpers.php

@@ -98,7 +98,12 @@ function our_address(Domain|Address $o=NULL): Collection|Address|NULL
 	});
 
 	$our = Cache::remember(sprintf('%d-akas',$so->system_id),60,function() use ($so) {
-		$so->load(['system.akas.zone.domain']);
+		$so->load([
+			'system:id,name',
+			'system.akas:addresses.id,addresses.zone_id,host_id,node_id,point_id,addresses.system_id,addresses.active,role',
+			'system.akas.zone:id,domain_id,zone_id',
+			'system.akas.zone.domain:id,name',
+		]);
 
 		return $so->system->akas;
 	});
@@ -114,16 +119,20 @@ function our_address(Domain|Address $o=NULL): Collection|Address|NULL
 	// We are requesting a list of addresses for a Domain, or a specific Address, and we have more than 1
 	switch (get_class($o)) {
 		case Address::class:
-			$filter = $our->filter(function($item) use ($o) { return $item->zone->domain_id === $o->zone->domain_id; })->sortBy('role');
+			$filter = $our
+				->filter(fn($item)=>$item->zone->domain_id === $o->zone->domain_id)
+				->sortBy('role_id');
 
 			// If we are looking for a specific address, and there is only 1 result, return it, otherwise return what we have
-			if (config('fido.strict') && ($x=$filter->filter(function($item) use ($o) { return $item->role_id <= $o->role_id; })->sortBy('role'))->count())
+			if (config('fido.strict') && ($x=$filter->filter(fn($item)=>$item->role_id <= $o->role_id)->sortBy('role_id'))->count())
 				$filter = $x;
 
-			return $filter->last();
+			return $filter->count() ? $filter->last()->unsetRelation('nodes_hub') : NULL;
 
 		case Domain::class:
-			return $our->filter(function($item) use ($o) { return $item->zone->domain_id === $o->id; })->sortBy('role');
+			return $our
+				->filter(fn($item)=>$item->zone->domain_id === $o->id)
+				->sortBy('role_id');
 
 		// We shouldnt get here
 		default:
@@ -171,11 +180,10 @@ function our_nodes(Domain $do=NULL): Collection
 {
 	return Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','addresses.system_id','role'])
 		->join('system_zone',['system_zone.system_id'=>'addresses.system_id','system_zone.zone_id'=>'addresses.zone_id'])
-		->when(! is_null($do),function($query) use ($do) {
-			return $query
+		->when(! is_null($do),
+			fn($query)=>$query
 				->join('zones',['zones.id'=>'addresses.zone_id'])
-				->where('domain_id',$do->id);
-		})
+				->where('domain_id',$do->id))
 		->active()
 		->FTNorder()
 		->get();

resources/views/domain/view.blade.php

@@ -338,7 +338,7 @@
 					'name'=>$key,
 					'dashStyle'=>'ShortDot',
 					'visible'=>$item->first()->show,
-					'data'=>$item->map(function($item) { return ['x'=>\Carbon\Carbon::createFromFormat('Y-m-d',$item->date)->timestamp,'y'=>$item->count]; })
+					'data'=>$item->map(function($item) { return ['x'=>\Carbon\Carbon::createFromFormat('Y-m-d',$item->date)->timestamp*1000,'y'=>$item->count]; })
 				]; })
 				->values() !!}
 		});