From 2b2482ba71d1fe3bff06eba37b28b7693a90fa78 Mon Sep 17 00:00:00 2001 From: Deon George Date: Sat, 25 Nov 2023 21:52:05 +1100 Subject: [PATCH] Rework crash polling, using optimised scope queries --- app/Http/Controllers/HomeController.php | 46 ++----- app/Jobs/MailSend.php | 157 +++++++----------------- app/Models/Address.php | 87 +++++++++++-- resources/views/status.blade.php | 13 +- 4 files changed, 141 insertions(+), 162 deletions(-) diff --git a/app/Http/Controllers/HomeController.php b/app/Http/Controllers/HomeController.php index 402cacc..61a77ec 100644 --- a/app/Http/Controllers/HomeController.php +++ b/app/Http/Controllers/HomeController.php @@ -248,55 +248,31 @@ class HomeController extends Controller { $date = Carbon::now()->yesterday()->endOfday(); - // Nodes with uncollected mail - $e = Address::select(['addresses.id',DB::raw('count(*) as e'),DB::raw('0 as n'),DB::raw('0 as f')]) - ->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id']) - ->join('echomails',['echomails.id'=>'echomail_seenby.echomail_id']) - ->whereNotNull('export_at') - ->whereNull('sent_at') - ->whereNull('echomails.deleted_at') - ->where('echomails.created_at','<',$date) - ->groupBy('addresses.id'); - - $n = Address::select(['addresses.id',DB::raw('0 as e'),DB::raw('count(*) as n'),DB::raw('0 as f')]) - ->join('netmails',['netmails.tftn_id'=>'addresses.id']) - ->where(function($query) { - return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT)) - ->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL)); - }) - ->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT)) - ->whereNull('netmails.deleted_at') - ->where('netmails.created_at','<',$date) - ->groupBy('addresses.id'); - - $f = Address::select(['addresses.id',DB::raw('0 as e'),DB::raw('0 as n'),DB::raw('count(*) as f')]) - ->join('file_seenby',['file_seenby.address_id'=>'addresses.id']) - ->join('files',['files.id'=>'file_seenby.file_id']) - ->whereNotNull('export_at') - ->whereNull('sent_at') - ->whereNull('files.deleted_at') - ->where('files.created_at','<',$date) - ->groupBy('addresses.id'); - $r = Address::select([ 'a.id', 'addresses.system_id', 'addresses.zone_id', + 'addresses.region_id', 'addresses.host_id', 'addresses.node_id', - DB::raw('sum(a.e) as e'), - DB::raw('sum(a.n) as n'), - DB::raw('sum(a.f) as f') + 'addresses.point_id', + 'addresses.hub_id', + 'addresses.role', + DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'), + DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'), + DB::raw('sum(a.uncollected_files) as uncollected_files') ]) - ->from($e->union($n)->union($f),'a') + ->from(Address::UncollectedEchomail()->union(Address::UncollectedNetmail())->union(Address::UncollectedFiles()),'a') + ->where('systems.active',true) ->where('addresses.active',TRUE) ->where('zones.active',TRUE) ->where('domains.active',TRUE) ->when(! ($x=Auth::user()) || (! $x->isAdmin()),function($query) { return $query->where('domains.public',TRUE); }) ->join('addresses',['addresses.id'=>'a.id']) + ->join('systems',['systems.id'=>'addresses.system_id']) ->join('zones',['zones.id'=>'addresses.zone_id']) ->join('domains',['domains.id'=>'zones.domain_id']) - ->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.host_id','addresses.node_id') + ->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','addresses.hub_id','addresses.role') ->with(['system','zone.domain']); return view('status') diff --git a/app/Jobs/MailSend.php b/app/Jobs/MailSend.php index 4d41bd3..435181c 100644 --- a/app/Jobs/MailSend.php +++ b/app/Jobs/MailSend.php @@ -12,7 +12,6 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Repat\LaravelJobs\Job; -use App\Classes\FTN\Message; use App\Models\Address; class MailSend implements ShouldQueue @@ -47,24 +46,33 @@ class MailSend implements ShouldQueue */ public function handle(): void { - // Netmail waiting by node (only netmail is routed) - $netmails = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(netmails.id) AS nm')]) - ->join('zones',['zones.id'=>'addresses.zone_id']) - ->join('domains',['domains.id'=>'zones.domain_id']) - ->join('netmails',['netmails.tftn_id'=>'addresses.id']) - ->join('systems',['systems.id'=>'addresses.system_id']) + $u = Address::select([ + 'a.id', + 'addresses.system_id', + 'addresses.zone_id', + 'addresses.region_id', + 'addresses.host_id', + 'addresses.node_id', + 'addresses.point_id', + 'addresses.hub_id', + 'addresses.role', + DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'), + DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'), + DB::raw('sum(a.uncollected_files) as uncollected_files') + ]) + ->from(Address::UncollectedEchomail()->union(Address::UncollectedNetmail())->union(Address::UncollectedFiles()),'a') + ->where('systems.active',true) ->where('addresses.active',TRUE) ->where('zones.active',TRUE) ->where('domains.active',TRUE) + ->join('addresses',['addresses.id'=>'a.id']) + ->join('systems',['systems.id'=>'addresses.system_id']) + ->join('zones',['zones.id'=>'addresses.zone_id']) + ->join('domains',['domains.id'=>'zones.domain_id']) ->where(function($query) { return $query->whereNull('autohold') ->orWhere('autohold',FALSE); }) - ->where(function($query) { - return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT)) - ->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL)); - }) - ->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT)) ->when(! is_null($this->crash),function($query) { return $query->when( $this->crash, @@ -76,115 +84,40 @@ class MailSend implements ShouldQueue } ); }) - ->groupBy('addresses.id') - ->havingRaw('count(*) > 0') + ->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','addresses.hub_id','addresses.role') + ->with(['system','zone.domain']) ->get(); - // Echomail waiting by node - $echomails = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(*) AS em')]) - ->distinct() - ->join('zones',['zones.id'=>'addresses.zone_id']) - ->join('domains',['domains.id'=>'zones.domain_id']) - ->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id']) - ->join('systems',['systems.id'=>'addresses.system_id']) - ->where('addresses.active',TRUE) - ->where('zones.active',TRUE) - ->where('domains.active',TRUE) - ->where(function($query) { - return $query->whereNull('autohold') - ->orWhere('autohold',FALSE); - }) - ->whereNotNull('export_at') - ->whereNull('sent_at') - ->when(! is_null($this->crash),function($query) { - return $query->when( - $this->crash, - function($query) { - return $query->where('pollmode',$this->crash); - }, - function($query) { - return $query->whereNotNull('pollmode'); - } - ); - }) - ->groupBy(['addresses.id']) - ->havingRaw('count(*) > 0') - ->FTNOrder() - ->get(); + // Return the system we poll + $u = $u->transform(function($item) { + if ($x=$item->parent()) { + $x->uncollected_echomail = $item->uncollected_echomail; + $x->uncollected_netmail = $item->uncollected_netmail; + $x->uncollected_files = $item->uncollected_files; - // Files waiting by node - $files = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(*) AS fs')]) - ->distinct() - ->join('zones',['zones.id'=>'addresses.zone_id']) - ->join('domains',['domains.id'=>'zones.domain_id']) - ->join('file_seenby',['file_seenby.address_id'=>'addresses.id']) - ->join('systems',['systems.id'=>'addresses.system_id']) - ->where('addresses.active',TRUE) - ->where('zones.active',TRUE) - ->where('domains.active',TRUE) - ->where(function($query) { - return $query->whereNull('autohold') - ->orWhere('autohold',FALSE); - }) - ->whereNotNull('export_at') - ->whereNull('sent_at') - ->when(! is_null($this->crash),function($query) { - return $query->when( - $this->crash, - function($query) { - return $query->where('pollmode',$this->crash); - }, - function($query) { - return $query->whereNotNull('pollmode'); - } - ); - }) - ->groupBy(['addresses.id']) - ->havingRaw('count(*) > 0') - ->FTNOrder() - ->get(); + return $x; - // Merge our netmails - foreach ($echomails as $ao) { - if (($x=$netmails->search(function($item) use ($ao) { return $item->id === $ao->id; })) !== FALSE) { - $netmails->get($x)->em = $ao->em; } else { - $netmails->push($ao); + return $item; } - } + }); + + foreach ($u->groupBy('ftn') as $oo) { + if (Job::where('queue','poll')->get()->pluck('command.address.id')->search(($x=$oo->first())->id) === FALSE) { + Log::info(sprintf('%s:- Polling [%s] - we have mail for [%d] links. (%d Netmail,%d Echomail,%d Files)', + self::LOGKEY, + $x->ftn, + $oo->count(), + $oo->sum('uncollected_netmail'), + $oo->sum('uncollected_echomail'), + $oo->sum('uncollected_files'), + )); + + AddressPoll::dispatch($x); - // Merge our files - foreach ($files as $ao) { - if (($x=$netmails->search(function($item) use ($ao) { return $item->id === $ao->id; })) !== FALSE) { - $netmails->get($x)->fs = $ao->fs; } else { - $netmails->push($ao); + Log::notice(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$x->ftn)); } } - - // Remove direct links - $netmails = $netmails->filter(function($item) { return $item->parent(); }); - - foreach ($netmails->groupBy(function($item) { return $item->parent()->ftn; }) as $oo) { - $ao = $oo->first(); - - Log::info(sprintf('%s:- Polling [%s] - we have mail for [%d] links. (%d Netmail,%d Echomail,%d Files)', - self::LOGKEY, - $ao->ftn, - $oo->count(), - $oo->sum('nm'), - $oo->sum('em'), - $oo->sum('fs'), - )); - - // @todo Only send crash mail - send normal mail with a schedule or hold mail - - if (Job::where('queue',$this->queue)->get()->pluck('command.address.id')->search($ao->id) !== FALSE) { - Log::alert(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$ao->ftn)); - continue; - } - - AddressPoll::dispatch($ao); - } } } \ No newline at end of file diff --git a/app/Models/Address.php b/app/Models/Address.php index 6252b74..1603f9d 100644 --- a/app/Models/Address.php +++ b/app/Models/Address.php @@ -2,6 +2,7 @@ namespace App\Models; +use Carbon\Carbon; use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; @@ -23,7 +24,7 @@ class Address extends Model protected $with = ['zone']; // http://ftsc.org/docs/frl-1028.002 - public const ftn_regex = '(\d+):(\d+)/(\d+)(?:\.(\d+))?(?:@([a-z0-9\-_~]{0,8}))?'; + public const ftn_regex = '(\d+):(\d+)/(\d+)(?:\.(\d+))?(?:@([a-zA-Z0-9\-_~]{0,8}))?'; public const NODE_ZC = 1<<0; // Zone public const NODE_RC = 1<<1; // Region @@ -65,13 +66,6 @@ class Address extends Model ->FTNorder(); } - public function scopeTrashed($query) - { - return $query->select($this->getTable().'.*') - ->withTrashed() - ->FTNorder(); - } - public function scopeFTNOrder($query) { return $query @@ -89,6 +83,72 @@ class Address extends Model ->orderBy('point_id'); } + public function scopeTrashed($query) + { + return $query->select($this->getTable().'.*') + ->withTrashed() + ->FTNorder(); + } + + /** + * Return a list of addresses and the amount of uncollected echomail + * + * @param $query + * @return mixed + */ + public function scopeUncollectedEchomail($query) + { + return $query + ->select(['addresses.id',DB::raw('count(*) as uncollected_echomail'),DB::raw('0 as uncollected_netmail'),DB::raw('0 as uncollected_files')]) + ->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id']) + ->join('echomails',['echomails.id'=>'echomail_seenby.echomail_id']) + ->whereNotNull('export_at') + ->whereNull('sent_at') + ->whereNull('echomails.deleted_at') + ->where('echomails.created_at','<',$this->yesterdayEOD()) + ->groupBy('addresses.id'); + } + + /** + * Return a list of addresses and the amount of uncollected files + * + * @param $query + * @return mixed + */ + public function scopeUncollectedFiles($query) + { + return $query + ->select(['addresses.id',DB::raw('0 as uncollected_echomail'),DB::raw('0 as uncollected_netmail'),DB::raw('count(*) as uncollected_files')]) + ->join('file_seenby',['file_seenby.address_id'=>'addresses.id']) + ->join('files',['files.id'=>'file_seenby.file_id']) + ->whereNotNull('export_at') + ->whereNull('sent_at') + ->whereNull('files.deleted_at') + ->where('files.created_at','<',$this->yesterdayEOD()) + ->groupBy('addresses.id'); + } + + /** + * Return a list of addresses and the amount of uncollected netmail + * + * @param $query + * @return mixed + */ + public function scopeUncollectedNetmail($query) + { + return $query + ->select(['addresses.id',DB::raw('0 as uncollected_echomail'),DB::raw('count(*) as uncollected_netmail'),DB::raw('0 as uncollected_files')]) + ->join('netmails',['netmails.tftn_id'=>'addresses.id']) + ->where(function($query) { + return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT)) + ->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL)); + }) + ->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT)) + ->whereNull('netmails.deleted_at') + ->where('netmails.created_at','<',$this->yesterdayEOD()) + ->groupBy('addresses.id'); + } + /* RELATIONS */ /** @@ -285,13 +345,13 @@ class Address extends Model // Normal Nodes case self::NODE_ACTIVE: // If we are a child of a hub, then check our hub - $parent = (($this->hub_id) + $parent = ($this->hub_id ? self::where('id',$this->hub_id) : self::where('zone_id',$this->zone_id) ->where('region_id',$this->region_id) ->where('host_id',$this->host_id) - ->where('node_id',0)) - ->where('role','<',$this->role) + ->where('node_id',0) + ->where('role','<',$this->role)) ->single(); break; @@ -865,4 +925,9 @@ class Address extends Model { return ($this->exists && ($x=$this->system->sessions->where('id',$this->zone_id)->first())) ? ($x->pivot->{$type} ?: '') : NULL; } + + private function yesterdayEOD(): Carbon + { + return Carbon::now()->yesterday()->endOfday(); + } } diff --git a/resources/views/status.blade.php b/resources/views/status.blade.php index 75a8e0a..c917612 100644 --- a/resources/views/status.blade.php +++ b/resources/views/status.blade.php @@ -33,11 +33,16 @@ @foreach($uncollected as $o) {{ $o->zone->domain->name }} - {{ $o->system->name }} + + {{ $o->system->name }} + @if (($x=$o->parent()) && ($x->id !== $o->id)) +
[via {{ $x->ftn4d }}] + @endif + {{ $o->ftn4d }} - {{ number_format($o->e ?? 0) }} - {{ number_format($o->n ?? 0) }} - {{ number_format($o->f ?? 0) }} + {{ number_format($o->uncollected_echomail ?? 0) }} + {{ number_format($o->uncollected_netmail ?? 0) }} + {{ number_format($o->uncollected_files ?? 0) }} {{ $o->system->last_session?->format('Y-m-d H:i') }} {{ is_null($o->system->pollmode) ? 'HOLD' : ($o->system->pollmode ? 'CRASH' : 'DAILY') }} {{ $o->system->autohold ? 'YES' : 'NO' }}