Rework crash polling, using optimised scope queries

app/Http/Controllers/HomeController.php

@@ -248,55 +248,31 @@ class HomeController extends Controller
 	{
 		$date = Carbon::now()->yesterday()->endOfday();
 
-		// Nodes with uncollected mail
-		$e = Address::select(['addresses.id',DB::raw('count(*) as e'),DB::raw('0 as n'),DB::raw('0 as f')])
-			->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id'])
-			->join('echomails',['echomails.id'=>'echomail_seenby.echomail_id'])
-			->whereNotNull('export_at')
-			->whereNull('sent_at')
-			->whereNull('echomails.deleted_at')
-			->where('echomails.created_at','<',$date)
-			->groupBy('addresses.id');
-
-		$n = Address::select(['addresses.id',DB::raw('0 as e'),DB::raw('count(*) as n'),DB::raw('0 as f')])
-			->join('netmails',['netmails.tftn_id'=>'addresses.id'])
-			->where(function($query) {
-				return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
-					->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL));
-			})
-			->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
-			->whereNull('netmails.deleted_at')
-			->where('netmails.created_at','<',$date)
-			->groupBy('addresses.id');
-
-		$f = Address::select(['addresses.id',DB::raw('0 as e'),DB::raw('0 as n'),DB::raw('count(*) as f')])
-			->join('file_seenby',['file_seenby.address_id'=>'addresses.id'])
-			->join('files',['files.id'=>'file_seenby.file_id'])
-			->whereNotNull('export_at')
-			->whereNull('sent_at')
-			->whereNull('files.deleted_at')
-			->where('files.created_at','<',$date)
-			->groupBy('addresses.id');
-
 		$r = Address::select([
 				'a.id',
 				'addresses.system_id',
 				'addresses.zone_id',
+				'addresses.region_id',
 				'addresses.host_id',
 				'addresses.node_id',
-				DB::raw('sum(a.e) as e'),
+				'addresses.point_id',
-				DB::raw('sum(a.n) as n'),
+				'addresses.hub_id',
-				DB::raw('sum(a.f) as f')
+				'addresses.role',
+				DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
+				DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
+				DB::raw('sum(a.uncollected_files) as uncollected_files')
 			])
-			->from($e->union($n)->union($f),'a')
+			->from(Address::UncollectedEchomail()->union(Address::UncollectedNetmail())->union(Address::UncollectedFiles()),'a')
+			->where('systems.active',true)
 			->where('addresses.active',TRUE)
 			->where('zones.active',TRUE)
 			->where('domains.active',TRUE)
 			->when(! ($x=Auth::user()) || (! $x->isAdmin()),function($query) { return $query->where('domains.public',TRUE); })
 			->join('addresses',['addresses.id'=>'a.id'])
+			->join('systems',['systems.id'=>'addresses.system_id'])
 			->join('zones',['zones.id'=>'addresses.zone_id'])
 			->join('domains',['domains.id'=>'zones.domain_id'])
-			->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.host_id','addresses.node_id')
+			->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','addresses.hub_id','addresses.role')
 			->with(['system','zone.domain']);
 
 		return view('status')

app/Jobs/MailSend.php

@@ -12,7 +12,6 @@ use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Repat\LaravelJobs\Job;
 
-use App\Classes\FTN\Message;
 use App\Models\Address;
 
 class MailSend implements ShouldQueue
@@ -47,24 +46,33 @@ class MailSend implements ShouldQueue
 	 */
 	public function handle(): void
 	{
-		// Netmail waiting by node (only netmail is routed)
+		$u = Address::select([
-		$netmails = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(netmails.id) AS nm')])
+			'a.id',
-			->join('zones',['zones.id'=>'addresses.zone_id'])
+			'addresses.system_id',
-			->join('domains',['domains.id'=>'zones.domain_id'])
+			'addresses.zone_id',
-			->join('netmails',['netmails.tftn_id'=>'addresses.id'])
+			'addresses.region_id',
-			->join('systems',['systems.id'=>'addresses.system_id'])
+			'addresses.host_id',
+			'addresses.node_id',
+			'addresses.point_id',
+			'addresses.hub_id',
+			'addresses.role',
+			DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
+			DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
+			DB::raw('sum(a.uncollected_files) as uncollected_files')
+		])
+			->from(Address::UncollectedEchomail()->union(Address::UncollectedNetmail())->union(Address::UncollectedFiles()),'a')
+			->where('systems.active',true)
 			->where('addresses.active',TRUE)
 			->where('zones.active',TRUE)
 			->where('domains.active',TRUE)
+			->join('addresses',['addresses.id'=>'a.id'])
+			->join('systems',['systems.id'=>'addresses.system_id'])
+			->join('zones',['zones.id'=>'addresses.zone_id'])
+			->join('domains',['domains.id'=>'zones.domain_id'])
 			->where(function($query) {
 				return $query->whereNull('autohold')
 					->orWhere('autohold',FALSE);
 			})
-			->where(function($query) {
-				return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
-					->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL));
-			})
-			->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
 			->when(! is_null($this->crash),function($query) {
 				return $query->when(
 					$this->crash,
@@ -76,115 +84,40 @@ class MailSend implements ShouldQueue
 					}
 				);
 			})
-			->groupBy('addresses.id')
+			->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','addresses.hub_id','addresses.role')
-			->havingRaw('count(*) > 0')
+			->with(['system','zone.domain'])
 			->get();
 
-		// Echomail waiting by node
+		// Return the system we poll
-		$echomails = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(*) AS em')])
+		$u = $u->transform(function($item) {
-			->distinct()
+			if ($x=$item->parent()) {
-			->join('zones',['zones.id'=>'addresses.zone_id'])
+				$x->uncollected_echomail = $item->uncollected_echomail;
-			->join('domains',['domains.id'=>'zones.domain_id'])
+				$x->uncollected_netmail = $item->uncollected_netmail;
-			->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id'])
+				$x->uncollected_files = $item->uncollected_files;
-			->join('systems',['systems.id'=>'addresses.system_id'])
-			->where('addresses.active',TRUE)
-			->where('zones.active',TRUE)
-			->where('domains.active',TRUE)
-			->where(function($query) {
-				return $query->whereNull('autohold')
-					->orWhere('autohold',FALSE);
-			})
-			->whereNotNull('export_at')
-			->whereNull('sent_at')
-			->when(! is_null($this->crash),function($query) {
-				return $query->when(
-					$this->crash,
-					function($query) {
-						return $query->where('pollmode',$this->crash);
-					},
-					function($query) {
-						return $query->whereNotNull('pollmode');
-					}
-				);
-			})
-			->groupBy(['addresses.id'])
-			->havingRaw('count(*) > 0')
-			->FTNOrder()
-			->get();
 
-		// Files waiting by node
+				return $x;
-		$files = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(*) AS fs')])
-			->distinct()
-			->join('zones',['zones.id'=>'addresses.zone_id'])
-			->join('domains',['domains.id'=>'zones.domain_id'])
-			->join('file_seenby',['file_seenby.address_id'=>'addresses.id'])
-			->join('systems',['systems.id'=>'addresses.system_id'])
-			->where('addresses.active',TRUE)
-			->where('zones.active',TRUE)
-			->where('domains.active',TRUE)
-			->where(function($query) {
-				return $query->whereNull('autohold')
-					->orWhere('autohold',FALSE);
-			})
-			->whereNotNull('export_at')
-			->whereNull('sent_at')
-			->when(! is_null($this->crash),function($query) {
-				return $query->when(
-					$this->crash,
-					function($query) {
-						return $query->where('pollmode',$this->crash);
-					},
-					function($query) {
-						return $query->whereNotNull('pollmode');
-					}
-				);
-			})
-			->groupBy(['addresses.id'])
-			->havingRaw('count(*) > 0')
-			->FTNOrder()
-			->get();
 
-		// Merge our netmails
-		foreach ($echomails as $ao) {
-			if (($x=$netmails->search(function($item) use ($ao) { return $item->id === $ao->id; })) !== FALSE) {
-				$netmails->get($x)->em = $ao->em;
 			} else {
-				$netmails->push($ao);
+				return $item;
-			}
 			}
+		});
 
-		// Merge our files
+		foreach ($u->groupBy('ftn') as $oo) {
-		foreach ($files as $ao) {
+			if (Job::where('queue','poll')->get()->pluck('command.address.id')->search(($x=$oo->first())->id) === FALSE) {
-			if (($x=$netmails->search(function($item) use ($ao) { return $item->id === $ao->id; })) !== FALSE) {
-				$netmails->get($x)->fs = $ao->fs;
-			} else {
-				$netmails->push($ao);
-			}
-		}
-
-		// Remove direct links
-		$netmails = $netmails->filter(function($item) { return $item->parent(); });
-
-		foreach ($netmails->groupBy(function($item) { return $item->parent()->ftn; }) as $oo) {
-			$ao = $oo->first();
-
 				Log::info(sprintf('%s:- Polling [%s] - we have mail for [%d] links. (%d Netmail,%d Echomail,%d Files)',
 					self::LOGKEY,
-				$ao->ftn,
+					$x->ftn,
 					$oo->count(),
-				$oo->sum('nm'),
+					$oo->sum('uncollected_netmail'),
-				$oo->sum('em'),
+					$oo->sum('uncollected_echomail'),
-				$oo->sum('fs'),
+					$oo->sum('uncollected_files'),
 				));
 
-			// @todo Only send crash mail - send normal mail with a schedule or hold mail
+				AddressPoll::dispatch($x);
 
-			if (Job::where('queue',$this->queue)->get()->pluck('command.address.id')->search($ao->id) !== FALSE) {
+			} else {
-				Log::alert(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$ao->ftn));
+				Log::notice(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$x->ftn));
-				continue;
+			}
-			}
-
-			AddressPoll::dispatch($ao);
 		}
 	}
 }

app/Models/Address.php

@@ -2,6 +2,7 @@
 
 namespace App\Models;
 
+use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Collection;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
@@ -23,7 +24,7 @@ class Address extends Model
 	protected $with = ['zone'];
 
 	// http://ftsc.org/docs/frl-1028.002
-	public const ftn_regex = '(\d+):(\d+)/(\d+)(?:\.(\d+))?(?:@([a-z0-9\-_~]{0,8}))?';
+	public const ftn_regex = '(\d+):(\d+)/(\d+)(?:\.(\d+))?(?:@([a-zA-Z0-9\-_~]{0,8}))?';
 
 	public const NODE_ZC		= 1<<0;		// Zone
 	public const NODE_RC		= 1<<1;		// Region
@@ -65,13 +66,6 @@ class Address extends Model
 			->FTNorder();
 	}
 
-	public function scopeTrashed($query)
-	{
-		return $query->select($this->getTable().'.*')
-			->withTrashed()
-			->FTNorder();
-	}
-
 	public function scopeFTNOrder($query)
 	{
 		return $query
@@ -89,6 +83,72 @@ class Address extends Model
 			->orderBy('point_id');
 	}
 
+	public function scopeTrashed($query)
+	{
+		return $query->select($this->getTable().'.*')
+			->withTrashed()
+			->FTNorder();
+	}
+
+	/**
+	 * Return a list of addresses and the amount of uncollected echomail
+	 *
+	 * @param $query
+	 * @return mixed
+	 */
+	public function scopeUncollectedEchomail($query)
+	{
+		return $query
+			->select(['addresses.id',DB::raw('count(*) as uncollected_echomail'),DB::raw('0 as uncollected_netmail'),DB::raw('0 as uncollected_files')])
+				->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id'])
+				->join('echomails',['echomails.id'=>'echomail_seenby.echomail_id'])
+				->whereNotNull('export_at')
+				->whereNull('sent_at')
+				->whereNull('echomails.deleted_at')
+				->where('echomails.created_at','<',$this->yesterdayEOD())
+				->groupBy('addresses.id');
+	}
+
+	/**
+	 * Return a list of addresses and the amount of uncollected files
+	 *
+	 * @param $query
+	 * @return mixed
+	 */
+	public function scopeUncollectedFiles($query)
+	{
+		return $query
+			->select(['addresses.id',DB::raw('0 as uncollected_echomail'),DB::raw('0 as uncollected_netmail'),DB::raw('count(*) as uncollected_files')])
+			->join('file_seenby',['file_seenby.address_id'=>'addresses.id'])
+			->join('files',['files.id'=>'file_seenby.file_id'])
+			->whereNotNull('export_at')
+			->whereNull('sent_at')
+			->whereNull('files.deleted_at')
+			->where('files.created_at','<',$this->yesterdayEOD())
+			->groupBy('addresses.id');
+	}
+
+	/**
+	 * Return a list of addresses and the amount of uncollected netmail
+	 *
+	 * @param $query
+	 * @return mixed
+	 */
+	public function scopeUncollectedNetmail($query)
+	{
+		return $query
+			->select(['addresses.id',DB::raw('0 as uncollected_echomail'),DB::raw('count(*) as uncollected_netmail'),DB::raw('0 as uncollected_files')])
+			->join('netmails',['netmails.tftn_id'=>'addresses.id'])
+			->where(function($query) {
+				return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
+					->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL));
+			})
+			->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
+			->whereNull('netmails.deleted_at')
+			->where('netmails.created_at','<',$this->yesterdayEOD())
+			->groupBy('addresses.id');
+	}
+
 	/* RELATIONS */
 
 	/**
@@ -285,13 +345,13 @@ class Address extends Model
 			// Normal Nodes
 			case self::NODE_ACTIVE:
 				// If we are a child of a hub, then check our hub
-				$parent = (($this->hub_id)
+				$parent = ($this->hub_id
 					? self::where('id',$this->hub_id)
 					: self::where('zone_id',$this->zone_id)
 					->where('region_id',$this->region_id)
 					->where('host_id',$this->host_id)
-					->where('node_id',0))
+					->where('node_id',0)
-					->where('role','<',$this->role)
+					->where('role','<',$this->role))
 					->single();
 
 				break;
@@ -865,4 +925,9 @@ class Address extends Model
 	{
 		return ($this->exists && ($x=$this->system->sessions->where('id',$this->zone_id)->first())) ? ($x->pivot->{$type} ?: '') : NULL;
 	}
+
+	private function yesterdayEOD(): Carbon
+	{
+		return Carbon::now()->yesterday()->endOfday();
+	}
 }

resources/views/status.blade.php

@@ -33,11 +33,16 @@
 				@foreach($uncollected as $o)
 					<tr>
 						<td>{{ $o->zone->domain->name }}</td>
-						<td><a href="{{ url('system/addedit',$o->system_id) }}">{{ $o->system->name }}</a></td>
+						<td>
+							<a href="{{ url('system/addedit',$o->system_id) }}">{{ $o->system->name }}</a>
+							@if (($x=$o->parent()) && ($x->id !== $o->id))
+								<br><small>[via <a href="{{ url('system/addedit',$x->system_id) }}">{{ $x->ftn4d }}</a>]</small>
+							@endif
+						</td>
 						<td>{{ $o->ftn4d }}</td>
-						<td class="text-right">{{ number_format($o->e ?? 0) }}</td>
+						<td class="text-right">{{ number_format($o->uncollected_echomail ?? 0) }}</td>
-						<td class="text-right">{{ number_format($o->n ?? 0) }}</td>
+						<td class="text-right">{{ number_format($o->uncollected_netmail ?? 0) }}</td>
-						<td class="text-right">{{ number_format($o->f ?? 0) }}</td>
+						<td class="text-right">{{ number_format($o->uncollected_files ?? 0) }}</td>
 						<td>{{ $o->system->last_session?->format('Y-m-d H:i') }}</td>
 						<td>{{ is_null($o->system->pollmode) ? 'HOLD' : ($o->system->pollmode ? 'CRASH' : 'DAILY') }}</td>
 						<td>{{ $o->system->autohold ? 'YES' : 'NO' }}</td>