Query optimisations for last_messages and traffic

.env.example

@@ -23,7 +23,7 @@ DB_MONGO_USERNAME=mongo
 DB_MONGO_PASSWORD=password
 
 BROADCAST_DRIVER=log
-CACHE_DRIVER=file
+CACHE_DRIVER=memcached
 QUEUE_CONNECTION=database
 SESSION_DRIVER=file
 SESSION_LIFETIME=120

app/Models/Domain.php

@@ -5,10 +5,8 @@ namespace App\Models;
 use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
-use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\DB;
-use MongoDB\BSON\UTCDateTime;
 
 use App\Traits\{QueryCacheableConfig,ScopeActive};
 
@@ -46,7 +44,7 @@ class Domain extends Model
 		return $this->hasMany(Zone::class);
 	}
 
-	/* CASTS */
+	/* ATTRIBUTES */
 
 	public function getHomePageAttribute($value)
 	{
@@ -60,82 +58,68 @@ class Domain extends Model
 
 	/* METHODS */
 
-	public function daily_area_stats(): Collection
+	/**
+	 * Get some message area stats for this domain
+	 *
+	 * @param bool $byarea
+	 * @param Collection|NULL $systems
+	 * @return Collection
+	 */
+	public function daily_area_stats(bool $byarea=FALSE,Collection $systems=NULL): Collection
 	{
 		if (! $this->echoareas->count())
 			return collect();
 
-		$key = sprintf('%s_%d','daily_area_stats',$this->id);
-
-Cache::forget($key);
-		return Cache::remember($key,self::CACHE_TIME,function() {
-			$gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime";
-
-			$echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')])
-				->whereIn('id',$this->echoareas->pluck('id')->toArray())
+		$echostats = Echomail::cacheFor(self::CACHE_TIME)->select([DB::raw('datetime::date as date'),'echoarea_id','echoareas.name',DB::raw('COUNT(*)')])
+				->join('echoareas',['echoareas.id'=>'echomails.echoarea_id'])
+				->join('domains',['domains.id'=>'echoareas.domain_id'])
+				->where('domain_id',$this->id)
+				->when($systems?->count(),function($query) use ($systems) { return $query->whereIn('fftn_id',$systems->pluck('addresses')->flatten()->pluck('id')->toArray()); })
 				->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())
-				->groupBy('datetime')
-				->orderBy('datetime')
+				->groupBy(['echoarea_id','echoareas.name','date'])
+				->orderBy('echoareas.name')
+				->orderBy('date')
+				->with(['echoarea'])
 				->get();
 
+		if ($byarea)
 			return $echostats
-				->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; })
+				->groupBy(['echoarea_id'])
+				->map(function($item,$key) {
+					return [
+						'name' => $item->first()->echoarea->name,
+						'data' => $item->groupby('date')->map(function($item) {
+							return [
+								'x' => Carbon::create($item->first()->date)->timestamp*1000,
+								'y' => $item->sum('count')
+							];
+						})->values(),
+						'dashStyle' => 'ShortDot',
+					];
+				})->values();
+
+		else
+			return $echostats
+				->groupBy('date')
+				->map(function($item) { return ['x'=>Carbon::create($item->first()->date)->timestamp*1000,'y'=>$item->sum('count')]; })
 				->values();
-		});
 	}
 
-	public function daily_echoarea_stats(Echoarea $o): Collection
+	/**
+	 * Get the latest message in each echomail area
+	 *
+	 * @return Collection
+	 */
+	public function latest_echomail_message(): Collection
 	{
-		if (! $this->echoareas->count())
-			return collect();
-
-		$key = sprintf('%s_%d-%d','daily_echoarea_stats',$this->id,$o->id);
-
-Cache::forget($key);
-		return Cache::remember($key,self::CACHE_TIME,function() use ($o) {
-			$gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime";
-
-			$echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')])
-				->whereIn('echoarea_id',[$o->id])
-				->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())
-				->groupBy('datetime')
-				->orderBy('datetime')
-				->get();
-
-			return $echostats
-				->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; })
-				->values();
-		});
-	}
-
-	public function stats(System $o=NULL): Collection
-	{
-		if (! $this->echoareas->count())
-			return collect();
-
-		$key = sprintf('%s_%d_%d','stats',$this->id,$o?->id);
-
-		return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() use ($o) {
-			$where = collect(['echoarea_id'=>$this->echoareas->pluck('id')->toArray()]);
-			$where->put('datetime',['$gte',new UTCDateTime(Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())]);
-
-			if ($o)
-				$where->put('fftn_id',$o->addresses()->pluck('id'));
-
-			$echostats = Echomail::countGroupBy(['echoarea_id'],$where->toArray());
-
-			return $this->echoareas->map(function($item) use ($echostats) {
-				$stats = $echostats->filter(function($x) use ($item) {
-					return $x->id->echoarea_id == $item->id;
-				});
-
-				$item->count = 0;
-
-				foreach ($stats as $o)
-					$item->count += $o->count;
-
-				return $item;
-			});
-		});
+		return Echoarea::cacheFor(self::CACHE_TIME)
+			->select([
+				'echoareas.*',DB::raw('max(datetime) as last_message')
+			])
+			->leftJoin('echomails',['echomails.echoarea_id'=>'echoareas.id'])
+			->where('domain_id',$this->id)
+			->groupBy('echoareas.id')
+			->orderBy('echoareas.name')
+			->get();
 	}
 }

app/Models/Echoarea.php

@@ -6,16 +6,18 @@ use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Collection;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
-use Illuminate\Support\Facades\Cache;
+use Rennokki\QueryCache\Traits\QueryCacheable;
 
 use App\Traits\ScopeActive;
 
 class Echoarea extends Model
 {
-	use SoftDeletes,ScopeActive;
+	use SoftDeletes,ScopeActive,QueryCacheable;
 
 	private const CACHE_TIME = 3600;
 
+	protected $dates = [ 'last_message' ];
+
 	/* RELATIONS */
 
 	public function addresses()
@@ -34,31 +36,30 @@ class Echoarea extends Model
 			->orderBy('datetime','ASC');
 	}
 
-	/* ATTRIBUTES */
-
-	public function getLastMessageAttribute(): ?Carbon
-	{
-		return $this->echomail?->last()->datetime;
-	}
-
 	/* METHODS */
 
 	public function messages_count(int $period): int
 	{
-		$key = sprintf('%s_%d_%d','echo_messages_count',$this->id,$period);
+		$eo = Echomail::cacheFor(self::CACHE_TIME)
+			->where('echoarea_id',$this->id);
 
-		return Cache::remember($key,self::CACHE_TIME,function() use ($period) {
-			switch ($period) {
-				case 1: // day
-					return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subDay())->count();
-				case 7: // week
-					return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subWeek())->count();
-				case 30: // month
-					return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subMonth())->count();
-				default:
-					return 0;
-			}
-		});
+		$dt = Carbon::now()->startOfday();
+
+		switch ($period) {
+			case 1: // day
+				$eo->where('datetime','>=',$dt->subDay());
+				break;
+			case 7: // week
+				$eo->where('datetime','>=',$dt->subWeek());
+				break;
+			case 30: // month
+				$eo->where('datetime','>=',$dt->subMonth());
+				break;
+			default:
+				return 0;
+		}
+
+		return $eo->count();
 	}
 
 	/**

app/Models/Echomail.php

@@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
+use Rennokki\QueryCache\Traits\QueryCacheable;
 
 use App\Classes\FTN\Message;
 use App\Interfaces\Packet;
@@ -14,7 +15,7 @@ use App\Traits\{EncodeUTF8,MsgID};
 
 final class Echomail extends Model implements Packet
 {
-	use SoftDeletes,EncodeUTF8,MsgID;
+	use SoftDeletes,EncodeUTF8,MsgID,QueryCacheable;
 
 	private const LOGKEY = 'ME-';
 	private array $set_seenby = [];

app/Models/System.php

@@ -8,11 +8,10 @@ use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Auth;
 
 use App\Http\Controllers\DomainController;
-use App\Traits\QueryCacheableConfig;
 
 class System extends Model
 {
-	use HasFactory,QueryCacheableConfig;
+	use HasFactory;
 
 	protected $dates = ['last_session'];
 

resources/views/about.blade.php

@@ -8,7 +8,7 @@
 
 	<p>Welcome to the FTN Clearing House.</p>
 
-	<div style="float:right; width: 50%; height: 20em;" id="network_traffic"></div>
+	<div class="float-end" id="network_traffic"></div>
 	<p>The FTN Clearing House (FCH) is both a FTN Mailer and FTN message tosser, where mail is stored internally in a DB. FCH can also hatch and toss files into FTN networks for both up/downstream nodes.</p>
 	<p>It was created as an idea to bring modern technology and capabilities to a legacy computing network that existed in the 1970's, 1980's and 1990's (before the Internet basically).</p>
 
@@ -130,7 +130,7 @@ If you have more than 1 BBS, then the Clearing House can receive all your mail f
 				}
 			},
 			series: [
-				@foreach (\App\Models\Domain::active()->public()->with(['echoareas'])->get() as $o)
+				@foreach (\App\Models\Domain::active()->public()->orderBy('name')->with(['echoareas'])->get() as $o)
 				{
 					name: '{{ $o->name }}',
 					data: {!! $o->daily_area_stats() !!},

resources/views/dashboard.blade.php

@@ -161,12 +161,11 @@
 						name: 'Networks',
 						colorByPoint: true,
 						data: [
-							@foreach(($xx=$user->systems->pluck('addresses')->flatten()->pluck('zone.domain')->sortBy('name')) as $oo)
-								@php($x = $oo->stats())
+							@foreach(($xx=$user->systems->pluck('addresses')->flatten()->pluck('zone.domain')->unique(function($item) { return $item->name; })->sortBy('name')) as $do)
 								{
-									name: '{{ $oo->name }}',
-									y: {{ $x->sum('count') }},
-									drilldown: 'n-{{ $oo->name }}',
+									name: '{{ $do->name }}',
+									y: {{ $do->daily_area_stats()->sum('y') }},
+									drilldown: 'n-{{ $do->name }}',
 								},
 							@endforeach
 						]
@@ -176,12 +175,11 @@
 						colorByPoint: true,
 						pointPlacement: 0.1,
 						data: [
-							@foreach($xx as $oo)
-								@php($x = $oo->stats($o))
+							@foreach($xx as $do)
 								{
-									name: '{{ $oo->name }}',
-									y: {{ $x->sum('count') }},
-									drilldown: 'ny-{{ $oo->name }}',
+									name: '{{ $do->name }}',
+									y: {{ $do->daily_area_stats(FALSE,$user->systems)->sum('y') }},
+									drilldown: 'ny-{{ $do->name }}',
 									color: Highcharts.color(Highcharts.getOptions().colors[{{$loop->index}}]).brighten(-0.2).get()
 							},
 							@endforeach
@@ -196,21 +194,19 @@
 						}
 					},
 					series: [
-						@foreach($xx as $oo)
-							@php($x = $oo->stats())
+						@foreach($xx as $do)
 							{
-								name: '{{ $oo->name }}',
-								id:  'n-{{ $oo->name }}',
-								data: {!! $x->sortBy('name')->map(function($item) use ($oo) { return ['name'=>$item->name,'y'=>$item->count,'drilldown'=>'e-'.$item->name]; })->values() !!}
+								name: '{{ $do->name }}',
+								id:  'n-{{ $do->name }}',
+								data: {!! $do->daily_area_stats(TRUE)->sortBy('name')->map(function($item) { return ['name'=>$item['name'],'y'=>$item['data']->sum('y'),'drilldown'=>'e-'.$item['name']]; })->values() !!}
 							},
 						@endforeach
 
-						@foreach($xx as $oo)
-							@php($x = $oo->stats($o))
+						@foreach($xx as $do)
 							{
-								name: '{{ $oo->name }}',
-								id:  'ny-{{ $oo->name }}',
-								data: {!! $x->sortBy('name')->map(function($item) { return ['name'=>$item->name,'y'=>$item->count,'drilldown'=>'ey-'.$item->name]; })->values() !!}
+								name: '{{ $do->name }}',
+								id:  'ny-{{ $do->name }}',
+								data: {!! $do->daily_area_stats(TRUE,$user->systems)->sortBy('name')->map(function($item) { return ['name'=>$item['name'],'y'=>$item['data']->sum('y'),'drilldown'=>'ey-'.$item['name']]; })->values() !!}
 							},
 						@endforeach
 					]

resources/views/domain/view.blade.php

@@ -15,7 +15,7 @@
 
 			<div id="collapse_about" class="accordion-collapse collapse show" aria-labelledby="about" data-bs-parent="#accordion_homepage">
 				<div class="accordion-body">
-					<div style="float:right; width: 50%; height: 20em;" id="network_traffic"></div>
+					<div class="float-end" style="max-height: 25em;" id="network_traffic"></div>
 					{!! \Illuminate\Mail\Markdown::parse($o->homepage) !!}
 				</div>
 			</div>
@@ -46,7 +46,7 @@
 							</thead>
 
 							<tbody>
-							@foreach ($o->echoareas->sortBy('name') as $oo)
+							@foreach ($o->latest_echomail_message() as $oo)
 								<tr>
 									<td style="width: 15%;"><a href="{{ url('ftn/echoarea/addedit',[$oo->id]) }}">{{ $oo->name }}</a></td>
 									<td>{{ $oo->description }}</td>
@@ -166,7 +166,7 @@
 	@css('datatables')
 	<style>
 		div#collapse_about {
-			min-height: 20em;
+			min-height: 25em;
 		}
 		div#collapse_about .collapse{
 			min-height: 0;
@@ -235,7 +235,8 @@
 				zoomType: 'x',
 				resetZoomButton: {
 					position: {
-						x: 0,
+						align: 'left',
+						x: -40,
 						y: -40,
 					}
 				}
@@ -263,7 +264,7 @@
 			legend: {
 				align: 'right',
 				layout: 'vertical',
-				symbolWidth: 40,
+				symbolWidth: 20,
 				floating: false,
 				navigation: {
 					arrowSize: 10
@@ -281,15 +282,7 @@
 					cursor: 'pointer'
 				}
 			},
-			series: [
-				@foreach($o->echoareas->sortBy('name') as $oo)
-				{
-					name: '{{ $oo->name }}',
-					data: {!! $o->daily_echoarea_stats($oo) !!},
-					dashStyle: 'ShortDot',
-				},
-				@endforeach
-			],
+			series: {!! $o->daily_area_stats('true') !!}
 		});
 	</script>
 @append