From 6b0bf32552247e09d24b53e4f804fde82e5bbdff Mon Sep 17 00:00:00 2001 From: Deon George Date: Thu, 6 Jan 2022 00:19:57 +1100 Subject: [PATCH] Query optimisations for last_messages and traffic --- .env.example | 2 +- app/Models/Domain.php | 120 +++++++++++--------------- app/Models/Echoarea.php | 45 +++++----- app/Models/Echomail.php | 3 +- app/Models/System.php | 3 +- resources/views/about.blade.php | 4 +- resources/views/dashboard.blade.php | 36 ++++---- resources/views/domain/view.blade.php | 21 ++--- 8 files changed, 104 insertions(+), 130 deletions(-) diff --git a/.env.example b/.env.example index 960a5be..32be104 100644 --- a/.env.example +++ b/.env.example @@ -23,7 +23,7 @@ DB_MONGO_USERNAME=mongo DB_MONGO_PASSWORD=password BROADCAST_DRIVER=log -CACHE_DRIVER=file +CACHE_DRIVER=memcached QUEUE_CONNECTION=database SESSION_DRIVER=file SESSION_LIFETIME=120 diff --git a/app/Models/Domain.php b/app/Models/Domain.php index 35eb4db..6d2254a 100644 --- a/app/Models/Domain.php +++ b/app/Models/Domain.php @@ -5,10 +5,8 @@ namespace App\Models; use Carbon\Carbon; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; -use Illuminate\Support\Facades\Cache; use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; -use MongoDB\BSON\UTCDateTime; use App\Traits\{QueryCacheableConfig,ScopeActive}; @@ -46,7 +44,7 @@ class Domain extends Model return $this->hasMany(Zone::class); } - /* CASTS */ + /* ATTRIBUTES */ public function getHomePageAttribute($value) { @@ -60,82 +58,68 @@ class Domain extends Model /* METHODS */ - public function daily_area_stats(): Collection + /** + * Get some message area stats for this domain + * + * @param bool $byarea + * @param Collection|NULL $systems + * @return Collection + */ + public function daily_area_stats(bool $byarea=FALSE,Collection $systems=NULL): Collection { if (! $this->echoareas->count()) return collect(); - $key = sprintf('%s_%d','daily_area_stats',$this->id); - -Cache::forget($key); - return Cache::remember($key,self::CACHE_TIME,function() { - $gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime"; - - $echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')]) - ->whereIn('id',$this->echoareas->pluck('id')->toArray()) + $echostats = Echomail::cacheFor(self::CACHE_TIME)->select([DB::raw('datetime::date as date'),'echoarea_id','echoareas.name',DB::raw('COUNT(*)')]) + ->join('echoareas',['echoareas.id'=>'echomails.echoarea_id']) + ->join('domains',['domains.id'=>'echoareas.domain_id']) + ->where('domain_id',$this->id) + ->when($systems?->count(),function($query) use ($systems) { return $query->whereIn('fftn_id',$systems->pluck('addresses')->flatten()->pluck('id')->toArray()); }) ->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth()) - ->groupBy('datetime') - ->orderBy('datetime') + ->groupBy(['echoarea_id','echoareas.name','date']) + ->orderBy('echoareas.name') + ->orderBy('date') + ->with(['echoarea']) ->get(); + if ($byarea) return $echostats - ->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; }) + ->groupBy(['echoarea_id']) + ->map(function($item,$key) { + return [ + 'name' => $item->first()->echoarea->name, + 'data' => $item->groupby('date')->map(function($item) { + return [ + 'x' => Carbon::create($item->first()->date)->timestamp*1000, + 'y' => $item->sum('count') + ]; + })->values(), + 'dashStyle' => 'ShortDot', + ]; + })->values(); + + else + return $echostats + ->groupBy('date') + ->map(function($item) { return ['x'=>Carbon::create($item->first()->date)->timestamp*1000,'y'=>$item->sum('count')]; }) ->values(); - }); } - public function daily_echoarea_stats(Echoarea $o): Collection + /** + * Get the latest message in each echomail area + * + * @return Collection + */ + public function latest_echomail_message(): Collection { - if (! $this->echoareas->count()) - return collect(); - - $key = sprintf('%s_%d-%d','daily_echoarea_stats',$this->id,$o->id); - -Cache::forget($key); - return Cache::remember($key,self::CACHE_TIME,function() use ($o) { - $gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime"; - - $echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')]) - ->whereIn('echoarea_id',[$o->id]) - ->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth()) - ->groupBy('datetime') - ->orderBy('datetime') - ->get(); - - return $echostats - ->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; }) - ->values(); - }); - } - - public function stats(System $o=NULL): Collection - { - if (! $this->echoareas->count()) - return collect(); - - $key = sprintf('%s_%d_%d','stats',$this->id,$o?->id); - - return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() use ($o) { - $where = collect(['echoarea_id'=>$this->echoareas->pluck('id')->toArray()]); - $where->put('datetime',['$gte',new UTCDateTime(Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())]); - - if ($o) - $where->put('fftn_id',$o->addresses()->pluck('id')); - - $echostats = Echomail::countGroupBy(['echoarea_id'],$where->toArray()); - - return $this->echoareas->map(function($item) use ($echostats) { - $stats = $echostats->filter(function($x) use ($item) { - return $x->id->echoarea_id == $item->id; - }); - - $item->count = 0; - - foreach ($stats as $o) - $item->count += $o->count; - - return $item; - }); - }); + return Echoarea::cacheFor(self::CACHE_TIME) + ->select([ + 'echoareas.*',DB::raw('max(datetime) as last_message') + ]) + ->leftJoin('echomails',['echomails.echoarea_id'=>'echoareas.id']) + ->where('domain_id',$this->id) + ->groupBy('echoareas.id') + ->orderBy('echoareas.name') + ->get(); } } \ No newline at end of file diff --git a/app/Models/Echoarea.php b/app/Models/Echoarea.php index 39dd5b0..9b21685 100644 --- a/app/Models/Echoarea.php +++ b/app/Models/Echoarea.php @@ -6,16 +6,18 @@ use Carbon\Carbon; use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; -use Illuminate\Support\Facades\Cache; +use Rennokki\QueryCache\Traits\QueryCacheable; use App\Traits\ScopeActive; class Echoarea extends Model { - use SoftDeletes,ScopeActive; + use SoftDeletes,ScopeActive,QueryCacheable; private const CACHE_TIME = 3600; + protected $dates = [ 'last_message' ]; + /* RELATIONS */ public function addresses() @@ -34,31 +36,30 @@ class Echoarea extends Model ->orderBy('datetime','ASC'); } - /* ATTRIBUTES */ - - public function getLastMessageAttribute(): ?Carbon - { - return $this->echomail?->last()->datetime; - } - /* METHODS */ public function messages_count(int $period): int { - $key = sprintf('%s_%d_%d','echo_messages_count',$this->id,$period); + $eo = Echomail::cacheFor(self::CACHE_TIME) + ->where('echoarea_id',$this->id); - return Cache::remember($key,self::CACHE_TIME,function() use ($period) { - switch ($period) { - case 1: // day - return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subDay())->count(); - case 7: // week - return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subWeek())->count(); - case 30: // month - return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subMonth())->count(); - default: - return 0; - } - }); + $dt = Carbon::now()->startOfday(); + + switch ($period) { + case 1: // day + $eo->where('datetime','>=',$dt->subDay()); + break; + case 7: // week + $eo->where('datetime','>=',$dt->subWeek()); + break; + case 30: // month + $eo->where('datetime','>=',$dt->subMonth()); + break; + default: + return 0; + } + + return $eo->count(); } /** diff --git a/app/Models/Echomail.php b/app/Models/Echomail.php index 09b7a76..790d6a3 100644 --- a/app/Models/Echomail.php +++ b/app/Models/Echomail.php @@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; +use Rennokki\QueryCache\Traits\QueryCacheable; use App\Classes\FTN\Message; use App\Interfaces\Packet; @@ -14,7 +15,7 @@ use App\Traits\{EncodeUTF8,MsgID}; final class Echomail extends Model implements Packet { - use SoftDeletes,EncodeUTF8,MsgID; + use SoftDeletes,EncodeUTF8,MsgID,QueryCacheable; private const LOGKEY = 'ME-'; private array $set_seenby = []; diff --git a/app/Models/System.php b/app/Models/System.php index 81d60f4..cba4f93 100644 --- a/app/Models/System.php +++ b/app/Models/System.php @@ -8,11 +8,10 @@ use Illuminate\Support\Collection; use Illuminate\Support\Facades\Auth; use App\Http\Controllers\DomainController; -use App\Traits\QueryCacheableConfig; class System extends Model { - use HasFactory,QueryCacheableConfig; + use HasFactory; protected $dates = ['last_session']; diff --git a/resources/views/about.blade.php b/resources/views/about.blade.php index 2c31c8e..10369af 100644 --- a/resources/views/about.blade.php +++ b/resources/views/about.blade.php @@ -8,7 +8,7 @@

Welcome to the FTN Clearing House.

-
+

The FTN Clearing House (FCH) is both a FTN Mailer and FTN message tosser, where mail is stored internally in a DB. FCH can also hatch and toss files into FTN networks for both up/downstream nodes.

It was created as an idea to bring modern technology and capabilities to a legacy computing network that existed in the 1970's, 1980's and 1990's (before the Internet basically).

@@ -130,7 +130,7 @@ If you have more than 1 BBS, then the Clearing House can receive all your mail f } }, series: [ - @foreach (\App\Models\Domain::active()->public()->with(['echoareas'])->get() as $o) + @foreach (\App\Models\Domain::active()->public()->orderBy('name')->with(['echoareas'])->get() as $o) { name: '{{ $o->name }}', data: {!! $o->daily_area_stats() !!}, diff --git a/resources/views/dashboard.blade.php b/resources/views/dashboard.blade.php index c917220..5fe5485 100644 --- a/resources/views/dashboard.blade.php +++ b/resources/views/dashboard.blade.php @@ -161,12 +161,11 @@ name: 'Networks', colorByPoint: true, data: [ - @foreach(($xx=$user->systems->pluck('addresses')->flatten()->pluck('zone.domain')->sortBy('name')) as $oo) - @php($x = $oo->stats()) + @foreach(($xx=$user->systems->pluck('addresses')->flatten()->pluck('zone.domain')->unique(function($item) { return $item->name; })->sortBy('name')) as $do) { - name: '{{ $oo->name }}', - y: {{ $x->sum('count') }}, - drilldown: 'n-{{ $oo->name }}', + name: '{{ $do->name }}', + y: {{ $do->daily_area_stats()->sum('y') }}, + drilldown: 'n-{{ $do->name }}', }, @endforeach ] @@ -176,12 +175,11 @@ colorByPoint: true, pointPlacement: 0.1, data: [ - @foreach($xx as $oo) - @php($x = $oo->stats($o)) + @foreach($xx as $do) { - name: '{{ $oo->name }}', - y: {{ $x->sum('count') }}, - drilldown: 'ny-{{ $oo->name }}', + name: '{{ $do->name }}', + y: {{ $do->daily_area_stats(FALSE,$user->systems)->sum('y') }}, + drilldown: 'ny-{{ $do->name }}', color: Highcharts.color(Highcharts.getOptions().colors[{{$loop->index}}]).brighten(-0.2).get() }, @endforeach @@ -196,21 +194,19 @@ } }, series: [ - @foreach($xx as $oo) - @php($x = $oo->stats()) + @foreach($xx as $do) { - name: '{{ $oo->name }}', - id: 'n-{{ $oo->name }}', - data: {!! $x->sortBy('name')->map(function($item) use ($oo) { return ['name'=>$item->name,'y'=>$item->count,'drilldown'=>'e-'.$item->name]; })->values() !!} + name: '{{ $do->name }}', + id: 'n-{{ $do->name }}', + data: {!! $do->daily_area_stats(TRUE)->sortBy('name')->map(function($item) { return ['name'=>$item['name'],'y'=>$item['data']->sum('y'),'drilldown'=>'e-'.$item['name']]; })->values() !!} }, @endforeach - @foreach($xx as $oo) - @php($x = $oo->stats($o)) + @foreach($xx as $do) { - name: '{{ $oo->name }}', - id: 'ny-{{ $oo->name }}', - data: {!! $x->sortBy('name')->map(function($item) { return ['name'=>$item->name,'y'=>$item->count,'drilldown'=>'ey-'.$item->name]; })->values() !!} + name: '{{ $do->name }}', + id: 'ny-{{ $do->name }}', + data: {!! $do->daily_area_stats(TRUE,$user->systems)->sortBy('name')->map(function($item) { return ['name'=>$item['name'],'y'=>$item['data']->sum('y'),'drilldown'=>'ey-'.$item['name']]; })->values() !!} }, @endforeach ] diff --git a/resources/views/domain/view.blade.php b/resources/views/domain/view.blade.php index b2894ec..9cc81dd 100644 --- a/resources/views/domain/view.blade.php +++ b/resources/views/domain/view.blade.php @@ -15,7 +15,7 @@
-
+
{!! \Illuminate\Mail\Markdown::parse($o->homepage) !!}
@@ -46,7 +46,7 @@ - @foreach ($o->echoareas->sortBy('name') as $oo) + @foreach ($o->latest_echomail_message() as $oo) {{ $oo->name }} {{ $oo->description }} @@ -166,7 +166,7 @@ @css('datatables')