From 1ac3583479ded42c434f5bc7189546e8ab52d4b0 Mon Sep 17 00:00:00 2001 From: Deon George Date: Sun, 26 Nov 2023 13:10:23 +1100 Subject: [PATCH] Implemented system heartbeat, to poll systems regularly that we havent heard from --- app/Classes/Protocol.php | 6 +- app/Console/Commands/SystemHeartbeat.php | 35 ++++ app/Console/Kernel.php | 64 +++--- app/Http/Controllers/SystemController.php | 2 +- app/Http/Requests/SystemRegister.php | 11 +- app/Jobs/SystemHeartbeat.php | 91 ++++++++ .../2023_11_26_001257_system_heartbeat.php | 28 +++ .../views/system/widget/form-system.blade.php | 196 +++++++++++++----- 8 files changed, 343 insertions(+), 90 deletions(-) create mode 100644 app/Console/Commands/SystemHeartbeat.php create mode 100644 app/Jobs/SystemHeartbeat.php create mode 100644 database/migrations/2023_11_26_001257_system_heartbeat.php diff --git a/app/Classes/Protocol.php b/app/Classes/Protocol.php index a18257e..df99049 100644 --- a/app/Classes/Protocol.php +++ b/app/Classes/Protocol.php @@ -27,11 +27,13 @@ abstract class Protocol protected const ERROR = -5; // Our sessions Types + /** @deprecated use Mailer::class */ public const SESSION_AUTO = 0; - /** @deprecate Use mailers:class */ + /** @deprecated Use mailers:class */ public const SESSION_EMSI = 1; - /** @deprecate Use mailers:class */ + /** @deprecated Use mailers:class */ public const SESSION_BINKP = 2; + /** @deprecated Use mailers:class */ public const SESSION_ZMODEM = 3; protected const MAX_PATH = 1024; diff --git a/app/Console/Commands/SystemHeartbeat.php b/app/Console/Commands/SystemHeartbeat.php new file mode 100644 index 0000000..e7cf71b --- /dev/null +++ b/app/Console/Commands/SystemHeartbeat.php @@ -0,0 +1,35 @@ +option('force')); + } +} \ No newline at end of file diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 38aa784..29042a5 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,42 +2,44 @@ namespace App\Console; -use App\Jobs\MailSend; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; +use App\Jobs\{MailSend,SystemHeartbeat}; + class Kernel extends ConsoleKernel { - /** - * The Artisan commands provided by your application. - * - * @var array - */ - protected $commands = [ - // - ]; + /** + * The Artisan commands provided by your application. + * + * @var array + */ + protected $commands = [ + // + ]; - /** - * Define the application's command schedule. - * - * @param \Illuminate\Console\Scheduling\Schedule $schedule - * @return void - */ - protected function schedule(Schedule $schedule) - { - $schedule->job(new MailSend(TRUE))->everyMinute()->withoutOverlapping(); - $schedule->job(new MailSend(FALSE))->twiceDaily(1,13); - } + /** + * Define the application's command schedule. + * + * @param \Illuminate\Console\Scheduling\Schedule $schedule + * @return void + */ + protected function schedule(Schedule $schedule) + { + $schedule->job(new MailSend(TRUE))->everyMinute()->withoutOverlapping(); + $schedule->job(new MailSend(FALSE))->twiceDaily(1,13); + $schedule->job(new SystemHeartbeat())->hourly(); + } - /** - * Register the commands for the application. - * - * @return void - */ - protected function commands() - { - $this->load(__DIR__.'/Commands'); + /** + * Register the commands for the application. + * + * @return void + */ + protected function commands() + { + $this->load(__DIR__.'/Commands'); - require base_path('routes/console.php'); - } -} + require base_path('routes/console.php'); + } +} \ No newline at end of file diff --git a/app/Http/Controllers/SystemController.php b/app/Http/Controllers/SystemController.php index 091ed0b..69993d7 100644 --- a/app/Http/Controllers/SystemController.php +++ b/app/Http/Controllers/SystemController.php @@ -32,7 +32,7 @@ class SystemController extends Controller $this->authorize('update',$o); if ($request->post()) { - foreach (['name','location','sysop','hold','phone','address','port','active','method','notes','zt_id','pkt_type'] as $key) + foreach (['name','location','sysop','hold','phone','address','port','active','method','notes','zt_id','pkt_type','heartbeat'] as $key) $o->{$key} = $request->post($key); switch ($request->post('pollmode')) { diff --git a/app/Http/Requests/SystemRegister.php b/app/Http/Requests/SystemRegister.php index 88a2536..ca8af41 100644 --- a/app/Http/Requests/SystemRegister.php +++ b/app/Http/Requests/SystemRegister.php @@ -56,7 +56,9 @@ class SystemRegister extends FormRequest if ((! $request->isMethod('post')) || ($request->action === 'register')) return []; - if ((! $this->so->exists) && ($request->action === 'create')) + $so = $this->route('o'); + + if ((! $so->exists) && ($request->action === 'create')) return [ 'name' => 'required|min:3', ]; @@ -65,7 +67,7 @@ class SystemRegister extends FormRequest [ 'name' => 'required|min:3', ], - ($this->so->exists || ($request->action !== 'create')) ? [ + ($so->exists || ($request->action !== 'create')) ? [ 'location' => 'required|min:3', 'sysop' => 'required|min:3', 'phone' => 'nullable|regex:/^([0-9-]+)$/', @@ -74,13 +76,14 @@ class SystemRegister extends FormRequest 'method' => 'nullable|numeric', 'mailer_details.*' => 'nullable|array', 'mailer_details.*.port' => 'nullable|digits_between:2,5', - 'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($this->so->exists ? $this->so->id : 0), + 'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($so->exists ? $so->id : 0), 'pkt_type' => ['required',Rule::in(array_keys(Packet::PACKET_TYPES))], ] : [], - $this->so->exists ? [ + $so->exists ? [ 'active' => 'required|boolean', 'hold' => 'required|boolean', 'pollmode' => 'required|integer|min:0|max:2', + 'heartbeat' => 'nullable|integer|min:0|max:48', ] : [], )); } diff --git a/app/Jobs/SystemHeartbeat.php b/app/Jobs/SystemHeartbeat.php new file mode 100644 index 0000000..b3ba088 --- /dev/null +++ b/app/Jobs/SystemHeartbeat.php @@ -0,0 +1,91 @@ +system->addresses->pluck('zone_id')->unique(); + + // Find our uplinks that are hubs, NC, RC or ZCs + // Find any system that also has heartbeat configured + $l = Address::select(['addresses.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','role','addresses.system_id']) + ->distinct('systems.id') + ->join('systems',['systems.id'=>'addresses.system_id']) + ->join('system_zone',['system_zone.system_id'=>'systems.id']) + ->join('zones',['zones.id'=>'addresses.zone_id']) + ->join('domains',['domains.id'=>'zones.domain_id']) + ->where('systems.active',true) + ->where('addresses.active',TRUE) + ->where('zones.active',TRUE) + ->where('domains.active',TRUE) + ->whereIN('addresses.zone_id',$our_zones) + ->whereNotNull('pollmode') + ->where(function($query) { + return $query + ->where('role','<',Address::NODE_ACTIVE) + ->orWhereNotNull('heartbeat'); + }) + ->when(! $this->force,function($query) { + return $query + ->where(function($query) { + return $query->whereNull('autohold') + ->orWhere('autohold',FALSE); + }); + }) + ->with(['system','zone.domain']) + ->get(); + + // If we havent polled in heatbeat hours, poll system + foreach ($l as $oo) { + if (Job::where('queue','poll')->get()->pluck('command.address.id')->search($oo->id) === FALSE) { + if ((! $oo->system->last_session) + || ($oo->system->hearbeat && ($oo->system->last_session->addHours($oo->system->heartbeat) < Carbon::now())) + || ((! $oo->system->hearbeat) && ($oo->role < Address::NODE_ACTIVE) && ($oo->system->last_session->addHours(6) < Carbon::now()))) + { + Log::info(sprintf('%s:- Polling [%s] (%s) - we havent seen them since [%s], heartbeat [%d]', + self::LOGKEY, + $oo->ftn, + $oo->system->name, + $oo->system->last_session ?: 'Never', + $oo->system->heartbeat, + )); + + AddressPoll::dispatch($oo); + + } else { + Log::debug(sprintf('%s:= Not scheduling poll to [%s], we saw them [%s], heartbeat [%d]', + self::LOGKEY, + $oo->ftn, + $oo->system->last_session, + $oo->system->heartbeat + )); + } + + } else { + Log::notice(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$oo->ftn)); + } + } + } +} \ No newline at end of file diff --git a/database/migrations/2023_11_26_001257_system_heartbeat.php b/database/migrations/2023_11_26_001257_system_heartbeat.php new file mode 100644 index 0000000..81b6752 --- /dev/null +++ b/database/migrations/2023_11_26_001257_system_heartbeat.php @@ -0,0 +1,28 @@ +smallInteger('heartbeat')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('systems', function (Blueprint $table) { + $table->dropColumn('heartbeat'); + }); + } +}; diff --git a/resources/views/system/widget/form-system.blade.php b/resources/views/system/widget/form-system.blade.php index 93ade40..07282c9 100644 --- a/resources/views/system/widget/form-system.blade.php +++ b/resources/views/system/widget/form-system.blade.php @@ -1,8 +1,4 @@ -@php - use App\Models\Setup; -@endphp -
@@ -205,55 +201,135 @@
- @if (! is_null($o->pollmode)) -
- - - @if($job = $o->poll()) - - - - - - - - - @if ($job->attempts) - - - - +
pollmode))d-none @endif" id="heartbeat_option"> + @can('admin',$o) +
+
+ +
+ + + + @error('heartbeat') + {{ $message }} + @enderror + +
+
+
+ @endcan + + @if (! is_null($o->pollmode)) +
+
+ @if($job = $o->poll()) +
+
+ @if($job->attempts)Last: @else Scheduled: @endif +
+
+ {{ $job->created_at }} +
+
+ +
+
+ Attempts: +
+
+ {{ $job->attempts ?: 0 }} +
+
+ + @if ($job->attempts) +
+
+ Next: +
+
+ {{ $job->available_at->diffForHumans(now(),$job->available_at->isFuture() ? \Carbon\CarbonInterface::DIFF_ABSOLUTE : \Carbon\CarbonInterface::DIFF_RELATIVE_TO_NOW) }} +
+
+ @endif + + @else +
+
+ Last Poll: +
+
+ {{ ($x=$o->logs->where('originate',TRUE)->last())?->created_at ?: 'Never' }} +
+
+ +
+
+ Method: +
+
+ {{ $x?->sessiontype ? \App\Models\Mailer::findOrFail($x->sessiontype)->name : '-' }} +
+
+ + @if ($o->heartbeat) +
+
+ Next Heartbeat: +
+
+ {{ $x ? $x->created_at->addHours($o->heartbeat) : Carbon::now() }} +
+
+ @endif @endif - @else -
- - - - - - - - @endif +
+
+ Status: +
+
+ + @if ($job) Queued + @elseif ($o->autohold)Auto Hold + @else + @switch($o->pollmode) + @case(TRUE) Crash @break; + @case(FALSE) Normal @break; + @default Hold + @endswitch + @endif + +
+
+ + - - - - - -
@if($job->attempts)Last Attempt @else Scheduled @endif:{{ $job->created_at }}
Attempts :{{ $job->attempts ?: 0 }}
Next Attempt :{{ $job->available_at->diffForHumans(now(),$job->available_at->isFuture() ? \Carbon\CarbonInterface::DIFF_ABSOLUTE : \Carbon\CarbonInterface::DIFF_RELATIVE_TO_NOW) }}
Last Poll :{{ ($x=$o->logs->where('originate',TRUE)->last())?->created_at ?: 'Never' }}
Method :{{ $x?->sessiontype ?: '-' }}
Status : - @if ($job) Queued - @elseif ($o->autohold)Auto Hold - @else - @switch($o->pollmode) - @case(TRUE) Crash @break; - @case(FALSE) Normal @break; - @default Hold - @endswitch - @endif -
-
- @endif + + {{-- +
+ + + + + + + + +
Status : + @if ($job) Queued + @elseif ($o->autohold)Auto Hold + @else + @switch($o->pollmode) + @case(TRUE) Crash @break; + @case(FALSE) Normal @break; + @default Hold + @endswitch + @endif +
+
+ --}} + @endif + @@ -283,7 +359,6 @@ - @@ -313,4 +388,21 @@ @endif - \ No newline at end of file + + +@section('page-scripts') + +@append \ No newline at end of file