Implemented system heartbeat, to poll systems regularly that we havent heard from

2023-11-26 13:10:23 +11:00 · 2023-11-26 13:10:23 +11:00 · 1ac3583479
parent 6e7e09ab50
commit 1ac3583479
8 changed files with 343 additions and 90 deletions
--- a/app/Classes/Protocol.php
+++ b/app/Classes/Protocol.php
@ -27,11 +27,13 @@ abstract class Protocol
 	protected const ERROR			= -5;

 	// Our sessions Types
+	/** @deprecated use Mailer::class */
 	public const SESSION_AUTO		= 0;
-	/** @deprecate Use mailers:class */
+	/** @deprecated Use mailers:class */
 	public const SESSION_EMSI		= 1;
-	/** @deprecate Use mailers:class */
+	/** @deprecated Use mailers:class */
 	public const SESSION_BINKP		= 2;
+	/** @deprecated Use mailers:class */
 	public const SESSION_ZMODEM		= 3;

 	protected const MAX_PATH		= 1024;
--- a/app/Console/Commands/SystemHeartbeat.php
+++ b/app/Console/Commands/SystemHeartbeat.php
@ -0,0 +1,35 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
+
+use App\Jobs\SystemHeartbeat as Job;
+
+class SystemHeartbeat extends Command
+{
+	/**
+	 * The name and signature of the console command.
+	 *
+	 * @var string
+	 */
+	protected $signature = 'system:heartbeat'
+	.' {--F|force : Force poll systems that are auto hold}';
+
+	/**
+	 * The console command description.
+	 *
+	 * @var string
+	 */
+	protected $description = 'Poll systems that we havent seen for a while';
+
+	/**
+	 * Execute the console command.
+	 */
+	public function handle()
+	{
+		Log::info('CSH:- Triggering heartbeat to systems');
+		Job::dispatchSync($this->option('force'));
+	}
+}
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@ -2,42 +2,44 @@

 namespace App\Console;

-use App\Jobs\MailSend;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;

+use App\Jobs\{MailSend,SystemHeartbeat};
+
 class Kernel extends ConsoleKernel
 {
-    /**
-     * The Artisan commands provided by your application.
-     *
-     * @var array
-     */
-    protected $commands = [
-        //
-    ];
+	/**
+	 * The Artisan commands provided by your application.
+	 *
+	 * @var array
+	 */
+	protected $commands = [
+		//
+	];

-    /**
-     * Define the application's command schedule.
-     *
-     * @param  \Illuminate\Console\Scheduling\Schedule  $schedule
-     * @return void
-     */
-    protected function schedule(Schedule $schedule)
-    {
-        $schedule->job(new MailSend(TRUE))->everyMinute()->withoutOverlapping();
-        $schedule->job(new MailSend(FALSE))->twiceDaily(1,13);
-    }
+	/**
+	 * Define the application's command schedule.
+	 *
+	 * @param  \Illuminate\Console\Scheduling\Schedule  $schedule
+	 * @return void
+	 */
+	protected function schedule(Schedule $schedule)
+	{
+		$schedule->job(new MailSend(TRUE))->everyMinute()->withoutOverlapping();
+		$schedule->job(new MailSend(FALSE))->twiceDaily(1,13);
+		$schedule->job(new SystemHeartbeat())->hourly();
+	}

-    /**
-     * Register the commands for the application.
-     *
-     * @return void
-     */
-    protected function commands()
-    {
-        $this->load(__DIR__.'/Commands');
+	/**
+	 * Register the commands for the application.
+	 *
+	 * @return void
+	 */
+	protected function commands()
+	{
+		$this->load(__DIR__.'/Commands');

-        require base_path('routes/console.php');
-    }
-}
+		require base_path('routes/console.php');
+	}
+}
--- a/app/Http/Controllers/SystemController.php
+++ b/app/Http/Controllers/SystemController.php
@ -32,7 +32,7 @@ class SystemController extends Controller
 		$this->authorize('update',$o);

 		if ($request->post()) {
-			foreach (['name','location','sysop','hold','phone','address','port','active','method','notes','zt_id','pkt_type'] as $key)
+			foreach (['name','location','sysop','hold','phone','address','port','active','method','notes','zt_id','pkt_type','heartbeat'] as $key)
 				$o->{$key} = $request->post($key);

 			switch ($request->post('pollmode')) {
--- a/app/Http/Requests/SystemRegister.php
+++ b/app/Http/Requests/SystemRegister.php
@ -56,7 +56,9 @@ class SystemRegister extends FormRequest
 		if ((! $request->isMethod('post')) || ($request->action === 'register'))
 			return [];

-		if ((! $this->so->exists) && ($request->action === 'create'))
+		$so = $this->route('o');
+
+		if ((! $so->exists) && ($request->action === 'create'))
 			return [
 				'name' => 'required|min:3',
 			];
@ -65,7 +67,7 @@ class SystemRegister extends FormRequest
 			[
 				'name' => 'required|min:3',
 			],
-			($this->so->exists || ($request->action !== 'create')) ? [
+			($so->exists || ($request->action !== 'create')) ? [
 				'location' => 'required|min:3',
 				'sysop' => 'required|min:3',
 				'phone' => 'nullable|regex:/^([0-9-]+)$/',
@ -74,13 +76,14 @@ class SystemRegister extends FormRequest
 				'method' => 'nullable|numeric',
 				'mailer_details.*' => 'nullable|array',
 				'mailer_details.*.port' => 'nullable|digits_between:2,5',
-				'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($this->so->exists ? $this->so->id : 0),
+				'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($so->exists ? $so->id : 0),
 				'pkt_type' => ['required',Rule::in(array_keys(Packet::PACKET_TYPES))],
 			] : [],
-			$this->so->exists ? [
+			$so->exists ? [
 				'active' => 'required|boolean',
 				'hold' => 'required|boolean',
 				'pollmode' => 'required|integer|min:0|max:2',
+				'heartbeat' => 'nullable|integer|min:0|max:48',
 			] : [],
 		));
 	}
--- a/app/Jobs/SystemHeartbeat.php
+++ b/app/Jobs/SystemHeartbeat.php
@ -0,0 +1,91 @@
+<?php
+
+namespace App\Jobs;
+
+use Carbon\Carbon;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Support\Facades\Log;
+use Repat\LaravelJobs\Job;
+
+use App\Models\{Address,Setup};
+
+class SystemHeartbeat #implements ShouldQueue
+{
+	use Dispatchable;
+
+	private const LOGKEY = 'JSH';
+
+	/**
+	 * @param bool $force Include systems that are autohold
+	 */
+	public function __construct(private ?bool $force=NULL) {}
+
+	/**
+	 * Execute the job.
+	 */
+	public function handle(): void
+	{
+		// Our zones
+		$our_zones = Setup::findOrFail(config('app.id'))->system->addresses->pluck('zone_id')->unique();
+
+		// Find our uplinks that are hubs, NC, RC or ZCs
+		// Find any system that also has heartbeat configured
+		$l = Address::select(['addresses.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','role','addresses.system_id'])
+			->distinct('systems.id')
+			->join('systems',['systems.id'=>'addresses.system_id'])
+			->join('system_zone',['system_zone.system_id'=>'systems.id'])
+			->join('zones',['zones.id'=>'addresses.zone_id'])
+			->join('domains',['domains.id'=>'zones.domain_id'])
+			->where('systems.active',true)
+			->where('addresses.active',TRUE)
+			->where('zones.active',TRUE)
+			->where('domains.active',TRUE)
+			->whereIN('addresses.zone_id',$our_zones)
+			->whereNotNull('pollmode')
+			->where(function($query) {
+				return $query
+					->where('role','<',Address::NODE_ACTIVE)
+					->orWhereNotNull('heartbeat');
+			})
+			->when(! $this->force,function($query) {
+				return $query
+					->where(function($query) {
+						return $query->whereNull('autohold')
+							->orWhere('autohold',FALSE);
+				});
+			})
+			->with(['system','zone.domain'])
+			->get();
+
+		// If we havent polled in heatbeat hours, poll system
+		foreach ($l as $oo) {
+			if (Job::where('queue','poll')->get()->pluck('command.address.id')->search($oo->id) === FALSE) {
+				if ((! $oo->system->last_session)
+					|| ($oo->system->hearbeat && ($oo->system->last_session->addHours($oo->system->heartbeat) < Carbon::now()))
+					|| ((! $oo->system->hearbeat) && ($oo->role < Address::NODE_ACTIVE) && ($oo->system->last_session->addHours(6) < Carbon::now())))
+				{
+					Log::info(sprintf('%s:- Polling [%s] (%s) - we havent seen them since [%s], heartbeat [%d]',
+						self::LOGKEY,
+						$oo->ftn,
+						$oo->system->name,
+						$oo->system->last_session ?: 'Never',
+						$oo->system->heartbeat,
+					));
+
+					AddressPoll::dispatch($oo);
+
+				} else {
+					Log::debug(sprintf('%s:= Not scheduling poll to [%s], we saw them [%s], heartbeat [%d]',
+						self::LOGKEY,
+						$oo->ftn,
+						$oo->system->last_session,
+						$oo->system->heartbeat
+					));
+				}
+
+			} else {
+				Log::notice(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$oo->ftn));
+			}
+		}
+	}
+}
--- a/database/migrations/2023_11_26_001257_system_heartbeat.php
+++ b/database/migrations/2023_11_26_001257_system_heartbeat.php
@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+		Schema::table('systems', function (Blueprint $table) {
+			$table->smallInteger('heartbeat')->nullable();
+		});
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+		Schema::table('systems', function (Blueprint $table) {
+			$table->dropColumn('heartbeat');
+		});
+    }
+};
--- a/resources/views/system/widget/form-system.blade.php
+++ b/resources/views/system/widget/form-system.blade.php
@ -1,8 +1,4 @@
 <!-- $o = System::class -->
-@php
-	use App\Models\Setup;
-@endphp
-
 <div class="row">
 	<!-- Name -->
 	<div class="col-4">
@ -205,55 +201,135 @@
 				</div>
 			</div>

-			@if (! is_null($o->pollmode))
-				<div class="offset-3 col-4">
-					<table class="table monotable m-0 p-0 small noborder">
-						<tbody style="border-style:dotted;">
-						@if($job = $o->poll())
-							<tr>
-								<td class="cap text-end">@if($job->attempts)Last Attempt @else Scheduled @endif:</td>
-								<td>{{ $job->created_at }} </td>
-							</tr>
-							<tr>
-								<td class="cap text-end">Attempts :</td>
-								<td>{{ $job->attempts ?: 0 }}</td>
-							</tr>
-							@if ($job->attempts)
-								<tr>
-									<td class="cap text-end">Next Attempt :</td>
-									<td>{{ $job->available_at->diffForHumans(now(),$job->available_at->isFuture() ? \Carbon\CarbonInterface::DIFF_ABSOLUTE : \Carbon\CarbonInterface::DIFF_RELATIVE_TO_NOW) }}</td>
-								</tr>
+			<div class="offset-3 col-4 @if((old('pollmode') === "0") || is_null($o->pollmode))d-none @endif" id="heartbeat_option">
+				@can('admin',$o)
+					<div class="row p-0">
+						<div class="offset-3 col-6">
+							<label for="method" class="form-label">Heartbeat <i class="bi bi-info-circle" title="Attempt contact after last seen"></i></label>
+							<div class="input-group has-validation">
+								<span class="input-group-text"><i class="bi bi-hourglass-bottom"></i></span>
+								<input type="text" class="form-control text-end @error('heartbeat') is-invalid @enderror" id="heartbeat" placeholder="Hours" name="heartbeat" value="{{ old('heartbeat',$o->heartbeat) }}">
+								<span class="invalid-feedback" role="alert">
+									@error('heartbeat')
+										{{ $message }}
+									@enderror
+								</span>
+							</div>
+						</div>
+					</div>
+				@endcan
+
+				@if (! is_null($o->pollmode))
+					<div class="row">
+						<div class="offset-3 col-9 bg-secondary rounded p-2 small">
+							@if($job = $o->poll())
+								<div class="row p-0">
+									<div class="col-4 text-dark">
+										@if($job->attempts)Last: @else Scheduled: @endif
+									</div>
+									<div class="col-8">
+										<strong class="highlight">{{ $job->created_at }}</strong>
+									</div>
+								</div>
+
+								<div class="row">
+									<div class="col-4 text-dark">
+										Attempts:
+									</div>
+									<div class="col-8">
+										<strong class="highlight">{{ $job->attempts ?: 0 }}</strong>
+									</div>
+								</div>
+
+								@if ($job->attempts)
+									<div class="row">
+										<div class="col-4 text-dark">
+											Next:
+										</div>
+										<div class="col-8">
+											<strong class="highlight">{{ $job->available_at->diffForHumans(now(),$job->available_at->isFuture() ? \Carbon\CarbonInterface::DIFF_ABSOLUTE : \Carbon\CarbonInterface::DIFF_RELATIVE_TO_NOW) }}</strong>
+										</div>
+									</div>
+								@endif
+
+							@else
+								<div class="row p-0">
+									<div class="col-4 text-dark">
+										Last Poll:
+									</div>
+									<div class="col-8">
+										<strong class="highlight">{{ ($x=$o->logs->where('originate',TRUE)->last())?->created_at ?: 'Never' }}</strong>
+									</div>
+								</div>
+
+								<div class="row">
+									<div class="col-4 text-dark">
+										Method:
+									</div>
+									<div class="col-8">
+										<strong class="highlight">{{ $x?->sessiontype ? \App\Models\Mailer::findOrFail($x->sessiontype)->name : '-' }}</strong>
+									</div>
+								</div>
+
+								@if ($o->heartbeat)
+									<div class="row">
+										<div class="col-4 text-dark">
+											Next Heartbeat:
+										</div>
+										<div class="col-8">
+											<strong class="highlight">{{ $x ? $x->created_at->addHours($o->heartbeat) : Carbon::now() }}</strong>
+										</div>
+									</div>
+								@endif
 							@endif

-						@else
-							<tr>
-								<td class="cap text-end">Last Poll :</td>
-								<td>{{ ($x=$o->logs->where('originate',TRUE)->last())?->created_at ?: 'Never' }}</td>
-							</tr>
-							<tr>
-								<td class="cap text-end">Method :</td>
-								<td>{{ $x?->sessiontype ?: '-' }}</td>
-							</tr>
-						@endif
+							<div class="row">
+								<div class="col-4 text-dark">
+									Status:
+								</div>
+								<div class="col-8">
+									<strong class="highlight">
+										@if ($job) Queued
+										@elseif ($o->autohold)Auto Hold
+										@else
+											@switch($o->pollmode)
+												@case(TRUE) Crash @break;
+												@case(FALSE) Normal @break;
+												@default Hold
+											@endswitch
+										@endif
+									</strong>
+								</div>
+							</div>
+						</div>
+					</div>

-						<tr>
-							<td class="cap text-end">Status :</td>
-							<td>
-								@if ($job) Queued
-								@elseif ($o->autohold)Auto Hold
-								@else
-									@switch($o->pollmode)
-										@case(TRUE) Crash @break;
-										@case(FALSE) Normal @break;
-										@default Hold
-									@endswitch
-								@endif
-							</td>
-						</tr>
-						</tbody>
-					</table>
-				</div>
-			@endif
+
+					{{--
+						<div class="col-12">
+							<table class="table monotable m-0 p-0 small noborder">
+								<tbody xstyle="border-style:dotted;">
+
+								<tr>
+									<td class="cap text-end">Status :</td>
+									<td>
+										@if ($job) Queued
+										@elseif ($o->autohold)Auto Hold
+										@else
+											@switch($o->pollmode)
+												@case(TRUE) Crash @break;
+												@case(FALSE) Normal @break;
+												@default Hold
+											@endswitch
+										@endif
+									</td>
+								</tr>
+								</tbody>
+							</table>
+						</div>
+					--}}
+				@endif
+			</div>
 		</div>
 	</div>
 </div>
@ -283,7 +359,6 @@
 					<input type="text" class="form-control text-end @error('port') is-invalid @enderror" id="port" placeholder="Port" name="port" value="{{ old('port',$o->port) }}" @cannot($action,$o)readonly @endcannot>
 				</div>
 			</div>
-
 		</div>
 	</div>
 </div>
@ -313,4 +388,21 @@
 			<button type="submit" class="btn btn-success float-end" name="submit" value="create">Register</button>
 		@endif
 	</div>
-</div>
+</div>
+
+@section('page-scripts')
+	<script type="text/javascript">
+		$(document).ready(function() {
+			$('#poll_normal').on('click',function() {
+				$('#heartbeat_option').removeClass('d-none');
+			})
+			$('#poll_crash').on('click',function() {
+				$('#heartbeat_option').removeClass('d-none');
+			})
+			$('#poll_hold').on('click',function() {
+				$('#heartbeat_option').addClass('d-none');
+				console.log('hold');
+			})
+		})
+	</script>
+@append