Implemented system heartbeat, to poll systems regularly that we havent heard from

This commit is contained in:
Deon George 2023-11-26 13:10:23 +11:00
parent 6e7e09ab50
commit 1ac3583479
8 changed files with 343 additions and 90 deletions

View File

@ -27,11 +27,13 @@ abstract class Protocol
protected const ERROR = -5; protected const ERROR = -5;
// Our sessions Types // Our sessions Types
/** @deprecated use Mailer::class */
public const SESSION_AUTO = 0; public const SESSION_AUTO = 0;
/** @deprecate Use mailers:class */ /** @deprecated Use mailers:class */
public const SESSION_EMSI = 1; public const SESSION_EMSI = 1;
/** @deprecate Use mailers:class */ /** @deprecated Use mailers:class */
public const SESSION_BINKP = 2; public const SESSION_BINKP = 2;
/** @deprecated Use mailers:class */
public const SESSION_ZMODEM = 3; public const SESSION_ZMODEM = 3;
protected const MAX_PATH = 1024; protected const MAX_PATH = 1024;

View File

@ -0,0 +1,35 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use App\Jobs\SystemHeartbeat as Job;
class SystemHeartbeat extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'system:heartbeat'
.' {--F|force : Force poll systems that are auto hold}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Poll systems that we havent seen for a while';
/**
* Execute the console command.
*/
public function handle()
{
Log::info('CSH:- Triggering heartbeat to systems');
Job::dispatchSync($this->option('force'));
}
}

View File

@ -2,42 +2,44 @@
namespace App\Console; namespace App\Console;
use App\Jobs\MailSend;
use Illuminate\Console\Scheduling\Schedule; use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel; use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use App\Jobs\{MailSend,SystemHeartbeat};
class Kernel extends ConsoleKernel class Kernel extends ConsoleKernel
{ {
/** /**
* The Artisan commands provided by your application. * The Artisan commands provided by your application.
* *
* @var array * @var array
*/ */
protected $commands = [ protected $commands = [
// //
]; ];
/** /**
* Define the application's command schedule. * Define the application's command schedule.
* *
* @param \Illuminate\Console\Scheduling\Schedule $schedule * @param \Illuminate\Console\Scheduling\Schedule $schedule
* @return void * @return void
*/ */
protected function schedule(Schedule $schedule) protected function schedule(Schedule $schedule)
{ {
$schedule->job(new MailSend(TRUE))->everyMinute()->withoutOverlapping(); $schedule->job(new MailSend(TRUE))->everyMinute()->withoutOverlapping();
$schedule->job(new MailSend(FALSE))->twiceDaily(1,13); $schedule->job(new MailSend(FALSE))->twiceDaily(1,13);
} $schedule->job(new SystemHeartbeat())->hourly();
}
/** /**
* Register the commands for the application. * Register the commands for the application.
* *
* @return void * @return void
*/ */
protected function commands() protected function commands()
{ {
$this->load(__DIR__.'/Commands'); $this->load(__DIR__.'/Commands');
require base_path('routes/console.php'); require base_path('routes/console.php');
} }
} }

View File

@ -32,7 +32,7 @@ class SystemController extends Controller
$this->authorize('update',$o); $this->authorize('update',$o);
if ($request->post()) { if ($request->post()) {
foreach (['name','location','sysop','hold','phone','address','port','active','method','notes','zt_id','pkt_type'] as $key) foreach (['name','location','sysop','hold','phone','address','port','active','method','notes','zt_id','pkt_type','heartbeat'] as $key)
$o->{$key} = $request->post($key); $o->{$key} = $request->post($key);
switch ($request->post('pollmode')) { switch ($request->post('pollmode')) {

View File

@ -56,7 +56,9 @@ class SystemRegister extends FormRequest
if ((! $request->isMethod('post')) || ($request->action === 'register')) if ((! $request->isMethod('post')) || ($request->action === 'register'))
return []; return [];
if ((! $this->so->exists) && ($request->action === 'create')) $so = $this->route('o');
if ((! $so->exists) && ($request->action === 'create'))
return [ return [
'name' => 'required|min:3', 'name' => 'required|min:3',
]; ];
@ -65,7 +67,7 @@ class SystemRegister extends FormRequest
[ [
'name' => 'required|min:3', 'name' => 'required|min:3',
], ],
($this->so->exists || ($request->action !== 'create')) ? [ ($so->exists || ($request->action !== 'create')) ? [
'location' => 'required|min:3', 'location' => 'required|min:3',
'sysop' => 'required|min:3', 'sysop' => 'required|min:3',
'phone' => 'nullable|regex:/^([0-9-]+)$/', 'phone' => 'nullable|regex:/^([0-9-]+)$/',
@ -74,13 +76,14 @@ class SystemRegister extends FormRequest
'method' => 'nullable|numeric', 'method' => 'nullable|numeric',
'mailer_details.*' => 'nullable|array', 'mailer_details.*' => 'nullable|array',
'mailer_details.*.port' => 'nullable|digits_between:2,5', 'mailer_details.*.port' => 'nullable|digits_between:2,5',
'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($this->so->exists ? $this->so->id : 0), 'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($so->exists ? $so->id : 0),
'pkt_type' => ['required',Rule::in(array_keys(Packet::PACKET_TYPES))], 'pkt_type' => ['required',Rule::in(array_keys(Packet::PACKET_TYPES))],
] : [], ] : [],
$this->so->exists ? [ $so->exists ? [
'active' => 'required|boolean', 'active' => 'required|boolean',
'hold' => 'required|boolean', 'hold' => 'required|boolean',
'pollmode' => 'required|integer|min:0|max:2', 'pollmode' => 'required|integer|min:0|max:2',
'heartbeat' => 'nullable|integer|min:0|max:48',
] : [], ] : [],
)); ));
} }

View File

@ -0,0 +1,91 @@
<?php
namespace App\Jobs;
use Carbon\Carbon;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Facades\Log;
use Repat\LaravelJobs\Job;
use App\Models\{Address,Setup};
class SystemHeartbeat #implements ShouldQueue
{
use Dispatchable;
private const LOGKEY = 'JSH';
/**
* @param bool $force Include systems that are autohold
*/
public function __construct(private ?bool $force=NULL) {}
/**
* Execute the job.
*/
public function handle(): void
{
// Our zones
$our_zones = Setup::findOrFail(config('app.id'))->system->addresses->pluck('zone_id')->unique();
// Find our uplinks that are hubs, NC, RC or ZCs
// Find any system that also has heartbeat configured
$l = Address::select(['addresses.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','role','addresses.system_id'])
->distinct('systems.id')
->join('systems',['systems.id'=>'addresses.system_id'])
->join('system_zone',['system_zone.system_id'=>'systems.id'])
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->where('systems.active',true)
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->whereIN('addresses.zone_id',$our_zones)
->whereNotNull('pollmode')
->where(function($query) {
return $query
->where('role','<',Address::NODE_ACTIVE)
->orWhereNotNull('heartbeat');
})
->when(! $this->force,function($query) {
return $query
->where(function($query) {
return $query->whereNull('autohold')
->orWhere('autohold',FALSE);
});
})
->with(['system','zone.domain'])
->get();
// If we havent polled in heatbeat hours, poll system
foreach ($l as $oo) {
if (Job::where('queue','poll')->get()->pluck('command.address.id')->search($oo->id) === FALSE) {
if ((! $oo->system->last_session)
|| ($oo->system->hearbeat && ($oo->system->last_session->addHours($oo->system->heartbeat) < Carbon::now()))
|| ((! $oo->system->hearbeat) && ($oo->role < Address::NODE_ACTIVE) && ($oo->system->last_session->addHours(6) < Carbon::now())))
{
Log::info(sprintf('%s:- Polling [%s] (%s) - we havent seen them since [%s], heartbeat [%d]',
self::LOGKEY,
$oo->ftn,
$oo->system->name,
$oo->system->last_session ?: 'Never',
$oo->system->heartbeat,
));
AddressPoll::dispatch($oo);
} else {
Log::debug(sprintf('%s:= Not scheduling poll to [%s], we saw them [%s], heartbeat [%d]',
self::LOGKEY,
$oo->ftn,
$oo->system->last_session,
$oo->system->heartbeat
));
}
} else {
Log::notice(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$oo->ftn));
}
}
}
}

View File

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('systems', function (Blueprint $table) {
$table->smallInteger('heartbeat')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('systems', function (Blueprint $table) {
$table->dropColumn('heartbeat');
});
}
};

View File

@ -1,8 +1,4 @@
<!-- $o = System::class --> <!-- $o = System::class -->
@php
use App\Models\Setup;
@endphp
<div class="row"> <div class="row">
<!-- Name --> <!-- Name -->
<div class="col-4"> <div class="col-4">
@ -205,55 +201,135 @@
</div> </div>
</div> </div>
@if (! is_null($o->pollmode)) <div class="offset-3 col-4 @if((old('pollmode') === "0") || is_null($o->pollmode))d-none @endif" id="heartbeat_option">
<div class="offset-3 col-4"> @can('admin',$o)
<table class="table monotable m-0 p-0 small noborder"> <div class="row p-0">
<tbody style="border-style:dotted;"> <div class="offset-3 col-6">
@if($job = $o->poll()) <label for="method" class="form-label">Heartbeat <i class="bi bi-info-circle" title="Attempt contact after last seen"></i></label>
<tr> <div class="input-group has-validation">
<td class="cap text-end">@if($job->attempts)Last Attempt @else Scheduled @endif:</td> <span class="input-group-text"><i class="bi bi-hourglass-bottom"></i></span>
<td>{{ $job->created_at }} </td> <input type="text" class="form-control text-end @error('heartbeat') is-invalid @enderror" id="heartbeat" placeholder="Hours" name="heartbeat" value="{{ old('heartbeat',$o->heartbeat) }}">
</tr> <span class="invalid-feedback" role="alert">
<tr> @error('heartbeat')
<td class="cap text-end">Attempts :</td> {{ $message }}
<td>{{ $job->attempts ?: 0 }}</td> @enderror
</tr> </span>
@if ($job->attempts) </div>
<tr> </div>
<td class="cap text-end">Next Attempt :</td> </div>
<td>{{ $job->available_at->diffForHumans(now(),$job->available_at->isFuture() ? \Carbon\CarbonInterface::DIFF_ABSOLUTE : \Carbon\CarbonInterface::DIFF_RELATIVE_TO_NOW) }}</td> @endcan
</tr>
@if (! is_null($o->pollmode))
<div class="row">
<div class="offset-3 col-9 bg-secondary rounded p-2 small">
@if($job = $o->poll())
<div class="row p-0">
<div class="col-4 text-dark">
@if($job->attempts)Last: @else Scheduled: @endif
</div>
<div class="col-8">
<strong class="highlight">{{ $job->created_at }}</strong>
</div>
</div>
<div class="row">
<div class="col-4 text-dark">
Attempts:
</div>
<div class="col-8">
<strong class="highlight">{{ $job->attempts ?: 0 }}</strong>
</div>
</div>
@if ($job->attempts)
<div class="row">
<div class="col-4 text-dark">
Next:
</div>
<div class="col-8">
<strong class="highlight">{{ $job->available_at->diffForHumans(now(),$job->available_at->isFuture() ? \Carbon\CarbonInterface::DIFF_ABSOLUTE : \Carbon\CarbonInterface::DIFF_RELATIVE_TO_NOW) }}</strong>
</div>
</div>
@endif
@else
<div class="row p-0">
<div class="col-4 text-dark">
Last Poll:
</div>
<div class="col-8">
<strong class="highlight">{{ ($x=$o->logs->where('originate',TRUE)->last())?->created_at ?: 'Never' }}</strong>
</div>
</div>
<div class="row">
<div class="col-4 text-dark">
Method:
</div>
<div class="col-8">
<strong class="highlight">{{ $x?->sessiontype ? \App\Models\Mailer::findOrFail($x->sessiontype)->name : '-' }}</strong>
</div>
</div>
@if ($o->heartbeat)
<div class="row">
<div class="col-4 text-dark">
Next Heartbeat:
</div>
<div class="col-8">
<strong class="highlight">{{ $x ? $x->created_at->addHours($o->heartbeat) : Carbon::now() }}</strong>
</div>
</div>
@endif
@endif @endif
@else <div class="row">
<tr> <div class="col-4 text-dark">
<td class="cap text-end">Last Poll :</td> Status:
<td>{{ ($x=$o->logs->where('originate',TRUE)->last())?->created_at ?: 'Never' }}</td> </div>
</tr> <div class="col-8">
<tr> <strong class="highlight">
<td class="cap text-end">Method :</td> @if ($job) Queued
<td>{{ $x?->sessiontype ?: '-' }}</td> @elseif ($o->autohold)Auto Hold
</tr> @else
@endif @switch($o->pollmode)
@case(TRUE) Crash @break;
@case(FALSE) Normal @break;
@default Hold
@endswitch
@endif
</strong>
</div>
</div>
</div>
</div>
<tr>
<td class="cap text-end">Status :</td> {{--
<td> <div class="col-12">
@if ($job) Queued <table class="table monotable m-0 p-0 small noborder">
@elseif ($o->autohold)Auto Hold <tbody xstyle="border-style:dotted;">
@else
@switch($o->pollmode) <tr>
@case(TRUE) Crash @break; <td class="cap text-end">Status :</td>
@case(FALSE) Normal @break; <td>
@default Hold @if ($job) Queued
@endswitch @elseif ($o->autohold)Auto Hold
@endif @else
</td> @switch($o->pollmode)
</tr> @case(TRUE) Crash @break;
</tbody> @case(FALSE) Normal @break;
</table> @default Hold
</div> @endswitch
@endif @endif
</td>
</tr>
</tbody>
</table>
</div>
--}}
@endif
</div>
</div> </div>
</div> </div>
</div> </div>
@ -283,7 +359,6 @@
<input type="text" class="form-control text-end @error('port') is-invalid @enderror" id="port" placeholder="Port" name="port" value="{{ old('port',$o->port) }}" @cannot($action,$o)readonly @endcannot> <input type="text" class="form-control text-end @error('port') is-invalid @enderror" id="port" placeholder="Port" name="port" value="{{ old('port',$o->port) }}" @cannot($action,$o)readonly @endcannot>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
@ -313,4 +388,21 @@
<button type="submit" class="btn btn-success float-end" name="submit" value="create">Register</button> <button type="submit" class="btn btn-success float-end" name="submit" value="create">Register</button>
@endif @endif
</div> </div>
</div> </div>
@section('page-scripts')
<script type="text/javascript">
$(document).ready(function() {
$('#poll_normal').on('click',function() {
$('#heartbeat_option').removeClass('d-none');
})
$('#poll_crash').on('click',function() {
$('#heartbeat_option').removeClass('d-none');
})
$('#poll_hold').on('click',function() {
$('#heartbeat_option').addClass('d-none');
console.log('hold');
})
})
</script>
@append