Implemented system heartbeat, to poll systems regularly that we havent heard from

This commit is contained in:
Deon George 2023-11-26 13:10:23 +11:00
parent 6e7e09ab50
commit 1ac3583479
8 changed files with 343 additions and 90 deletions

View File

@ -27,11 +27,13 @@ abstract class Protocol
protected const ERROR = -5;
// Our sessions Types
/** @deprecated use Mailer::class */
public const SESSION_AUTO = 0;
/** @deprecate Use mailers:class */
/** @deprecated Use mailers:class */
public const SESSION_EMSI = 1;
/** @deprecate Use mailers:class */
/** @deprecated Use mailers:class */
public const SESSION_BINKP = 2;
/** @deprecated Use mailers:class */
public const SESSION_ZMODEM = 3;
protected const MAX_PATH = 1024;

View File

@ -0,0 +1,35 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use App\Jobs\SystemHeartbeat as Job;
class SystemHeartbeat extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'system:heartbeat'
.' {--F|force : Force poll systems that are auto hold}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Poll systems that we havent seen for a while';
/**
* Execute the console command.
*/
public function handle()
{
Log::info('CSH:- Triggering heartbeat to systems');
Job::dispatchSync($this->option('force'));
}
}

View File

@ -2,10 +2,11 @@
namespace App\Console;
use App\Jobs\MailSend;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use App\Jobs\{MailSend,SystemHeartbeat};
class Kernel extends ConsoleKernel
{
/**
@ -27,6 +28,7 @@ class Kernel extends ConsoleKernel
{
$schedule->job(new MailSend(TRUE))->everyMinute()->withoutOverlapping();
$schedule->job(new MailSend(FALSE))->twiceDaily(1,13);
$schedule->job(new SystemHeartbeat())->hourly();
}
/**

View File

@ -32,7 +32,7 @@ class SystemController extends Controller
$this->authorize('update',$o);
if ($request->post()) {
foreach (['name','location','sysop','hold','phone','address','port','active','method','notes','zt_id','pkt_type'] as $key)
foreach (['name','location','sysop','hold','phone','address','port','active','method','notes','zt_id','pkt_type','heartbeat'] as $key)
$o->{$key} = $request->post($key);
switch ($request->post('pollmode')) {

View File

@ -56,7 +56,9 @@ class SystemRegister extends FormRequest
if ((! $request->isMethod('post')) || ($request->action === 'register'))
return [];
if ((! $this->so->exists) && ($request->action === 'create'))
$so = $this->route('o');
if ((! $so->exists) && ($request->action === 'create'))
return [
'name' => 'required|min:3',
];
@ -65,7 +67,7 @@ class SystemRegister extends FormRequest
[
'name' => 'required|min:3',
],
($this->so->exists || ($request->action !== 'create')) ? [
($so->exists || ($request->action !== 'create')) ? [
'location' => 'required|min:3',
'sysop' => 'required|min:3',
'phone' => 'nullable|regex:/^([0-9-]+)$/',
@ -74,13 +76,14 @@ class SystemRegister extends FormRequest
'method' => 'nullable|numeric',
'mailer_details.*' => 'nullable|array',
'mailer_details.*.port' => 'nullable|digits_between:2,5',
'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($this->so->exists ? $this->so->id : 0),
'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($so->exists ? $so->id : 0),
'pkt_type' => ['required',Rule::in(array_keys(Packet::PACKET_TYPES))],
] : [],
$this->so->exists ? [
$so->exists ? [
'active' => 'required|boolean',
'hold' => 'required|boolean',
'pollmode' => 'required|integer|min:0|max:2',
'heartbeat' => 'nullable|integer|min:0|max:48',
] : [],
));
}

View File

@ -0,0 +1,91 @@
<?php
namespace App\Jobs;
use Carbon\Carbon;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Facades\Log;
use Repat\LaravelJobs\Job;
use App\Models\{Address,Setup};
class SystemHeartbeat #implements ShouldQueue
{
use Dispatchable;
private const LOGKEY = 'JSH';
/**
* @param bool $force Include systems that are autohold
*/
public function __construct(private ?bool $force=NULL) {}
/**
* Execute the job.
*/
public function handle(): void
{
// Our zones
$our_zones = Setup::findOrFail(config('app.id'))->system->addresses->pluck('zone_id')->unique();
// Find our uplinks that are hubs, NC, RC or ZCs
// Find any system that also has heartbeat configured
$l = Address::select(['addresses.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','role','addresses.system_id'])
->distinct('systems.id')
->join('systems',['systems.id'=>'addresses.system_id'])
->join('system_zone',['system_zone.system_id'=>'systems.id'])
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->where('systems.active',true)
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->whereIN('addresses.zone_id',$our_zones)
->whereNotNull('pollmode')
->where(function($query) {
return $query
->where('role','<',Address::NODE_ACTIVE)
->orWhereNotNull('heartbeat');
})
->when(! $this->force,function($query) {
return $query
->where(function($query) {
return $query->whereNull('autohold')
->orWhere('autohold',FALSE);
});
})
->with(['system','zone.domain'])
->get();
// If we havent polled in heatbeat hours, poll system
foreach ($l as $oo) {
if (Job::where('queue','poll')->get()->pluck('command.address.id')->search($oo->id) === FALSE) {
if ((! $oo->system->last_session)
|| ($oo->system->hearbeat && ($oo->system->last_session->addHours($oo->system->heartbeat) < Carbon::now()))
|| ((! $oo->system->hearbeat) && ($oo->role < Address::NODE_ACTIVE) && ($oo->system->last_session->addHours(6) < Carbon::now())))
{
Log::info(sprintf('%s:- Polling [%s] (%s) - we havent seen them since [%s], heartbeat [%d]',
self::LOGKEY,
$oo->ftn,
$oo->system->name,
$oo->system->last_session ?: 'Never',
$oo->system->heartbeat,
));
AddressPoll::dispatch($oo);
} else {
Log::debug(sprintf('%s:= Not scheduling poll to [%s], we saw them [%s], heartbeat [%d]',
self::LOGKEY,
$oo->ftn,
$oo->system->last_session,
$oo->system->heartbeat
));
}
} else {
Log::notice(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$oo->ftn));
}
}
}
}

View File

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('systems', function (Blueprint $table) {
$table->smallInteger('heartbeat')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('systems', function (Blueprint $table) {
$table->dropColumn('heartbeat');
});
}
};

View File

@ -1,8 +1,4 @@
<!-- $o = System::class -->
@php
use App\Models\Setup;
@endphp
<div class="row">
<!-- Name -->
<div class="col-4">
@ -205,36 +201,114 @@
</div>
</div>
<div class="offset-3 col-4 @if((old('pollmode') === "0") || is_null($o->pollmode))d-none @endif" id="heartbeat_option">
@can('admin',$o)
<div class="row p-0">
<div class="offset-3 col-6">
<label for="method" class="form-label">Heartbeat <i class="bi bi-info-circle" title="Attempt contact after last seen"></i></label>
<div class="input-group has-validation">
<span class="input-group-text"><i class="bi bi-hourglass-bottom"></i></span>
<input type="text" class="form-control text-end @error('heartbeat') is-invalid @enderror" id="heartbeat" placeholder="Hours" name="heartbeat" value="{{ old('heartbeat',$o->heartbeat) }}">
<span class="invalid-feedback" role="alert">
@error('heartbeat')
{{ $message }}
@enderror
</span>
</div>
</div>
</div>
@endcan
@if (! is_null($o->pollmode))
<div class="offset-3 col-4">
<table class="table monotable m-0 p-0 small noborder">
<tbody style="border-style:dotted;">
<div class="row">
<div class="offset-3 col-9 bg-secondary rounded p-2 small">
@if($job = $o->poll())
<tr>
<td class="cap text-end">@if($job->attempts)Last Attempt @else Scheduled @endif:</td>
<td>{{ $job->created_at }} </td>
</tr>
<tr>
<td class="cap text-end">Attempts :</td>
<td>{{ $job->attempts ?: 0 }}</td>
</tr>
<div class="row p-0">
<div class="col-4 text-dark">
@if($job->attempts)Last: @else Scheduled: @endif
</div>
<div class="col-8">
<strong class="highlight">{{ $job->created_at }}</strong>
</div>
</div>
<div class="row">
<div class="col-4 text-dark">
Attempts:
</div>
<div class="col-8">
<strong class="highlight">{{ $job->attempts ?: 0 }}</strong>
</div>
</div>
@if ($job->attempts)
<tr>
<td class="cap text-end">Next Attempt :</td>
<td>{{ $job->available_at->diffForHumans(now(),$job->available_at->isFuture() ? \Carbon\CarbonInterface::DIFF_ABSOLUTE : \Carbon\CarbonInterface::DIFF_RELATIVE_TO_NOW) }}</td>
</tr>
<div class="row">
<div class="col-4 text-dark">
Next:
</div>
<div class="col-8">
<strong class="highlight">{{ $job->available_at->diffForHumans(now(),$job->available_at->isFuture() ? \Carbon\CarbonInterface::DIFF_ABSOLUTE : \Carbon\CarbonInterface::DIFF_RELATIVE_TO_NOW) }}</strong>
</div>
</div>
@endif
@else
<tr>
<td class="cap text-end">Last Poll :</td>
<td>{{ ($x=$o->logs->where('originate',TRUE)->last())?->created_at ?: 'Never' }}</td>
</tr>
<tr>
<td class="cap text-end">Method :</td>
<td>{{ $x?->sessiontype ?: '-' }}</td>
</tr>
<div class="row p-0">
<div class="col-4 text-dark">
Last Poll:
</div>
<div class="col-8">
<strong class="highlight">{{ ($x=$o->logs->where('originate',TRUE)->last())?->created_at ?: 'Never' }}</strong>
</div>
</div>
<div class="row">
<div class="col-4 text-dark">
Method:
</div>
<div class="col-8">
<strong class="highlight">{{ $x?->sessiontype ? \App\Models\Mailer::findOrFail($x->sessiontype)->name : '-' }}</strong>
</div>
</div>
@if ($o->heartbeat)
<div class="row">
<div class="col-4 text-dark">
Next Heartbeat:
</div>
<div class="col-8">
<strong class="highlight">{{ $x ? $x->created_at->addHours($o->heartbeat) : Carbon::now() }}</strong>
</div>
</div>
@endif
@endif
<div class="row">
<div class="col-4 text-dark">
Status:
</div>
<div class="col-8">
<strong class="highlight">
@if ($job) Queued
@elseif ($o->autohold)Auto Hold
@else
@switch($o->pollmode)
@case(TRUE) Crash @break;
@case(FALSE) Normal @break;
@default Hold
@endswitch
@endif
</strong>
</div>
</div>
</div>
</div>
{{--
<div class="col-12">
<table class="table monotable m-0 p-0 small noborder">
<tbody xstyle="border-style:dotted;">
<tr>
<td class="cap text-end">Status :</td>
@ -253,10 +327,12 @@
</tbody>
</table>
</div>
--}}
@endif
</div>
</div>
</div>
</div>
<div class="row">
<!-- BBS Details -->
@ -283,7 +359,6 @@
<input type="text" class="form-control text-end @error('port') is-invalid @enderror" id="port" placeholder="Port" name="port" value="{{ old('port',$o->port) }}" @cannot($action,$o)readonly @endcannot>
</div>
</div>
</div>
</div>
</div>
@ -314,3 +389,20 @@
@endif
</div>
</div>
@section('page-scripts')
<script type="text/javascript">
$(document).ready(function() {
$('#poll_normal').on('click',function() {
$('#heartbeat_option').removeClass('d-none');
})
$('#poll_crash').on('click',function() {
$('#heartbeat_option').removeClass('d-none');
})
$('#poll_hold').on('click',function() {
$('#heartbeat_option').addClass('d-none');
console.log('hold');
})
})
</script>
@append