Added usage graph (ADSL), improved logging for usage collection (ADSL)

This commit is contained in:
Deon George 2021-02-18 00:22:50 +11:00
parent 338296982b
commit a301fa7fc0
11 changed files with 258 additions and 115 deletions

View File

@ -11,6 +11,8 @@ use Illuminate\Support\Facades\Log;
abstract class Supplier abstract class Supplier
{ {
private const LOGKEY = 'AS-';
protected $o = NULL; protected $o = NULL;
protected $_columns = []; protected $_columns = [];
@ -23,16 +25,16 @@ abstract class Supplier
/** /**
* Connect and pull down traffic data * Connect and pull down traffic data
* *
* @param array $args * @return Collection
* @return mixed
*/ */
public function connect(array $args=[]) public function fetch(): Collection
{ {
if ($x=$this->mustPause()) { if ($x=$this->mustPause()) {
Log::error('API Throttle, waiting .',['m'=>__METHOD__]); Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]);
sleep($x); sleep($x);
} }
Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,$this->o->stats_lastupdate),['m'=>__METHOD__]);
$result = Cache::remember('Supplier:'.$this->o->id.$this->o->stats_lastupdate,86400,function() { $result = Cache::remember('Supplier:'.$this->o->id.$this->o->stats_lastupdate,86400,function() {
$client = $this->getClient(); $client = $this->getClient();
@ -53,14 +55,22 @@ abstract class Supplier
$api_reset = Arr::get($result->getHeader('X-RateLimit-Reset'),0); $api_reset = Arr::get($result->getHeader('X-RateLimit-Reset'),0);
if ($api_remain === 0 AND $api_reset) { if ($api_remain === 0 AND $api_reset) {
Log::error('API Throttle.',['m'=>__METHOD__]); Log::notice(sprintf('%s:API Throttle [%d].',self::LOGKEY,$api_reset),['m'=>__METHOD__]);
Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset)); Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset));
} }
return $result->getBody()->getContents(); // Assume the supplier provides an ASCII output for text/html
if (preg_match('#^text/html;#',$x=Arr::get($result->getHeader('Content-Type'),'0'))) {
return collect(explode("\n",$result->getBody()->getContents()))->filter();
} else {
Log::error(sprintf('%s:Havent handled header type [%s]',self::LOGKEY,$x),['m'=>__METHOD__]);
throw new \Exception('Unhandled Content Type');
}
}); });
Log::debug(sprintf('%s:Supplier [%d], records returned [%d]...',self::LOGKEY,$this->o->id,$result->count()),['m'=>__METHOD__]);
return $result; return $result;
} }

View File

@ -21,17 +21,7 @@ class BroadbandTraffic extends Command
* *
* @var string * @var string
*/ */
protected $description = 'Input Broadband Traffic from Suppliers'; protected $description = 'Import Broadband Traffic from Suppliers';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/** /**
* Execute the console command. * Execute the console command.
@ -40,8 +30,7 @@ class BroadbandTraffic extends Command
*/ */
public function handle() public function handle()
{ {
foreach (AdslSupplier::active()->get() as $o) { foreach (AdslSupplier::active()->get() as $o)
Job::dispatchNow($o); Job::dispatch($o);
}
} }
} }

View File

@ -0,0 +1,16 @@
<?php
namespace App\Interfaces;
use Illuminate\Support\Collection;
interface ServiceUsage
{
/**
* This service provides usage information
*
* @param int $days
* @return Collection
*/
public function usage(int $days): Collection;
}

View File

@ -28,7 +28,9 @@ class BroadbandTraffic implements ShouldQueue
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $aso = NULL; private const LOGKEY = 'JBT';
protected $aso = NULL; // The supplier we are updating from
private $class_prefix = 'App\Classes\External\Supplier\\'; private $class_prefix = 'App\Classes\External\Supplier\\';
public function __construct(AdslSupplier $o) public function __construct(AdslSupplier $o)
@ -40,13 +42,17 @@ class BroadbandTraffic implements ShouldQueue
* Execute the job. * Execute the job.
* *
* @return void * @return void
* @throws \Exception
* @todo The column stats_lastupdate is actually the "next" date that stats should be retrieved. Rename it.
*/ */
public function handle() public function handle()
{ {
Log::info(sprintf('%s:Importing Broadband Traffic from [%s]',self::LOGKEY,$this->aso->name),['m'=>__METHOD__]);
$u = 0; $u = 0;
// Load our class for this supplier
$class = $this->class_prefix.$this->aso->name; $class = $this->class_prefix.$this->aso->name;
if (class_exists($class)) { if (class_exists($class)) {
$o = new $class($this->aso); $o = new $class($this->aso);
@ -55,16 +61,17 @@ class BroadbandTraffic implements ShouldQueue
exit(1); exit(1);
} }
// Repeat pull traffic data until yesterday
while ($this->aso->stats_lastupdate < Carbon::now()->subDay()) {
Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$this->aso->stats_lastupdate->format('Y-m-d')),['m'=>__METHOD__]);
// Delete traffic, since we'll refresh it.
AdslTraffic::where('supplier_id',$this->aso->id) AdslTraffic::where('supplier_id',$this->aso->id)
->where('date',$this->aso->stats_lastupdate) ->where('date',$this->aso->stats_lastupdate)
->delete(); ->delete();
// @todo Need to trap errors from getting data
$c = 0; $c = 0;
foreach (explode("\n",$o->connect()) as $line) { foreach ($o->fetch() as $line) {
if (! trim($line))
continue;
// The first row is our header // The first row is our header
if (! $c++) { if (! $c++) {
$fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header())); $fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header()));
@ -77,8 +84,10 @@ class BroadbandTraffic implements ShouldQueue
$row = str_getcsv(trim($line)); $row = str_getcsv(trim($line));
try { try {
// @todo Put the date format in the DB.
$date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]); $date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]);
// Find the right service dependant on the dates we supplied the service
$oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')]) $oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')])
->select(DB::raw('ab_service__adsl.*')) ->select(DB::raw('ab_service__adsl.*'))
->join('ab_service','ab_service.id','=','service_id') ->join('ab_service','ab_service.id','=','service_id')
@ -89,54 +98,48 @@ class BroadbandTraffic implements ShouldQueue
}) })
->get(); ->get();
$to = new AdslTraffic;
$to->site_id = 1; // @todo TO ADDRESS
$to->date = $this->aso->stats_lastupdate;
$to->supplier_id = $this->aso->id;
$to->up_peak = $row[$o->getColumnKey('Peak upload')];
$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
$to->down_peak = $row[$o->getColumnKey('Peak download')];
$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
// $to->peer
// $to->internal
$to->time = '24:00'; // @todo
// If we have no records // If we have no records
if ($oo->count() != 1) { if ($oo->count() != 1) {
Log::error(sprintf('! Records Errors for:%s (%s) [%s]',$row[$o->getColumnKey('Login')],$date,$oo->count())); Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['m'=>__METHOD__,'date'=>$date,'count'=>$oo->count()]);
$to = new AdslTraffic;
$to->site_id = 1; // @todo TO ADDRESS
$to->service = $row[$o->getColumnKey('Login')]; $to->service = $row[$o->getColumnKey('Login')];
$to->date = $this->aso->stats_lastupdate;
$to->supplier_id = $this->aso->id;
$to->up_peak = $row[$o->getColumnKey('Peak upload')];
$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
$to->down_peak = $row[$o->getColumnKey('Peak download')];
$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
// $to->peer
// $to->internal
$to->time = '24:00'; // @todo
$to->save(); $to->save();
$u++;
} else { } else {
$to = new AdslTraffic;
$to->site_id = 1; // @todo TO ADDRESS
$to->date = $this->aso->stats_lastupdate;
$to->supplier_id = $this->aso->id;
$to->up_peak = $row[$o->getColumnKey('Peak upload')];
$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
$to->down_peak = $row[$o->getColumnKey('Peak download')];
$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
// $to->peer
// $to->internal
$to->time = '24:00'; // @todo
$oo->first()->traffic()->save($to); $oo->first()->traffic()->save($to);
$u++;
} }
$u++;
} catch (\Exception $e) { } catch (\Exception $e) {
dd(['row'=>$row,'line'=>$line]); Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['m'=>__METHOD__,'row'=>$row,'line'=>$line]);
throw new \Exception('Error while storing traffic date');
} }
} }
Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$this->aso->stats_lastupdate->format('Y-m-d')),['m'=>__METHOD__]);
if ($u) { if ($u) {
$this->aso->stats_lastupdate = $this->aso->stats_lastupdate->addDay(); $this->aso->stats_lastupdate = $this->aso->stats_lastupdate->addDay();
$this->aso->save(); $this->aso->save();
if ($this->aso->traffic_mismatch($date)->count()) if ($this->aso->trafficMismatch($date)->count())
Mail::to('deon@graytech.net.au') // @todo To change Mail::to('deon@graytech.net.au') // @todo To change
->send(new TrafficMismatch($this->aso,$date)); ->send(new TrafficMismatch($this->aso,$date));
} }
}
Log::info(sprintf('%s: Records Imported: %d',get_class($this),$u));
} }
} }

View File

@ -29,7 +29,13 @@ class AdslSupplier extends Model
/** METHODS **/ /** METHODS **/
public function traffic_mismatch(Carbon $date): Collection /**
* Return the traffic records, that were not matched to a service.
*
* @param Carbon $date
* @return Collection
*/
public function trafficMismatch(Carbon $date): Collection
{ {
return AdslTraffic::where('date',$date->format('Y-m-d')) return AdslTraffic::where('date',$date->format('Y-m-d'))
->where('supplier_id',$this->id) ->where('supplier_id',$this->id)

View File

@ -28,6 +28,8 @@ class Product extends Model
protected $with = ['descriptions']; protected $with = ['descriptions'];
/* RELATIONS */
public function descriptions() public function descriptions()
{ {
return $this->hasMany(ProductTranslate::class); return $this->hasMany(ProductTranslate::class);
@ -48,6 +50,8 @@ class Product extends Model
return $this->morphTo(null,'model','prod_plugin_data'); return $this->morphTo(null,'model','prod_plugin_data');
} }
/* ATTRIBUTES */
/** /**
* Get the service category (from the product) * Get the service category (from the product)
* *
@ -161,6 +165,17 @@ class Product extends Model
return Arr::get($this->price_array,sprintf('%s.1.price_setup',$this->price_recurr_default))*1.1; return Arr::get($this->price_array,sprintf('%s.1.price_setup',$this->price_recurr_default))*1.1;
} }
/**
* Return if this product captures usage data
*
* @return bool
*/
public function hasUsage(): bool
{
// @todo This should be configured in the DB
return in_array($this->model, ['App\Models\Product\Adsl']);
}
public function scopeActive() public function scopeActive()
{ {
return $this->where('active',TRUE); return $this->where('active',TRUE);

View File

@ -228,6 +228,8 @@ class Service extends Model
], ],
]; ];
/* RELATIONS */
/** /**
* Account the service belongs to * Account the service belongs to
* *
@ -335,7 +337,7 @@ class Service extends Model
return $this->morphTo(null,'model','id','service_id'); return $this->morphTo(null,'model','id','service_id');
} }
/** SCOPES **/ /* SCOPES */
/** /**
* Only query active categories * Only query active categories
@ -384,7 +386,7 @@ class Service extends Model
return $query->where('id','like','%'.$term.'%'); return $query->where('id','like','%'.$term.'%');
} }
/** ATTRIBUTES **/ /* ATTRIBUTES */
/** /**
* Name of the account for this service * Name of the account for this service
@ -846,7 +848,7 @@ class Service extends Model
return sprintf('<a href="/u/service/%s">%s</a>',$this->id,$this->service_id); return sprintf('<a href="/u/service/%s">%s</a>',$this->id,$this->service_id);
} }
/** SETTERS **/ /* SETTERS */
public function setDateOrigAttribute($value) public function setDateOrigAttribute($value)
{ {
@ -858,7 +860,7 @@ class Service extends Model
$this->attributes['date_last'] = $value->timestamp; $this->attributes['date_last'] = $value->timestamp;
} }
/** FUNCTIONS **/ /* FUNCTIONS */
// The action methods will return: NULL for no progress|FALSE for a failed status|next stage name. // The action methods will return: NULL for no progress|FALSE for a failed status|next stage name.
@ -1140,6 +1142,16 @@ class Service extends Model
}); });
} }
/**
* Does this service have traffic data to be graphed
*
* @return bool
*/
public function hasUsage(): bool
{
return $this->product->hasUsage();
}
/** /**
* Determine if a service is active. It is active, if active=1, or the order_status is not in inactive_status[] * Determine if a service is active. It is active, if active=1, or the order_status is not in inactive_status[]
* *

View File

@ -2,16 +2,19 @@
namespace App\Models\Service; namespace App\Models\Service;
use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Support\Collection;
use Illuminate\Database\Eloquent\Relations\MorphOne;
use App\Interfaces\ServiceItem; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use App\Interfaces\{ServiceItem,ServiceUsage};
use App\Models\AdslSupplierPlan;
use App\Models\Base\ServiceType; use App\Models\Base\ServiceType;
use App\Models\Service;
use App\Traits\NextKey; use App\Traits\NextKey;
class Adsl extends ServiceType implements ServiceItem class Adsl extends ServiceType implements ServiceItem,ServiceUsage
{ {
private const LOGKEY = 'MSA';
use NextKey; use NextKey;
const RECORD_ID = 'service__adsl'; const RECORD_ID = 'service__adsl';
@ -30,6 +33,7 @@ class Adsl extends ServiceType implements ServiceItem
*/ */
public function traffic() public function traffic()
{ {
// @todo Need to include site_id in this relation
return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id'); return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id');
} }
@ -91,4 +95,24 @@ class Adsl extends ServiceType implements ServiceItem
{ {
return $this->service_contract_date AND $this->service_contract_date->addMonths($this->contract_term)->isFuture(); return $this->service_contract_date AND $this->service_contract_date->addMonths($this->contract_term)->isFuture();
} }
/**
* Return service usage data
*
* @param int $days
* @return Collection
*/
public function usage(int $days=31): Collection
{
$maxdate = self::traffic()
->select(DB::raw('max(date) as max'))
->pluck('max')->pop();
Log::debug(sprintf('%s:Getting Usage data for [%d] days from [%s]',self::LOGKEY,$days,$maxdate),['m'=>__METHOD__]);
return $this->traffic()
->where('date','<=',$maxdate)
->where('date','>=',DB::raw(sprintf('date_sub(\'%s\',INTERVAL %s DAY)',$maxdate,$days)))
->get();
}
} }

View File

@ -8,6 +8,7 @@ class AdslTraffic extends Model
{ {
protected $table = 'ab_service__adsl_traffic'; protected $table = 'ab_service__adsl_traffic';
public $timestamps = FALSE; public $timestamps = FALSE;
protected $dates = ['date'];
public function broadband() public function broadband()
{ {

View File

@ -25,7 +25,6 @@
<div class="col-7"> <div class="col-7">
<div class="card"> <div class="card">
<div class="card-header bg-dark d-flex p-0"> <div class="card-header bg-dark d-flex p-0">
<span class="p-3"><i class="fa fa-bars"></i></span>
<ul class="nav nav-pills p-2 w-100"> <ul class="nav nav-pills p-2 w-100">
{{-- {{--
<li class="nav-item"><a class="nav-link active" href="#product" data-toggle="tab">Product</a></li> <li class="nav-item"><a class="nav-link active" href="#product" data-toggle="tab">Product</a></li>
@ -34,6 +33,10 @@
@if (! $o->suspend_billing AND ! $o->external_billing) @if (! $o->suspend_billing AND ! $o->external_billing)
<li class="nav-item active"><a class="nav-link" href="#pending_items" data-toggle="tab">Pending Items</a></li> <li class="nav-item active"><a class="nav-link" href="#pending_items" data-toggle="tab">Pending Items</a></li>
@endif @endif
@if ($o->hasUsage())
<li class="nav-item active"><a class="nav-link" href="#traffic" data-toggle="tab">Traffic</a></li>
@endif
{{-- {{--
<li class="nav-item"><a class="nav-link" href="#invoices" data-toggle="tab">Invoices</a></li> <li class="nav-item"><a class="nav-link" href="#invoices" data-toggle="tab">Invoices</a></li>
<li class="nav-item"><a class="nav-link" href="#emails" data-toggle="tab">Emails</a></li> <li class="nav-item"><a class="nav-link" href="#emails" data-toggle="tab">Emails</a></li>
@ -62,9 +65,6 @@
<div class="card-body"> <div class="card-body">
<div class="tab-content"> <div class="tab-content">
<div class="tab-pane fade" id="traffic" role="tabpanel">
Traffic.
</div>
<div class="tab-pane fade" id="product" role="tabpanel"> <div class="tab-pane fade" id="product" role="tabpanel">
Product. Product.
</div> </div>
@ -73,6 +73,11 @@
@include('common.service.widget.invoice') @include('common.service.widget.invoice')
</div> </div>
@endif @endif
@if ($o->hasUsage())
<div class="tab-pane fade show" id="traffic" role="tabpanel">
@include('u.service.widgets.'.$o->stype.'.usagegraph',['o'=>$o->type])
</div>
@endif
<div class="tab-pane fade" id="invoices" role="tabpanel"> <div class="tab-pane fade" id="invoices" role="tabpanel">
Invoices. Invoices.
</div> </div>

View File

@ -0,0 +1,62 @@
<div class="card">
<div class="card-header bg-gray-dark">
<h3 class="card-title">Broadband Traffic</h3>
</div>
<div class="card-body">
<div id="graph"></div>
</div>
</div>
@section('page-scripts')
@js('//code.highcharts.com/highcharts.js','highcharts')
@js('//code.highcharts.com/highcharts-more.js','highcharts-more','highcharts')
@js('//code.highcharts.com/modules/xrange.js','highcharts-xrange','highcharts')
@js('//code.highcharts.com/modules/exporting.js','highcharts-export','highcharts')
@js('//code.highcharts.com/modules/offline-exporting.js','highcharts-export-offline','highcharts-export')
<script>
Highcharts.chart('graph', {
chart: {
type: 'areaspline'
},
title: {
text: 'Usage Traffic up to {{ $o->usage(30)->max('date')->format('Y-m-d') }}'
},
legend: {
layout: 'vertical',
align: 'left',
verticalAlign: 'top',
x: 150,
y: 100,
floating: true,
borderWidth: 1,
backgroundColor:
Highcharts.defaultOptions.legend.backgroundColor || '#FFFFFF'
},
xAxis: {
type: 'datetime'
},
yAxis: {
title: {
text: 'MB'
}
},
tooltip: {
shared: true,
valueSuffix: ' MB'
},
credits: {
enabled: false
},
plotOptions: {
areaspline: {
fillOpacity: 0.5
}
},
series: [{
name: 'Traffic',
data: {!! $o->usage(30)->map(function($item) { return ['x'=>$item->date->timestamp*1000,'y'=>$item->up_peak+$item->down_peak+$item->up_offpeak+$item->down_offpeak];}) !!}
}]
});
</script>
@append