deon
/
osb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added usage graph (ADSL), improved logging for usage collection (ADSL)

This commit is contained in:
Deon George 2021-02-18 00:22:50 +11:00
parent 338296982b
commit a301fa7fc0
11 changed files with 258 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
app/Classes/External/Supplier.php vendored
View File

@ -11,6 +11,8 @@ use Illuminate\Support\Facades\Log;
abstract class Supplier
{
private const LOGKEY = 'AS-';
protected $o = NULL;
protected $_columns = [];
@ -23,16 +25,16 @@ abstract class Supplier
/**
* Connect and pull down traffic data
*
* @param array $args
* @return mixed
* @return Collection
*/
public function connect(array $args=[])
public function fetch(): Collection
{
if ($x=$this->mustPause()) {
Log::error('API Throttle, waiting .',['m'=>__METHOD__]);
Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]);
sleep($x);
}
Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,$this->o->stats_lastupdate),['m'=>__METHOD__]);
$result = Cache::remember('Supplier:'.$this->o->id.$this->o->stats_lastupdate,86400,function() {
$client = $this->getClient();
@ -53,14 +55,22 @@ abstract class Supplier
$api_reset = Arr::get($result->getHeader('X-RateLimit-Reset'),0);
if ($api_remain === 0 AND $api_reset) {
Log::error('API Throttle.',['m'=>__METHOD__]);
Log::notice(sprintf('%s:API Throttle [%d].',self::LOGKEY,$api_reset),['m'=>__METHOD__]);
Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset));
}
return $result->getBody()->getContents();
// Assume the supplier provides an ASCII output for text/html
if (preg_match('#^text/html;#',$x=Arr::get($result->getHeader('Content-Type'),'0'))) {
return collect(explode("\n",$result->getBody()->getContents()))->filter();
} else {
Log::error(sprintf('%s:Havent handled header type [%s]',self::LOGKEY,$x),['m'=>__METHOD__]);
throw new \Exception('Unhandled Content Type');
}
});
Log::debug(sprintf('%s:Supplier [%d], records returned [%d]...',self::LOGKEY,$this->o->id,$result->count()),['m'=>__METHOD__]);
return $result;
}

17
app/Console/Commands/BroadbandTraffic.php
View File

@ -21,17 +21,7 @@ class BroadbandTraffic extends Command
*
* @var string
*/
protected $description = 'Input Broadband Traffic from Suppliers';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
protected $description = 'Import Broadband Traffic from Suppliers';
/**
* Execute the console command.
@ -40,8 +30,7 @@ class BroadbandTraffic extends Command
*/
public function handle()
{
foreach (AdslSupplier::active()->get() as $o) {
Job::dispatchNow($o);
}
foreach (AdslSupplier::active()->get() as $o)
Job::dispatch($o);
}
}

16
app/Interfaces/ServiceUsage.php Normal file
View File

@ -0,0 +1,16 @@
<?php
namespace App\Interfaces;
use Illuminate\Support\Collection;
interface ServiceUsage
{
/**
* This service provides usage information
*
* @param int $days
* @return Collection
*/
public function usage(int $days): Collection;
}

163
app/Jobs/BroadbandTraffic.php
View File

@ -28,7 +28,9 @@ class BroadbandTraffic implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $aso = NULL;
private const LOGKEY = 'JBT';
protected $aso = NULL; // The supplier we are updating from
private $class_prefix = 'App\Classes\External\Supplier\\';
public function __construct(AdslSupplier $o)
@ -40,13 +42,17 @@ class BroadbandTraffic implements ShouldQueue
* Execute the job.
*
* @return void
* @throws \Exception
* @todo The column stats_lastupdate is actually the "next" date that stats should be retrieved. Rename it.
*/
public function handle()
{
Log::info(sprintf('%s:Importing Broadband Traffic from [%s]',self::LOGKEY,$this->aso->name),['m'=>__METHOD__]);
$u = 0;
// Load our class for this supplier
$class = $this->class_prefix.$this->aso->name;
if (class_exists($class)) {
$o = new $class($this->aso);
@ -55,88 +61,85 @@ class BroadbandTraffic implements ShouldQueue
exit(1);
}
AdslTraffic::where('supplier_id',$this->aso->id)
->where('date',$this->aso->stats_lastupdate)
->delete();
// Repeat pull traffic data until yesterday
while ($this->aso->stats_lastupdate < Carbon::now()->subDay()) {
Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$this->aso->stats_lastupdate->format('Y-m-d')),['m'=>__METHOD__]);
// @todo Need to trap errors from getting data
$c = 0;
foreach (explode("\n",$o->connect()) as $line) {
if (! trim($line))
continue;
// Delete traffic, since we'll refresh it.
AdslTraffic::where('supplier_id',$this->aso->id)
->where('date',$this->aso->stats_lastupdate)
->delete();
// The first row is our header
if (! $c++) {
$fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header()));
continue;
}
if (! $fields->count())
abort(500,'? No fields in data exportupda');
$row = str_getcsv(trim($line));
try {
$date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]);
$oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')])
->select(DB::raw('ab_service__adsl.*'))
->join('ab_service','ab_service.id','=','service_id')
->where('ab_service.date_start','<=',$date->format('U'))
->where(function($query) use ($date) {
$query->whereNULL('ab_service.date_end')
->orWhere('ab_service.date_end','<=',$date->format('U'));
})
->get();
// If we have no records
if ($oo->count() != 1) {
Log::error(sprintf('! Records Errors for:%s (%s) [%s]',$row[$o->getColumnKey('Login')],$date,$oo->count()));
$to = new AdslTraffic;
$to->site_id = 1; // @todo TO ADDRESS
$to->service = $row[$o->getColumnKey('Login')];
$to->date = $this->aso->stats_lastupdate;
$to->supplier_id = $this->aso->id;
$to->up_peak = $row[$o->getColumnKey('Peak upload')];
$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
$to->down_peak = $row[$o->getColumnKey('Peak download')];
$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
// $to->peer
// $to->internal
$to->time = '24:00'; // @todo
$to->save();
$u++;
} else {
$to = new AdslTraffic;
$to->site_id = 1; // @todo TO ADDRESS
$to->date = $this->aso->stats_lastupdate;
$to->supplier_id = $this->aso->id;
$to->up_peak = $row[$o->getColumnKey('Peak upload')];
$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
$to->down_peak = $row[$o->getColumnKey('Peak download')];
$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
// $to->peer
// $to->internal
$to->time = '24:00'; // @todo
$oo->first()->traffic()->save($to);
$u++;
$c = 0;
foreach ($o->fetch() as $line) {
// The first row is our header
if (! $c++) {
$fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header()));
continue;
}
} catch (\Exception $e) {
dd(['row'=>$row,'line'=>$line]);
if (! $fields->count())
abort(500,'? No fields in data exportupda');
$row = str_getcsv(trim($line));
try {
// @todo Put the date format in the DB.
$date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]);
// Find the right service dependant on the dates we supplied the service
$oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')])
->select(DB::raw('ab_service__adsl.*'))
->join('ab_service','ab_service.id','=','service_id')
->where('ab_service.date_start','<=',$date->format('U'))
->where(function($query) use ($date) {
$query->whereNULL('ab_service.date_end')
->orWhere('ab_service.date_end','<=',$date->format('U'));
})
->get();
$to = new AdslTraffic;
$to->site_id = 1; // @todo TO ADDRESS
$to->date = $this->aso->stats_lastupdate;
$to->supplier_id = $this->aso->id;
$to->up_peak = $row[$o->getColumnKey('Peak upload')];
$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
$to->down_peak = $row[$o->getColumnKey('Peak download')];
$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
// $to->peer
// $to->internal
$to->time = '24:00'; // @todo
// If we have no records
if ($oo->count() != 1) {
Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['m'=>__METHOD__,'date'=>$date,'count'=>$oo->count()]);
$to->service = $row[$o->getColumnKey('Login')];
$to->save();
} else {
$oo->first()->traffic()->save($to);
}
$u++;
} catch (\Exception $e) {
Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['m'=>__METHOD__,'row'=>$row,'line'=>$line]);
throw new \Exception('Error while storing traffic date');
}
}
Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$this->aso->stats_lastupdate->format('Y-m-d')),['m'=>__METHOD__]);
if ($u) {
$this->aso->stats_lastupdate = $this->aso->stats_lastupdate->addDay();
$this->aso->save();
if ($this->aso->trafficMismatch($date)->count())
Mail::to('deon@graytech.net.au') // @todo To change
->send(new TrafficMismatch($this->aso,$date));
}
}
if ($u) {
$this->aso->stats_lastupdate = $this->aso->stats_lastupdate->addDay();
$this->aso->save();
if ($this->aso->traffic_mismatch($date)->count())
Mail::to('deon@graytech.net.au') // @todo To change
->send(new TrafficMismatch($this->aso,$date));
}
Log::info(sprintf('%s: Records Imported: %d',get_class($this),$u));
}
}

8
app/Models/AdslSupplier.php
View File

@ -29,7 +29,13 @@ class AdslSupplier extends Model
/** METHODS **/
public function traffic_mismatch(Carbon $date): Collection
/**
* Return the traffic records, that were not matched to a service.
*
* @param Carbon $date
* @return Collection
*/
public function trafficMismatch(Carbon $date): Collection
{
return AdslTraffic::where('date',$date->format('Y-m-d'))
->where('supplier_id',$this->id)

15
app/Models/Product.php
View File

@ -28,6 +28,8 @@ class Product extends Model
protected $with = ['descriptions'];
/* RELATIONS */
public function descriptions()
{
return $this->hasMany(ProductTranslate::class);
@ -48,6 +50,8 @@ class Product extends Model
return $this->morphTo(null,'model','prod_plugin_data');
}
/* ATTRIBUTES */
/**
* Get the service category (from the product)
*
@ -161,6 +165,17 @@ class Product extends Model
return Arr::get($this->price_array,sprintf('%s.1.price_setup',$this->price_recurr_default))*1.1;
}
/**
* Return if this product captures usage data
*
* @return bool
*/
public function hasUsage(): bool
{
// @todo This should be configured in the DB
return in_array($this->model, ['App\Models\Product\Adsl']);
}
public function scopeActive()
{
return $this->where('active',TRUE);

20
app/Models/Service.php
View File

@ -228,6 +228,8 @@ class Service extends Model
],
];
/* RELATIONS */
/**
* Account the service belongs to
*
@ -335,7 +337,7 @@ class Service extends Model
return $this->morphTo(null,'model','id','service_id');
}
/** SCOPES **/
/* SCOPES */
/**
* Only query active categories
@ -384,7 +386,7 @@ class Service extends Model
return $query->where('id','like','%'.$term.'%');
}
/** ATTRIBUTES **/
/* ATTRIBUTES */
/**
* Name of the account for this service
@ -846,7 +848,7 @@ class Service extends Model
return sprintf('<a href="/u/service/%s">%s</a>',$this->id,$this->service_id);
}
/** SETTERS **/
/* SETTERS */
public function setDateOrigAttribute($value)
{
@ -858,7 +860,7 @@ class Service extends Model
$this->attributes['date_last'] = $value->timestamp;
}
/** FUNCTIONS **/
/* FUNCTIONS */
// The action methods will return: NULL for no progress|FALSE for a failed status|next stage name.
@ -1140,6 +1142,16 @@ class Service extends Model
});
}
/**
* Does this service have traffic data to be graphed
*
* @return bool
*/
public function hasUsage(): bool
{
return $this->product->hasUsage();
}
/**
* Determine if a service is active. It is active, if active=1, or the order_status is not in inactive_status[]
*

36
app/Models/Service/Adsl.php
View File

@ -2,16 +2,19 @@
namespace App\Models\Service;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\MorphOne;
use Illuminate\Support\Collection;
use App\Interfaces\ServiceItem;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use App\Interfaces\{ServiceItem,ServiceUsage};
use App\Models\AdslSupplierPlan;
use App\Models\Base\ServiceType;
use App\Models\Service;
use App\Traits\NextKey;
class Adsl extends ServiceType implements ServiceItem
class Adsl extends ServiceType implements ServiceItem,ServiceUsage
{
private const LOGKEY = 'MSA';
use NextKey;
const RECORD_ID = 'service__adsl';
@ -30,6 +33,7 @@ class Adsl extends ServiceType implements ServiceItem
*/
public function traffic()
{
// @todo Need to include site_id in this relation
return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id');
}
@ -91,4 +95,24 @@ class Adsl extends ServiceType implements ServiceItem
{
return $this->service_contract_date AND $this->service_contract_date->addMonths($this->contract_term)->isFuture();
}
}
/**
* Return service usage data
*
* @param int $days
* @return Collection
*/
public function usage(int $days=31): Collection
{
$maxdate = self::traffic()
->select(DB::raw('max(date) as max'))
->pluck('max')->pop();
Log::debug(sprintf('%s:Getting Usage data for [%d] days from [%s]',self::LOGKEY,$days,$maxdate),['m'=>__METHOD__]);
return $this->traffic()
->where('date','<=',$maxdate)
->where('date','>=',DB::raw(sprintf('date_sub(\'%s\',INTERVAL %s DAY)',$maxdate,$days)))
->get();
}
}

1
app/Models/Service/AdslTraffic.php
View File

@ -8,6 +8,7 @@ class AdslTraffic extends Model
{
protected $table = 'ab_service__adsl_traffic';
public $timestamps = FALSE;
protected $dates = ['date'];
public function broadband()
{

13
resources/views/theme/backend/adminlte/u/service/home.blade.php
View File

@ -25,7 +25,6 @@
<div class="col-7">
<div class="card">
<div class="card-header bg-dark d-flex p-0">
<span class="p-3"><i class="fa fa-bars"></i></span>
<ul class="nav nav-pills p-2 w-100">
{{--
<li class="nav-item"><a class="nav-link active" href="#product" data-toggle="tab">Product</a></li>
@ -34,6 +33,10 @@
@if (! $o->suspend_billing AND ! $o->external_billing)
<li class="nav-item active"><a class="nav-link" href="#pending_items" data-toggle="tab">Pending Items</a></li>
@endif
@if ($o->hasUsage())
<li class="nav-item active"><a class="nav-link" href="#traffic" data-toggle="tab">Traffic</a></li>
@endif
{{--
<li class="nav-item"><a class="nav-link" href="#invoices" data-toggle="tab">Invoices</a></li>
<li class="nav-item"><a class="nav-link" href="#emails" data-toggle="tab">Emails</a></li>
@ -62,9 +65,6 @@
<div class="card-body">
<div class="tab-content">
<div class="tab-pane fade" id="traffic" role="tabpanel">
Traffic.
</div>
<div class="tab-pane fade" id="product" role="tabpanel">
Product.
</div>
@ -73,6 +73,11 @@
@include('common.service.widget.invoice')
</div>
@endif
@if ($o->hasUsage())
<div class="tab-pane fade show" id="traffic" role="tabpanel">
@include('u.service.widgets.'.$o->stype.'.usagegraph',['o'=>$o->type])
</div>
@endif
<div class="tab-pane fade" id="invoices" role="tabpanel">
Invoices.
</div>

62
resources/views/theme/backend/adminlte/u/service/widgets/broadband/usagegraph.blade.php Normal file
View File

@ -0,0 +1,62 @@
<div class="card">
<div class="card-header bg-gray-dark">
<h3 class="card-title">Broadband Traffic</h3>
</div>
<div class="card-body">
<div id="graph"></div>
</div>
</div>
@section('page-scripts')
@js('//code.highcharts.com/highcharts.js','highcharts')
@js('//code.highcharts.com/highcharts-more.js','highcharts-more','highcharts')
@js('//code.highcharts.com/modules/xrange.js','highcharts-xrange','highcharts')
@js('//code.highcharts.com/modules/exporting.js','highcharts-export','highcharts')
@js('//code.highcharts.com/modules/offline-exporting.js','highcharts-export-offline','highcharts-export')
<script>
Highcharts.chart('graph', {
chart: {
type: 'areaspline'
},
title: {
text: 'Usage Traffic up to {{ $o->usage(30)->max('date')->format('Y-m-d') }}'
},
legend: {
layout: 'vertical',
align: 'left',
verticalAlign: 'top',
x: 150,
y: 100,
floating: true,
borderWidth: 1,
backgroundColor:
Highcharts.defaultOptions.legend.backgroundColor || '#FFFFFF'
},
xAxis: {
type: 'datetime'
},
yAxis: {
title: {
text: 'MB'
}
},
tooltip: {
shared: true,
valueSuffix: ' MB'
},
credits: {
enabled: false
},
plotOptions: {
areaspline: {
fillOpacity: 0.5
}
},
series: [{
name: 'Traffic',
data: {!! $o->usage(30)->map(function($item) { return ['x'=>$item->date->timestamp*1000,'y'=>$item->up_peak+$item->down_peak+$item->up_offpeak+$item->down_offpeak];}) !!}
}]
});
</script>
@append