From a301fa7fc09b0ecac8d2d66ba09cda3fe0a0ade4 Mon Sep 17 00:00:00 2001 From: Deon George Date: Thu, 18 Feb 2021 00:22:50 +1100 Subject: [PATCH] Added usage graph (ADSL), improved logging for usage collection (ADSL) --- app/Classes/External/Supplier.php | 22 ++- app/Console/Commands/BroadbandTraffic.php | 17 +- app/Interfaces/ServiceUsage.php | 16 ++ app/Jobs/BroadbandTraffic.php | 163 +++++++++--------- app/Models/AdslSupplier.php | 8 +- app/Models/Product.php | 15 ++ app/Models/Service.php | 20 ++- app/Models/Service/Adsl.php | 36 +++- app/Models/Service/AdslTraffic.php | 1 + .../backend/adminlte/u/service/home.blade.php | 13 +- .../widgets/broadband/usagegraph.blade.php | 62 +++++++ 11 files changed, 258 insertions(+), 115 deletions(-) create mode 100644 app/Interfaces/ServiceUsage.php create mode 100644 resources/views/theme/backend/adminlte/u/service/widgets/broadband/usagegraph.blade.php diff --git a/app/Classes/External/Supplier.php b/app/Classes/External/Supplier.php index f6620fb..2049b2d 100644 --- a/app/Classes/External/Supplier.php +++ b/app/Classes/External/Supplier.php @@ -11,6 +11,8 @@ use Illuminate\Support\Facades\Log; abstract class Supplier { + private const LOGKEY = 'AS-'; + protected $o = NULL; protected $_columns = []; @@ -23,16 +25,16 @@ abstract class Supplier /** * Connect and pull down traffic data * - * @param array $args - * @return mixed + * @return Collection */ - public function connect(array $args=[]) + public function fetch(): Collection { if ($x=$this->mustPause()) { - Log::error('API Throttle, waiting .',['m'=>__METHOD__]); + Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]); sleep($x); } + Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,$this->o->stats_lastupdate),['m'=>__METHOD__]); $result = Cache::remember('Supplier:'.$this->o->id.$this->o->stats_lastupdate,86400,function() { $client = $this->getClient(); @@ -53,14 +55,22 @@ abstract class Supplier $api_reset = Arr::get($result->getHeader('X-RateLimit-Reset'),0); if ($api_remain === 0 AND $api_reset) { - Log::error('API Throttle.',['m'=>__METHOD__]); + Log::notice(sprintf('%s:API Throttle [%d].',self::LOGKEY,$api_reset),['m'=>__METHOD__]); Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset)); } - return $result->getBody()->getContents(); + // Assume the supplier provides an ASCII output for text/html + if (preg_match('#^text/html;#',$x=Arr::get($result->getHeader('Content-Type'),'0'))) { + return collect(explode("\n",$result->getBody()->getContents()))->filter(); + } else { + Log::error(sprintf('%s:Havent handled header type [%s]',self::LOGKEY,$x),['m'=>__METHOD__]); + throw new \Exception('Unhandled Content Type'); + } }); + Log::debug(sprintf('%s:Supplier [%d], records returned [%d]...',self::LOGKEY,$this->o->id,$result->count()),['m'=>__METHOD__]); + return $result; } diff --git a/app/Console/Commands/BroadbandTraffic.php b/app/Console/Commands/BroadbandTraffic.php index 998e81e..20a5f15 100644 --- a/app/Console/Commands/BroadbandTraffic.php +++ b/app/Console/Commands/BroadbandTraffic.php @@ -21,17 +21,7 @@ class BroadbandTraffic extends Command * * @var string */ - protected $description = 'Input Broadband Traffic from Suppliers'; - - /** - * Create a new command instance. - * - * @return void - */ - public function __construct() - { - parent::__construct(); - } + protected $description = 'Import Broadband Traffic from Suppliers'; /** * Execute the console command. @@ -40,8 +30,7 @@ class BroadbandTraffic extends Command */ public function handle() { - foreach (AdslSupplier::active()->get() as $o) { - Job::dispatchNow($o); - } + foreach (AdslSupplier::active()->get() as $o) + Job::dispatch($o); } } \ No newline at end of file diff --git a/app/Interfaces/ServiceUsage.php b/app/Interfaces/ServiceUsage.php new file mode 100644 index 0000000..713fee0 --- /dev/null +++ b/app/Interfaces/ServiceUsage.php @@ -0,0 +1,16 @@ +aso->name),['m'=>__METHOD__]); + $u = 0; + // Load our class for this supplier $class = $this->class_prefix.$this->aso->name; - if (class_exists($class)) { $o = new $class($this->aso); @@ -55,88 +61,85 @@ class BroadbandTraffic implements ShouldQueue exit(1); } - AdslTraffic::where('supplier_id',$this->aso->id) - ->where('date',$this->aso->stats_lastupdate) - ->delete(); + // Repeat pull traffic data until yesterday + while ($this->aso->stats_lastupdate < Carbon::now()->subDay()) { + Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$this->aso->stats_lastupdate->format('Y-m-d')),['m'=>__METHOD__]); - // @todo Need to trap errors from getting data - $c = 0; - foreach (explode("\n",$o->connect()) as $line) { - if (! trim($line)) - continue; + // Delete traffic, since we'll refresh it. + AdslTraffic::where('supplier_id',$this->aso->id) + ->where('date',$this->aso->stats_lastupdate) + ->delete(); - // The first row is our header - if (! $c++) { - $fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header())); - continue; - } - - if (! $fields->count()) - abort(500,'? No fields in data exportupda'); - - $row = str_getcsv(trim($line)); - - try { - $date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]); - - $oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')]) - ->select(DB::raw('ab_service__adsl.*')) - ->join('ab_service','ab_service.id','=','service_id') - ->where('ab_service.date_start','<=',$date->format('U')) - ->where(function($query) use ($date) { - $query->whereNULL('ab_service.date_end') - ->orWhere('ab_service.date_end','<=',$date->format('U')); - }) - ->get(); - - // If we have no records - if ($oo->count() != 1) { - Log::error(sprintf('! Records Errors for:%s (%s) [%s]',$row[$o->getColumnKey('Login')],$date,$oo->count())); - - $to = new AdslTraffic; - $to->site_id = 1; // @todo TO ADDRESS - $to->service = $row[$o->getColumnKey('Login')]; - $to->date = $this->aso->stats_lastupdate; - $to->supplier_id = $this->aso->id; - $to->up_peak = $row[$o->getColumnKey('Peak upload')]; - $to->up_offpeak = $row[$o->getColumnKey('Off peak upload')]; - $to->down_peak = $row[$o->getColumnKey('Peak download')]; - $to->down_offpeak = $row[$o->getColumnKey('Off peak download')]; - // $to->peer - // $to->internal - $to->time = '24:00'; // @todo - $to->save(); - $u++; - - } else { - $to = new AdslTraffic; - $to->site_id = 1; // @todo TO ADDRESS - $to->date = $this->aso->stats_lastupdate; - $to->supplier_id = $this->aso->id; - $to->up_peak = $row[$o->getColumnKey('Peak upload')]; - $to->up_offpeak = $row[$o->getColumnKey('Off peak upload')]; - $to->down_peak = $row[$o->getColumnKey('Peak download')]; - $to->down_offpeak = $row[$o->getColumnKey('Off peak download')]; - // $to->peer - // $to->internal - $to->time = '24:00'; // @todo - $oo->first()->traffic()->save($to); - $u++; + $c = 0; + foreach ($o->fetch() as $line) { + // The first row is our header + if (! $c++) { + $fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header())); + continue; } - } catch (\Exception $e) { - dd(['row'=>$row,'line'=>$line]); + + if (! $fields->count()) + abort(500,'? No fields in data exportupda'); + + $row = str_getcsv(trim($line)); + + try { + // @todo Put the date format in the DB. + $date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]); + + // Find the right service dependant on the dates we supplied the service + $oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')]) + ->select(DB::raw('ab_service__adsl.*')) + ->join('ab_service','ab_service.id','=','service_id') + ->where('ab_service.date_start','<=',$date->format('U')) + ->where(function($query) use ($date) { + $query->whereNULL('ab_service.date_end') + ->orWhere('ab_service.date_end','<=',$date->format('U')); + }) + ->get(); + + $to = new AdslTraffic; + $to->site_id = 1; // @todo TO ADDRESS + $to->date = $this->aso->stats_lastupdate; + $to->supplier_id = $this->aso->id; + $to->up_peak = $row[$o->getColumnKey('Peak upload')]; + $to->up_offpeak = $row[$o->getColumnKey('Off peak upload')]; + $to->down_peak = $row[$o->getColumnKey('Peak download')]; + $to->down_offpeak = $row[$o->getColumnKey('Off peak download')]; + // $to->peer + // $to->internal + $to->time = '24:00'; // @todo + + // If we have no records + if ($oo->count() != 1) { + Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['m'=>__METHOD__,'date'=>$date,'count'=>$oo->count()]); + + $to->service = $row[$o->getColumnKey('Login')]; + $to->save(); + + } else { + $oo->first()->traffic()->save($to); + } + + $u++; + + } catch (\Exception $e) { + Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['m'=>__METHOD__,'row'=>$row,'line'=>$line]); + throw new \Exception('Error while storing traffic date'); + } + } + + Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$this->aso->stats_lastupdate->format('Y-m-d')),['m'=>__METHOD__]); + + if ($u) { + + $this->aso->stats_lastupdate = $this->aso->stats_lastupdate->addDay(); + $this->aso->save(); + + if ($this->aso->trafficMismatch($date)->count()) + Mail::to('deon@graytech.net.au') // @todo To change + ->send(new TrafficMismatch($this->aso,$date)); } } - - if ($u) { - $this->aso->stats_lastupdate = $this->aso->stats_lastupdate->addDay(); - $this->aso->save(); - - if ($this->aso->traffic_mismatch($date)->count()) - Mail::to('deon@graytech.net.au') // @todo To change - ->send(new TrafficMismatch($this->aso,$date)); - } - - Log::info(sprintf('%s: Records Imported: %d',get_class($this),$u)); } } \ No newline at end of file diff --git a/app/Models/AdslSupplier.php b/app/Models/AdslSupplier.php index 5fca535..91e0780 100644 --- a/app/Models/AdslSupplier.php +++ b/app/Models/AdslSupplier.php @@ -29,7 +29,13 @@ class AdslSupplier extends Model /** METHODS **/ - public function traffic_mismatch(Carbon $date): Collection + /** + * Return the traffic records, that were not matched to a service. + * + * @param Carbon $date + * @return Collection + */ + public function trafficMismatch(Carbon $date): Collection { return AdslTraffic::where('date',$date->format('Y-m-d')) ->where('supplier_id',$this->id) diff --git a/app/Models/Product.php b/app/Models/Product.php index af6d3a6..0d14b0a 100644 --- a/app/Models/Product.php +++ b/app/Models/Product.php @@ -28,6 +28,8 @@ class Product extends Model protected $with = ['descriptions']; + /* RELATIONS */ + public function descriptions() { return $this->hasMany(ProductTranslate::class); @@ -48,6 +50,8 @@ class Product extends Model return $this->morphTo(null,'model','prod_plugin_data'); } + /* ATTRIBUTES */ + /** * Get the service category (from the product) * @@ -161,6 +165,17 @@ class Product extends Model return Arr::get($this->price_array,sprintf('%s.1.price_setup',$this->price_recurr_default))*1.1; } + /** + * Return if this product captures usage data + * + * @return bool + */ + public function hasUsage(): bool + { + // @todo This should be configured in the DB + return in_array($this->model, ['App\Models\Product\Adsl']); + } + public function scopeActive() { return $this->where('active',TRUE); diff --git a/app/Models/Service.php b/app/Models/Service.php index 0acec54..1efcb83 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -228,6 +228,8 @@ class Service extends Model ], ]; + /* RELATIONS */ + /** * Account the service belongs to * @@ -335,7 +337,7 @@ class Service extends Model return $this->morphTo(null,'model','id','service_id'); } - /** SCOPES **/ + /* SCOPES */ /** * Only query active categories @@ -384,7 +386,7 @@ class Service extends Model return $query->where('id','like','%'.$term.'%'); } - /** ATTRIBUTES **/ + /* ATTRIBUTES */ /** * Name of the account for this service @@ -846,7 +848,7 @@ class Service extends Model return sprintf('%s',$this->id,$this->service_id); } - /** SETTERS **/ + /* SETTERS */ public function setDateOrigAttribute($value) { @@ -858,7 +860,7 @@ class Service extends Model $this->attributes['date_last'] = $value->timestamp; } - /** FUNCTIONS **/ + /* FUNCTIONS */ // The action methods will return: NULL for no progress|FALSE for a failed status|next stage name. @@ -1140,6 +1142,16 @@ class Service extends Model }); } + /** + * Does this service have traffic data to be graphed + * + * @return bool + */ + public function hasUsage(): bool + { + return $this->product->hasUsage(); + } + /** * Determine if a service is active. It is active, if active=1, or the order_status is not in inactive_status[] * diff --git a/app/Models/Service/Adsl.php b/app/Models/Service/Adsl.php index bb48012..76d3b93 100644 --- a/app/Models/Service/Adsl.php +++ b/app/Models/Service/Adsl.php @@ -2,16 +2,19 @@ namespace App\Models\Service; -use Illuminate\Database\Eloquent\Relations\BelongsTo; -use Illuminate\Database\Eloquent\Relations\MorphOne; +use Illuminate\Support\Collection; -use App\Interfaces\ServiceItem; +use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Log; +use App\Interfaces\{ServiceItem,ServiceUsage}; +use App\Models\AdslSupplierPlan; use App\Models\Base\ServiceType; -use App\Models\Service; use App\Traits\NextKey; -class Adsl extends ServiceType implements ServiceItem +class Adsl extends ServiceType implements ServiceItem,ServiceUsage { + private const LOGKEY = 'MSA'; + use NextKey; const RECORD_ID = 'service__adsl'; @@ -30,6 +33,7 @@ class Adsl extends ServiceType implements ServiceItem */ public function traffic() { + // @todo Need to include site_id in this relation return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id'); } @@ -91,4 +95,24 @@ class Adsl extends ServiceType implements ServiceItem { return $this->service_contract_date AND $this->service_contract_date->addMonths($this->contract_term)->isFuture(); } -} \ No newline at end of file + + /** + * Return service usage data + * + * @param int $days + * @return Collection + */ + public function usage(int $days=31): Collection + { + $maxdate = self::traffic() + ->select(DB::raw('max(date) as max')) + ->pluck('max')->pop(); + + Log::debug(sprintf('%s:Getting Usage data for [%d] days from [%s]',self::LOGKEY,$days,$maxdate),['m'=>__METHOD__]); + + return $this->traffic() + ->where('date','<=',$maxdate) + ->where('date','>=',DB::raw(sprintf('date_sub(\'%s\',INTERVAL %s DAY)',$maxdate,$days))) + ->get(); + } +} diff --git a/app/Models/Service/AdslTraffic.php b/app/Models/Service/AdslTraffic.php index e3c9062..f21136e 100644 --- a/app/Models/Service/AdslTraffic.php +++ b/app/Models/Service/AdslTraffic.php @@ -8,6 +8,7 @@ class AdslTraffic extends Model { protected $table = 'ab_service__adsl_traffic'; public $timestamps = FALSE; + protected $dates = ['date']; public function broadband() { diff --git a/resources/views/theme/backend/adminlte/u/service/home.blade.php b/resources/views/theme/backend/adminlte/u/service/home.blade.php index c20bb60..2cbf723 100644 --- a/resources/views/theme/backend/adminlte/u/service/home.blade.php +++ b/resources/views/theme/backend/adminlte/u/service/home.blade.php @@ -25,7 +25,6 @@
-
    {{--
  • Product
    • @@ -34,6 +33,10 @@ @if (! $o->suspend_billing AND ! $o->external_billing)
  • Pending Items
    • @endif + @if ($o->hasUsage()) +
  • Traffic
    • + @endif + {{--
  • Invoices
  • Emails
    • @@ -62,9 +65,6 @@
    -
    - Traffic. -
    Product.
    @@ -73,6 +73,11 @@ @include('common.service.widget.invoice')
    @endif + @if ($o->hasUsage()) +
    + @include('u.service.widgets.'.$o->stype.'.usagegraph',['o'=>$o->type]) +
    + @endif
    Invoices.
    diff --git a/resources/views/theme/backend/adminlte/u/service/widgets/broadband/usagegraph.blade.php b/resources/views/theme/backend/adminlte/u/service/widgets/broadband/usagegraph.blade.php new file mode 100644 index 0000000..b102ba5 --- /dev/null +++ b/resources/views/theme/backend/adminlte/u/service/widgets/broadband/usagegraph.blade.php @@ -0,0 +1,62 @@ +
    +
    +

    Broadband Traffic

    +
    + +
    +
    +
    +
    + +@section('page-scripts') + @js('//code.highcharts.com/highcharts.js','highcharts') + @js('//code.highcharts.com/highcharts-more.js','highcharts-more','highcharts') + @js('//code.highcharts.com/modules/xrange.js','highcharts-xrange','highcharts') + @js('//code.highcharts.com/modules/exporting.js','highcharts-export','highcharts') + @js('//code.highcharts.com/modules/offline-exporting.js','highcharts-export-offline','highcharts-export') + +@append \ No newline at end of file