Added usage graph (ADSL), improved logging for usage collection (ADSL)

app/Classes/External/Supplier.php

@@ -11,6 +11,8 @@ use Illuminate\Support\Facades\Log;
 
 abstract class Supplier
 {
+	private const LOGKEY = 'AS-';
+
 	protected $o = NULL;
 	protected $_columns = [];
 
@@ -23,16 +25,16 @@ abstract class Supplier
 	/**
 	 * Connect and pull down traffic data
 	 *
-	 * @param array $args
+	 * @return Collection
-	 * @return mixed
 	 */
-	public function connect(array $args=[])
+	public function fetch(): Collection
 	{
 		if ($x=$this->mustPause()) {
-			Log::error('API Throttle, waiting .',['m'=>__METHOD__]);
+			Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]);
 			sleep($x);
 		}
 
+		Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,$this->o->stats_lastupdate),['m'=>__METHOD__]);
 		$result = Cache::remember('Supplier:'.$this->o->id.$this->o->stats_lastupdate,86400,function() {
 			$client = $this->getClient();
 
@@ -53,14 +55,22 @@ abstract class Supplier
 			$api_reset = Arr::get($result->getHeader('X-RateLimit-Reset'),0);
 
 			if ($api_remain === 0 AND $api_reset) {
-				Log::error('API Throttle.',['m'=>__METHOD__]);
+				Log::notice(sprintf('%s:API Throttle [%d].',self::LOGKEY,$api_reset),['m'=>__METHOD__]);
 				Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset));
 			}
 
-			return $result->getBody()->getContents();
+			// Assume the supplier provides an ASCII output for text/html
+			if (preg_match('#^text/html;#',$x=Arr::get($result->getHeader('Content-Type'),'0'))) {
+				return collect(explode("\n",$result->getBody()->getContents()))->filter();
 
+			} else {
+				Log::error(sprintf('%s:Havent handled header type [%s]',self::LOGKEY,$x),['m'=>__METHOD__]);
+				throw new \Exception('Unhandled Content Type');
+			}
 		});
 
+		Log::debug(sprintf('%s:Supplier [%d], records returned [%d]...',self::LOGKEY,$this->o->id,$result->count()),['m'=>__METHOD__]);
+
 		return $result;
 	}
 

app/Console/Commands/BroadbandTraffic.php

@@ -21,17 +21,7 @@ class BroadbandTraffic extends Command
 	 *
 	 * @var string
 	 */
-	protected $description = 'Input Broadband Traffic from Suppliers';
+	protected $description = 'Import Broadband Traffic from Suppliers';
-
-	/**
-	 * Create a new command instance.
-	 *
-	 * @return void
-	 */
-	public function __construct()
-	{
-		parent::__construct();
-	}
 
 	/**
 	 * Execute the console command.
@@ -40,8 +30,7 @@ class BroadbandTraffic extends Command
 	 */
 	public function handle()
 	{
-		foreach (AdslSupplier::active()->get() as $o) {
+		foreach (AdslSupplier::active()->get() as $o)
-			Job::dispatchNow($o);
+			Job::dispatch($o);
-		}
 	}
 }

app/Interfaces/ServiceUsage.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace App\Interfaces;
+
+use Illuminate\Support\Collection;
+
+interface ServiceUsage
+{
+	/**
+	 * This service provides usage information
+	 *
+	 * @param int $days
+	 * @return Collection
+	 */
+	public function usage(int $days): Collection;
+}

app/Jobs/BroadbandTraffic.php

@@ -28,7 +28,9 @@ class BroadbandTraffic implements ShouldQueue
 {
 	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
-	protected $aso = NULL;
+	private const LOGKEY = 'JBT';
+
+	protected $aso = NULL;			// The supplier we are updating from
 	private $class_prefix = 'App\Classes\External\Supplier\\';
 
 	public function __construct(AdslSupplier $o)
@@ -40,13 +42,17 @@ class BroadbandTraffic implements ShouldQueue
 	 * Execute the job.
 	 *
 	 * @return void
+	 * @throws \Exception
+	 * @todo The column stats_lastupdate is actually the "next" date that stats should be retrieved. Rename it.
 	 */
 	public function handle()
 	{
+		Log::info(sprintf('%s:Importing Broadband Traffic from [%s]',self::LOGKEY,$this->aso->name),['m'=>__METHOD__]);
+
 		$u = 0;
 
+		// Load our class for this supplier
 		$class = $this->class_prefix.$this->aso->name;
-
 		if (class_exists($class)) {
 			$o = new $class($this->aso);
 
@@ -55,16 +61,17 @@ class BroadbandTraffic implements ShouldQueue
 			exit(1);
 		}
 
+		// Repeat pull traffic data until yesterday
+		while ($this->aso->stats_lastupdate < Carbon::now()->subDay()) {
+			Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$this->aso->stats_lastupdate->format('Y-m-d')),['m'=>__METHOD__]);
+
+			// Delete traffic, since we'll refresh it.
 			AdslTraffic::where('supplier_id',$this->aso->id)
 				->where('date',$this->aso->stats_lastupdate)
 				->delete();
 
-		// @todo Need to trap errors from getting data
 			$c = 0;
-		foreach (explode("\n",$o->connect()) as $line) {
+			foreach ($o->fetch() as $line) {
-			if (! trim($line))
-				continue;
-
 				// The first row is our header
 				if (! $c++) {
 					$fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header()));
@@ -77,8 +84,10 @@ class BroadbandTraffic implements ShouldQueue
 				$row = str_getcsv(trim($line));
 
 				try {
+					// @todo Put the date format in the DB.
 					$date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]);
 
+					// Find the right service dependant on the dates we supplied the service
 					$oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')])
 						->select(DB::raw('ab_service__adsl.*'))
 						->join('ab_service','ab_service.id','=','service_id')
@@ -89,54 +98,48 @@ class BroadbandTraffic implements ShouldQueue
 						})
 						->get();
 
+					$to = new AdslTraffic;
+					$to->site_id = 1; // @todo TO ADDRESS
+					$to->date = $this->aso->stats_lastupdate;
+					$to->supplier_id = $this->aso->id;
+					$to->up_peak = $row[$o->getColumnKey('Peak upload')];
+					$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
+					$to->down_peak = $row[$o->getColumnKey('Peak download')];
+					$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
+					// $to->peer
+					// $to->internal
+					$to->time = '24:00'; // @todo
+
 					// If we have no records
 					if ($oo->count() != 1) {
-					Log::error(sprintf('! Records Errors for:%s (%s) [%s]',$row[$o->getColumnKey('Login')],$date,$oo->count()));
+						Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['m'=>__METHOD__,'date'=>$date,'count'=>$oo->count()]);
 
-					$to = new AdslTraffic;
-					$to->site_id = 1; // @todo TO ADDRESS
 						$to->service = $row[$o->getColumnKey('Login')];
-					$to->date = $this->aso->stats_lastupdate;
-					$to->supplier_id = $this->aso->id;
-					$to->up_peak = $row[$o->getColumnKey('Peak upload')];
-					$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
-					$to->down_peak = $row[$o->getColumnKey('Peak download')];
-					$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
-					// $to->peer
-					// $to->internal
-					$to->time = '24:00'; // @todo
 						$to->save();
-					$u++;
 
 					} else {
-					$to = new AdslTraffic;
-					$to->site_id = 1; // @todo TO ADDRESS
-					$to->date = $this->aso->stats_lastupdate;
-					$to->supplier_id = $this->aso->id;
-					$to->up_peak = $row[$o->getColumnKey('Peak upload')];
-					$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
-					$to->down_peak = $row[$o->getColumnKey('Peak download')];
-					$to->down_offpeak = $row[$o->getColumnKey('Off peak download')];
-					// $to->peer
-					// $to->internal
-					$to->time = '24:00'; // @todo
 						$oo->first()->traffic()->save($to);
-					$u++;
 					}
+
+					$u++;
+
 				} catch (\Exception $e) {
-				dd(['row'=>$row,'line'=>$line]);
+					Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['m'=>__METHOD__,'row'=>$row,'line'=>$line]);
+					throw new \Exception('Error while storing traffic date');
 				}
 			}
 
+			Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$this->aso->stats_lastupdate->format('Y-m-d')),['m'=>__METHOD__]);
+
 			if ($u) {
+
 				$this->aso->stats_lastupdate = $this->aso->stats_lastupdate->addDay();
 				$this->aso->save();
 
-			if ($this->aso->traffic_mismatch($date)->count())
+				if ($this->aso->trafficMismatch($date)->count())
 					Mail::to('deon@graytech.net.au')        // @todo To change
 						->send(new TrafficMismatch($this->aso,$date));
 			}
-
+		}
-		Log::info(sprintf('%s: Records Imported: %d',get_class($this),$u));
 	}
 }

app/Models/AdslSupplier.php

@@ -29,7 +29,13 @@ class AdslSupplier extends Model
 
 	/** METHODS **/
 
-	public function traffic_mismatch(Carbon $date): Collection
+	/**
+	 * Return the traffic records, that were not matched to a service.
+	 *
+	 * @param Carbon $date
+	 * @return Collection
+	 */
+	public function trafficMismatch(Carbon $date): Collection
 	{
 		return AdslTraffic::where('date',$date->format('Y-m-d'))
 			->where('supplier_id',$this->id)

app/Models/Product.php

@@ -28,6 +28,8 @@ class Product extends Model
 
 	protected $with = ['descriptions'];
 
+	/* RELATIONS */
+
 	public function descriptions()
 	{
 		return $this->hasMany(ProductTranslate::class);
@@ -48,6 +50,8 @@ class Product extends Model
 		return $this->morphTo(null,'model','prod_plugin_data');
 	}
 
+	/* ATTRIBUTES */
+
 	/**
 	 * Get the service category (from the product)
 	 *
@@ -161,6 +165,17 @@ class Product extends Model
 		return Arr::get($this->price_array,sprintf('%s.1.price_setup',$this->price_recurr_default))*1.1;
 	}
 
+	/**
+	 * Return if this product captures usage data
+	 *
+	 * @return bool
+	 */
+	public function hasUsage(): bool
+	{
+		// @todo This should be configured in the DB
+		return in_array($this->model, ['App\Models\Product\Adsl']);
+	}
+
 	public function scopeActive()
 	{
 		return $this->where('active',TRUE);

app/Models/Service.php

@@ -228,6 +228,8 @@ class Service extends Model
 		],
 	];
 
+	/* RELATIONS */
+
 	/**
 	 * Account the service belongs to
 	 *
@@ -335,7 +337,7 @@ class Service extends Model
 		return $this->morphTo(null,'model','id','service_id');
 	}
 
-	/** SCOPES **/
+	/* SCOPES */
 
 	/**
 	 * Only query active categories
@@ -384,7 +386,7 @@ class Service extends Model
 		return $query->where('id','like','%'.$term.'%');
 	}
 
-	/** ATTRIBUTES **/
+	/* ATTRIBUTES */
 
 	/**
 	 * Name of the account for this service
@@ -846,7 +848,7 @@ class Service extends Model
 		return sprintf('<a href="/u/service/%s">%s</a>',$this->id,$this->service_id);
 	}
 
-	/** SETTERS **/
+	/* SETTERS */
 
 	public function setDateOrigAttribute($value)
 	{
@@ -858,7 +860,7 @@ class Service extends Model
 		$this->attributes['date_last'] = $value->timestamp;
 	}
 
-	/** FUNCTIONS **/
+	/* FUNCTIONS */
 
 	// The action methods will return: NULL for no progress|FALSE for a failed status|next stage name.
 
@@ -1140,6 +1142,16 @@ class Service extends Model
 		});
 	}
 
+	/**
+	 * Does this service have traffic data to be graphed
+	 *
+	 * @return bool
+	 */
+	public function hasUsage(): bool
+	{
+		return $this->product->hasUsage();
+	}
+
 	/**
 	 * Determine if a service is active. It is active, if active=1, or the order_status is not in inactive_status[]
 	 *

app/Models/Service/Adsl.php

@@ -2,16 +2,19 @@
 
 namespace App\Models\Service;
 
-use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Support\Collection;
-use Illuminate\Database\Eloquent\Relations\MorphOne;
 
-use App\Interfaces\ServiceItem;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+use App\Interfaces\{ServiceItem,ServiceUsage};
+use App\Models\AdslSupplierPlan;
 use App\Models\Base\ServiceType;
-use App\Models\Service;
 use App\Traits\NextKey;
 
-class Adsl extends ServiceType implements ServiceItem
+class Adsl extends ServiceType implements ServiceItem,ServiceUsage
 {
+	private const LOGKEY = 'MSA';
+
 	use NextKey;
 	const RECORD_ID = 'service__adsl';
 
@@ -30,6 +33,7 @@ class Adsl extends ServiceType implements ServiceItem
 	 */
 	public function traffic()
 	{
+		// @todo Need to include site_id in this relation
 		return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id');
 	}
 
@@ -91,4 +95,24 @@ class Adsl extends ServiceType implements ServiceItem
 	{
 		return $this->service_contract_date AND $this->service_contract_date->addMonths($this->contract_term)->isFuture();
 	}
+
+	/**
+	 * Return service usage data
+	 *
+	 * @param int $days
+	 * @return Collection
+	 */
+	public function usage(int $days=31): Collection
+	{
+		$maxdate = self::traffic()
+			->select(DB::raw('max(date) as max'))
+			->pluck('max')->pop();
+
+		Log::debug(sprintf('%s:Getting Usage data for [%d] days from [%s]',self::LOGKEY,$days,$maxdate),['m'=>__METHOD__]);
+
+		return $this->traffic()
+			->where('date','<=',$maxdate)
+			->where('date','>=',DB::raw(sprintf('date_sub(\'%s\',INTERVAL %s DAY)',$maxdate,$days)))
+			->get();
+	}
 }

app/Models/Service/AdslTraffic.php

@@ -8,6 +8,7 @@ class AdslTraffic extends Model
 {
 	protected $table = 'ab_service__adsl_traffic';
 	public $timestamps = FALSE;
+	protected $dates = ['date'];
 
 	public function broadband()
 	{

resources/views/theme/backend/adminlte/u/service/home.blade.php

@@ -25,7 +25,6 @@
 		<div class="col-7">
 			<div class="card">
 				<div class="card-header bg-dark d-flex p-0">
-					<span class="p-3"><i class="fa fa-bars"></i></span>
 					<ul class="nav nav-pills p-2 w-100">
 						{{--
 						<li class="nav-item"><a class="nav-link active" href="#product" data-toggle="tab">Product</a></li>
@@ -34,6 +33,10 @@
 						@if (! $o->suspend_billing AND ! $o->external_billing)
 							<li class="nav-item active"><a class="nav-link" href="#pending_items" data-toggle="tab">Pending Items</a></li>
 						@endif
+						@if ($o->hasUsage())
+							<li class="nav-item active"><a class="nav-link" href="#traffic" data-toggle="tab">Traffic</a></li>
+						@endif
+
 						{{--
 						<li class="nav-item"><a class="nav-link" href="#invoices" data-toggle="tab">Invoices</a></li>
 						<li class="nav-item"><a class="nav-link" href="#emails" data-toggle="tab">Emails</a></li>
@@ -62,9 +65,6 @@
 
 				<div class="card-body">
 					<div class="tab-content">
-						<div class="tab-pane fade" id="traffic" role="tabpanel">
-							Traffic.
-						</div>
 						<div class="tab-pane fade" id="product" role="tabpanel">
 							Product.
 						</div>
@@ -73,6 +73,11 @@
 								@include('common.service.widget.invoice')
 							</div>
 						@endif
+						@if ($o->hasUsage())
+							<div class="tab-pane fade show" id="traffic" role="tabpanel">
+								@include('u.service.widgets.'.$o->stype.'.usagegraph',['o'=>$o->type])
+							</div>
+						@endif
 						<div class="tab-pane fade" id="invoices" role="tabpanel">
 							Invoices.
 						</div>

resources/views/theme/backend/adminlte/u/service/widgets/broadband/usagegraph.blade.php

@@ -0,0 +1,62 @@
+<div class="card">
+	<div class="card-header bg-gray-dark">
+		<h3 class="card-title">Broadband Traffic</h3>
+	</div>
+
+	<div class="card-body">
+		<div id="graph"></div>
+	</div>
+</div>
+
+@section('page-scripts')
+	@js('//code.highcharts.com/highcharts.js','highcharts')
+	@js('//code.highcharts.com/highcharts-more.js','highcharts-more','highcharts')
+	@js('//code.highcharts.com/modules/xrange.js','highcharts-xrange','highcharts')
+	@js('//code.highcharts.com/modules/exporting.js','highcharts-export','highcharts')
+	@js('//code.highcharts.com/modules/offline-exporting.js','highcharts-export-offline','highcharts-export')
+	<script>
+		Highcharts.chart('graph', {
+			chart: {
+				type: 'areaspline'
+			},
+			title: {
+				text: 'Usage Traffic up to {{ $o->usage(30)->max('date')->format('Y-m-d') }}'
+			},
+			legend: {
+				layout: 'vertical',
+				align: 'left',
+				verticalAlign: 'top',
+				x: 150,
+				y: 100,
+				floating: true,
+				borderWidth: 1,
+				backgroundColor:
+					Highcharts.defaultOptions.legend.backgroundColor || '#FFFFFF'
+			},
+			xAxis: {
+				type: 'datetime'
+			},
+			yAxis: {
+				title: {
+					text: 'MB'
+				}
+			},
+			tooltip: {
+				shared: true,
+				valueSuffix: ' MB'
+			},
+			credits: {
+				enabled: false
+			},
+			plotOptions: {
+				areaspline: {
+					fillOpacity: 0.5
+				}
+			},
+			series: [{
+				name: 'Traffic',
+				data: {!! $o->usage(30)->map(function($item) { return ['x'=>$item->date->timestamp*1000,'y'=>$item->up_peak+$item->down_peak+$item->up_offpeak+$item->down_offpeak];}) !!}
+			}]
+		});
+	</script>
+@append