Update and fix broadband traffic

This commit is contained in:
Deon George 2022-04-20 16:24:58 +10:00
parent 621a132e35
commit 16b7e0b493
13 changed files with 161 additions and 112 deletions

View File

@ -17,6 +17,8 @@ abstract class Supplier
protected $o = NULL; protected $o = NULL;
protected $_columns = []; protected $_columns = [];
public const traffic_connection_keys = ['user','pass','url'];
public function __construct(Model $o) public function __construct(Model $o)
{ {
$this->o = $o; $this->o = $o;
@ -26,24 +28,29 @@ abstract class Supplier
/** /**
* Connect and pull down traffic data * Connect and pull down traffic data
* *
* @param array $connection
* @param string $type
* @return Collection * @return Collection
* @throws \Exception
*/ */
public function fetch(): Collection public function fetch(array $connection,string $type): Collection
{ {
if (count(array_intersect(array_keys($connection),self::traffic_connection_keys)) !== 3)
throw new \Exception('No or missing connection details for:'.$type);
if ($x=$this->mustPause()) { if ($x=$this->mustPause()) {
Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]); Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]);
sleep($x); sleep($x);
} }
Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,$this->o->stats_lastupdate),['m'=>__METHOD__]); Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,Arr::get($connection,'last')),['m'=>__METHOD__]);
$key = 'Supplier:'.$this->o->id.$this->o->stats_lastupdate; $key = 'Supplier:'.$this->o->id.Arr::get($connection,'last');
$result = Cache::remember($key,86400,function() {
$client = $this->getClient();
$response = Http::get($this->o->stats_url,[ $result = Cache::remember($key,86400,function() use ($connection) {
$this->login_user_field => $this->o->stats_username, $response = Http::get(Arr::get($connection,'url'),[
$this->login_pass_field => $this->o->stats_password, $this->login_user_field => Arr::get($connection,'user'),
$this->date_field => $this->o->stats_lastupdate->format('Y-m-d'), $this->login_pass_field => Arr::get($connection,'pass'),
$this->date_field => Arr::get($connection,'last'),
]); ]);
// @todo These API rate limiting is untested. // @todo These API rate limiting is untested.
@ -55,7 +62,6 @@ abstract class Supplier
Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset)); Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset));
} }
//dd($response->header('Content-Type'),$response->headers());
// Assume the supplier provides an ASCII output for text/html // Assume the supplier provides an ASCII output for text/html
if (preg_match('#^text/html;#',$x=$response->header('Content-Type'))) { if (preg_match('#^text/html;#',$x=$response->header('Content-Type'))) {
return collect(explode("\n",$response->body()))->filter(); return collect(explode("\n",$response->body()))->filter();

View File

@ -5,7 +5,7 @@ namespace App\Console\Commands;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use App\Jobs\BroadbandTraffic as Job; use App\Jobs\BroadbandTraffic as Job;
use App\Models\AdslSupplier; use App\Models\Supplier;
class BroadbandTraffic extends Command class BroadbandTraffic extends Command
{ {
@ -14,7 +14,8 @@ class BroadbandTraffic extends Command
* *
* @var string * @var string
*/ */
protected $signature = 'broadband:traffic:import'; protected $signature = 'broadband:traffic:import'.
' {--s|supplier= : Supplier Name}';
/** /**
* The console command description. * The console command description.
@ -30,7 +31,14 @@ class BroadbandTraffic extends Command
*/ */
public function handle() public function handle()
{ {
foreach (AdslSupplier::active()->get() as $o) if ($this->option('supplier')) {
$o = Supplier::where('name','like',$this->option('supplier'))->singleOrFail();
Job::dispatch($o);
return;
}
foreach (Supplier::active()->get() as $o)
Job::dispatch($o); Job::dispatch($o);
} }
} }

View File

@ -5,8 +5,8 @@ namespace App\Console;
use Illuminate\Console\Scheduling\Schedule; use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel; use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use App\Models\AdslSupplier;
use App\Jobs\BroadbandTraffic; use App\Jobs\BroadbandTraffic;
use App\Models\Supplier;
class Kernel extends ConsoleKernel class Kernel extends ConsoleKernel
{ {
@ -29,7 +29,7 @@ class Kernel extends ConsoleKernel
{ {
// @todo This needs to be more generic and dynamic // @todo This needs to be more generic and dynamic
// Exetel Traffic // Exetel Traffic
$schedule->job(new BroadbandTraffic(AdslSupplier::find(1)))->timezone('Australia/Melbourne')->dailyAt('10:00'); $schedule->job(new BroadbandTraffic(Supplier::find(1)))->timezone('Australia/Melbourne')->dailyAt('10:00');
} }
/** /**

View File

@ -6,6 +6,13 @@ use Illuminate\Support\Collection;
interface ServiceUsage interface ServiceUsage
{ {
/**
* Our model that holds traffic information
*
* @return mixed
*/
public function traffic();
/** /**
* This service provides usage information * This service provides usage information
* *

View File

@ -9,14 +9,16 @@ use Illuminate\Database\Eloquent\Model;
use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels; use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Mail; use Illuminate\Support\Facades\Mail;
use App\Classes\External\Supplier as ExternalSupplier;
use App\Mail\TrafficMismatch; use App\Mail\TrafficMismatch;
use App\Models\Service\Adsl; use App\Models\Supplier;
use App\Models\Service\AdslTraffic; use App\Models\Service\Broadband as ServiceBroadband;
use App\Models\AdslSupplier; use App\Models\Usage\Broadband as UsageBroadband;
/** /**
* Class BroadbandTraffic * Class BroadbandTraffic
@ -33,7 +35,9 @@ final class BroadbandTraffic implements ShouldQueue
protected Model $o; // The supplier we are updating from protected Model $o; // The supplier we are updating from
private const class_prefix = 'App\Classes\External\Supplier\\'; private const class_prefix = 'App\Classes\External\Supplier\\';
public function __construct(AdslSupplier $o) private const traffic = 'broadband';
public function __construct(Supplier $o)
{ {
$this->o = $o; $this->o = $o;
} }
@ -43,12 +47,14 @@ final class BroadbandTraffic implements ShouldQueue
* *
* @return void * @return void
* @throws \Exception * @throws \Exception
* @todo The column stats_lastupdate is actually the "next" date that stats should be retrieved. Rename it.
*/ */
public function handle() public function handle()
{ {
Log::info(sprintf('%s:Importing Broadband Traffic from [%s]',self::LOGKEY,$this->o->name)); Log::info(sprintf('%s:Importing Broadband Traffic from [%s]',self::LOGKEY,$this->o->name));
if ((! $connection=$this->o->detail->connections->get('broadband')) || (count(array_intersect(array_keys($connection),ExternalSupplier::traffic_connection_keys)) !== 3))
throw new \Exception('No or missing connection details for:'.self::traffic);
$u = 0; $u = 0;
// Load our class for this supplier // Load our class for this supplier
@ -57,21 +63,24 @@ final class BroadbandTraffic implements ShouldQueue
$o = new $class($this->o); $o = new $class($this->o);
} else { } else {
Log::error(sprintf('%s: Class doesnt exist: %d',get_class($this),$class)); Log::error(sprintf('%s: Class doesnt exist: %s',self::LOGKEY,$class));
exit(1); exit(1);
} }
$last_update = Carbon::create(Arr::get($connection,'last'))->addDay();
Arr::set($connection,'last',$last_update->format('Y-m-d'));
// Repeat pull traffic data until yesterday // Repeat pull traffic data until yesterday
while ($this->o->stats_lastupdate < Carbon::now()->subDay()) { while ($last_update < Carbon::now()->subDay()) {
Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$this->o->stats_lastupdate->format('Y-m-d'))); Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$last_update->format('Y-m-d')));
// Delete traffic, since we'll refresh it. // Delete traffic, since we'll refresh it.
AdslTraffic::where('supplier_id',$this->o->id) UsageBroadband::where('supplier_id',$this->o->id)
->where('date',$this->o->stats_lastupdate) ->where('date',$last_update->format('Y-m-d'))
->delete(); ->delete();
$c = 0; $c = 0;
foreach ($o->fetch() as $line) { foreach ($o->fetch($connection,self::traffic) as $line) {
// The first row is our header // The first row is our header
if (! $c++) { if (! $c++) {
$fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header())); $fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header()));
@ -79,28 +88,26 @@ final class BroadbandTraffic implements ShouldQueue
} }
if (! $fields->count()) if (! $fields->count())
abort(500,'? No fields in data exportupda'); abort(500,'? No fields in data export');
$row = str_getcsv(trim($line)); $row = str_getcsv(trim($line));
try { try {
// @todo Put the date format in the DB.
$date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]); $date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]);
// Find the right service dependent on the dates we supplied the service
// Find the right service dependant on the dates we supplied the service $oo = ServiceBroadband::where('service_username',$row[$o->getColumnKey('Login')])
$oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')]) ->select(DB::raw('service_broadband.*'))
->select(DB::raw('ab_service__adsl.*'))
->join('services','services.id','=','service_id') ->join('services','services.id','=','service_id')
->where('services.start_at','<=',$date) ->where('services.start_at','<=',$date)
->where(function($query) use ($date) { ->where(function($query) use ($date) {
$query->whereNULL('services.stop_at') $query->whereNULL('services.stop_at')
->orWhere('services.stop_at','<=',$date); ->orWhere('services.stop_at','<=',$date);
}) })
->get(); ->single();
$to = new AdslTraffic; $to = new UsageBroadband;
$to->site_id = 1; // @todo TO ADDRESS $to->site_id = $oo->site_id;
$to->date = $this->o->stats_lastupdate; $to->date = $last_update;
$to->supplier_id = $this->o->id; $to->supplier_id = $this->o->id;
$to->up_peak = $row[$o->getColumnKey('Peak upload')]; $to->up_peak = $row[$o->getColumnKey('Peak upload')];
$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')]; $to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
@ -111,30 +118,35 @@ final class BroadbandTraffic implements ShouldQueue
$to->time = '24:00'; // @todo $to->time = '24:00'; // @todo
// If we have no records // If we have no records
if ($oo->count() != 1) { if (! $oo) {
Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['date'=>$date,'count'=>$oo->count()]); Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['date'=>$date,'count'=>$oo->count()]);
$to->service = $row[$o->getColumnKey('Login')]; $to->service = $row[$o->getColumnKey('Login')];
$to->save();
} else { } else {
$oo->first()->traffic()->save($to); $to->service_item_id = $oo->id;
} }
if ($to->save())
$u++; $u++;
} catch (\Exception $e) { } catch (\Exception $e) {
dump($to);
Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['row'=>$row,'line'=>$line]); Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['row'=>$row,'line'=>$line]);
throw new \Exception('Error while storing traffic date'); throw new \Exception('Error while storing traffic date');
} }
} }
Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$last_update->format('Y-m-d')));
Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$this->o->stats_lastupdate->format('Y-m-d'))); // Save our current progress.
$this->o->detail->connections = $this->o->detail->connections->put(self::traffic,array_merge($connection,['last'=>$last_update->format('Y-m-d')]));
$this->o->detail->save();
// Update our details for the next iteration.
$last_update = $last_update->addDay();
Arr::set($connection,'last',$last_update->format('Y-m-d'));
if ($u) { if ($u) {
$this->o->stats_lastupdate = $this->o->stats_lastupdate->addDay();
$this->o->save();
if ($this->o->trafficMismatch($date)->count()) if ($this->o->trafficMismatch($date)->count())
Mail::to('deon@graytech.net.au') // @todo To change Mail::to('deon@graytech.net.au') // @todo To change
->send(new TrafficMismatch($this->o,$date)); ->send(new TrafficMismatch($this->o,$date));

View File

@ -8,13 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Mail\Mailable; use Illuminate\Mail\Mailable;
use Illuminate\Queue\SerializesModels; use Illuminate\Queue\SerializesModels;
use App\Models\{AdslSupplier,Site}; use App\Models\{Supplier,Site};
class TrafficMismatch extends Mailable class TrafficMismatch extends Mailable
{ {
use Queueable, SerializesModels; use Queueable, SerializesModels;
public AdslSupplier $aso; public Supplier $aso;
public Carbon $date; public Carbon $date;
/** /**
@ -22,7 +22,7 @@ class TrafficMismatch extends Mailable
* *
* @return void * @return void
*/ */
public function __construct(AdslSupplier $o,Carbon $date) public function __construct(Supplier $o,Carbon $date)
{ {
$this->aso = $o; $this->aso = $o;
$this->date = $date; $this->date = $date;

View File

@ -1,48 +0,0 @@
<?php
namespace App\Models;
use App\Models\Service\AdslTraffic;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
/**
* @deprecated
*/
class AdslSupplier extends Model
{
protected $table = 'ab_adsl_supplier';
protected $dates = [
'stats_lastupdate',
];
public $timestamps = FALSE;
/** SCOPES */
/**
* Only query active categories
*/
public function scopeActive($query)
{
return $query->where('active',TRUE);
}
/** METHODS **/
/**
* Return the traffic records, that were not matched to a service.
*
* @param Carbon $date
* @return Collection
*/
public function trafficMismatch(Carbon $date): Collection
{
return AdslTraffic::where('date',$date->format('Y-m-d'))
->where('supplier_id',$this->id)
->whereNULL('ab_service_adsl_id')
->get();
}
}

View File

@ -22,7 +22,7 @@ final class Broadband extends Type implements ProductItem
'options.address'=>[ 'options.address'=>[
'request'=>'options.address', 'request'=>'options.address',
'key'=>'service_address', 'key'=>'service_address',
'validation'=>'required|string:10|unique:ab_service__adsl,service_address', 'validation'=>'required|string:10|unique:service_broadband,service_address',
'validation_message'=>'Address is a required field.', 'validation_message'=>'Address is a required field.',
], ],
'options.notes'=>[ 'options.notes'=>[

View File

@ -10,6 +10,7 @@ use App\Interfaces\ServiceUsage;
use App\Models\Base\ServiceType; use App\Models\Base\ServiceType;
use App\Models\Supplier\Broadband as SupplierBroadband; use App\Models\Supplier\Broadband as SupplierBroadband;
use App\Models\Supplier\Type; use App\Models\Supplier\Type;
use App\Models\Usage\Broadband as UsageBroadband;
/** /**
* Class Broadband (Service) * Class Broadband (Service)
@ -65,17 +66,15 @@ class Broadband extends ServiceType implements ServiceUsage
return $this->service_number ?: ($this->service_address ?: '-'); return $this->service_number ?: ($this->service_address ?: '-');
} }
/* RELATIONS */
/** /**
* The accounts that this user manages * The usage information for broadband
* *
* @return \Illuminate\Database\Eloquent\Relations\HasMany * @return \Illuminate\Database\Eloquent\Relations\HasMany
*/ */
public function traffic() public function traffic()
{ {
// @todo Need to include site_id in this relation return $this->hasMany(UsageBroadband::class,'service_item_id')
return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id'); ->where('site_id',$this->site_id);
} }
/* ATTRIBUTES */ /* ATTRIBUTES */
@ -128,8 +127,6 @@ class Broadband extends ServiceType implements ServiceUsage
if (! $maxdate) if (! $maxdate)
return collect(); return collect();
Log::debug(sprintf('%s:Getting Usage data for [%d] days from [%s]',self::LOGKEY,$days,$maxdate->date->format('Y-m-d')),['m'=>__METHOD__]);
return $this->traffic() return $this->traffic()
->where('date','<=',$maxdate->date->format('Y-m-d')) ->where('date','<=',$maxdate->date->format('Y-m-d'))
->where('date','>=',$maxdate->date->subDays($days)->format('Y-m-d')) ->where('date','>=',$maxdate->date->subDays($days)->format('Y-m-d'))
@ -139,9 +136,9 @@ class Broadband extends ServiceType implements ServiceUsage
/** /**
* Find the last date any traffic was recorded for a service * Find the last date any traffic was recorded for a service
* *
* @return AdslTraffic|null * @return UsageBroadband|null
*/ */
private function usage_last_date(): ?AdslTraffic private function usage_last_date(): ?UsageBroadband
{ {
return $this->traffic return $this->traffic
->sortBy('date') ->sortBy('date')

View File

@ -2,6 +2,7 @@
namespace App\Models; namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;
use Illuminate\Support\Collection; use Illuminate\Support\Collection;
@ -62,7 +63,8 @@ class Supplier extends Model
// EG: Crazy Domains, "domains" and "hosting". // EG: Crazy Domains, "domains" and "hosting".
public function detail() public function detail()
{ {
return $this->hasOne(SupplierDetail::class); return $this->hasOne(SupplierDetail::class)
->withoutGlobalScope(\App\Models\Scopes\SiteScope::class);
} }
/* METHODS */ /* METHODS */
@ -103,4 +105,18 @@ class Supplier extends Model
return $result; return $result;
} }
/**
* Return the traffic records, that were not matched to a service.
*
* @param Carbon $date
* @return \Illuminate\Database\Eloquent\Collection
*/
public function trafficMismatch(Carbon $date): Collection
{
return Usage\Broadband::where('date',$date->format('Y-m-d'))
->where('supplier_id',$this->id)
->whereNULL('service_item_id')
->get();
}
} }

View File

@ -1,23 +1,29 @@
<?php <?php
namespace App\Models\Service; namespace App\Models\Usage;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
use Leenooks\Carbon; use Leenooks\Carbon;
class AdslTraffic extends Model use App\Models\Service\Broadband as ServiceBroadband;
class Broadband extends Model
{ {
protected $table = 'ab_service__adsl_traffic';
public $timestamps = FALSE;
protected $dates = ['date']; protected $dates = ['date'];
public $dateFormat = 'U'; protected $table = 'usage_broadband';
public $timestamps = FALSE;
private $traffic_end = 14; private $traffic_end = 14;
/* RELATIONS */
public function broadband() public function broadband()
{ {
return $this->belongsTo(Adsl::class); return $this->belongsTo(ServiceBroadband::class);
} }
/* ATTRIBUTES */
public function getTotalAttribute() { public function getTotalAttribute() {
return $this->up_peak+$this->down_peak+$this->up_offpeak+$this->down_offpeak; return $this->up_peak+$this->down_peak+$this->up_offpeak+$this->down_offpeak;
} }

View File

@ -266,6 +266,8 @@ class RenameServiceTables extends Migration
DB::statement('ALTER TABLE service_broadband MODIFY service_id int unsigned NOT NULL'); DB::statement('ALTER TABLE service_broadband MODIFY service_id int unsigned NOT NULL');
DB::statement('ALTER TABLE service_broadband MODIFY service_stats_collect tinyint(1) DEFAULT NULL'); DB::statement('ALTER TABLE service_broadband MODIFY service_stats_collect tinyint(1) DEFAULT NULL');
DB::statement('ALTER TABLE service_broadband RENAME COLUMN service_stats_lastupdate TO service_stats_at'); DB::statement('ALTER TABLE service_broadband RENAME COLUMN service_stats_lastupdate TO service_stats_at');
DB::statement('ALTER TABLE service_broadband RENAME COLUMN provided_adsl_plan_id TO provided_supplier_broadband_id');
DB::statement('ALTER TABLE service_broadband MODIFY provided_supplier_broadband_id int unsigned DEFAULT NULL');
// @todo drop column provided_adsl_plan_id // @todo drop column provided_adsl_plan_id
Schema::table('service_broadband', function (Blueprint $table) { Schema::table('service_broadband', function (Blueprint $table) {
@ -281,6 +283,7 @@ class RenameServiceTables extends Migration
$table->dropIndex('ab_service__adsl_id_site_id_index'); $table->dropIndex('ab_service__adsl_id_site_id_index');
$table->foreign(['service_id','site_id'])->references(['id','site_id'])->on('services'); $table->foreign(['service_id','site_id'])->references(['id','site_id'])->on('services');
$table->foreign(['provided_supplier_broadband_id','site_id'])->references(['id','site_id'])->on('supplier_broadband');
}); });
// Convert our dates // Convert our dates

View File

@ -0,0 +1,42 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class RenameTraffic extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
DB::statement('ALTER TABLE ab_service__adsl_traffic RENAME TO usage_broadband');
DB::statement('ALTER TABLE usage_broadband RENAME COLUMN ab_service_adsl_id TO service_item_id');
DB::statement('ALTER TABLE usage_broadband MODIFY supplier_id int unsigned NOT NULL'); // @todo This should be deleted, it could be obtained by the product
DB::statement('ALTER TABLE usage_broadband MODIFY service_item_id int unsigned DEFAULT NULL');
DB::statement('ALTER TABLE usage_broadband MODIFY site_id int unsigned NOT NULL');
Schema::table('usage_broadband', function (Blueprint $table) {
$table->unique(['service_item_id','site_id','date','time']);
$table->foreign(['service_item_id','site_id'])->references(['id','site_id'])->on('service_broadband');
$table->foreign(['supplier_id'])->references(['id'])->on('suppliers');
});
Schema::table('suppliers', function (Blueprint $table) {
$table->date('usage_last')->nullable();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
abort(500,'Cant go back');
}
}