From 16b7e0b493a11c8b3c72185b1379bb2ce633273d Mon Sep 17 00:00:00 2001 From: Deon George Date: Wed, 20 Apr 2022 16:24:58 +1000 Subject: [PATCH] Update and fix broadband traffic --- app/Classes/External/Supplier.php | 26 ++++--- app/Console/Commands/BroadbandTraffic.php | 14 +++- app/Console/Kernel.php | 4 +- app/Interfaces/ServiceUsage.php | 7 ++ app/Jobs/BroadbandTraffic.php | 70 +++++++++++-------- app/Mail/TrafficMismatch.php | 6 +- app/Models/AdslSupplier.php | 48 ------------- app/Models/Product/Broadband.php | 2 +- app/Models/Service/Broadband.php | 15 ++-- app/Models/Supplier.php | 18 ++++- .../AdslTraffic.php => Usage/Broadband.php} | 18 +++-- ...022_04_19_121452_rename_service_tables.php | 3 + .../2022_04_20_134953_rename_traffic.php | 42 +++++++++++ 13 files changed, 161 insertions(+), 112 deletions(-) delete mode 100644 app/Models/AdslSupplier.php rename app/Models/{Service/AdslTraffic.php => Usage/Broadband.php} (82%) create mode 100644 database/migrations/2022_04_20_134953_rename_traffic.php diff --git a/app/Classes/External/Supplier.php b/app/Classes/External/Supplier.php index 9a7d22d..ecc8b24 100644 --- a/app/Classes/External/Supplier.php +++ b/app/Classes/External/Supplier.php @@ -17,6 +17,8 @@ abstract class Supplier protected $o = NULL; protected $_columns = []; + public const traffic_connection_keys = ['user','pass','url']; + public function __construct(Model $o) { $this->o = $o; @@ -26,24 +28,29 @@ abstract class Supplier /** * Connect and pull down traffic data * + * @param array $connection + * @param string $type * @return Collection + * @throws \Exception */ - public function fetch(): Collection + public function fetch(array $connection,string $type): Collection { + if (count(array_intersect(array_keys($connection),self::traffic_connection_keys)) !== 3) + throw new \Exception('No or missing connection details for:'.$type); + if ($x=$this->mustPause()) { Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]); sleep($x); } - Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,$this->o->stats_lastupdate),['m'=>__METHOD__]); - $key = 'Supplier:'.$this->o->id.$this->o->stats_lastupdate; - $result = Cache::remember($key,86400,function() { - $client = $this->getClient(); + Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,Arr::get($connection,'last')),['m'=>__METHOD__]); + $key = 'Supplier:'.$this->o->id.Arr::get($connection,'last'); - $response = Http::get($this->o->stats_url,[ - $this->login_user_field => $this->o->stats_username, - $this->login_pass_field => $this->o->stats_password, - $this->date_field => $this->o->stats_lastupdate->format('Y-m-d'), + $result = Cache::remember($key,86400,function() use ($connection) { + $response = Http::get(Arr::get($connection,'url'),[ + $this->login_user_field => Arr::get($connection,'user'), + $this->login_pass_field => Arr::get($connection,'pass'), + $this->date_field => Arr::get($connection,'last'), ]); // @todo These API rate limiting is untested. @@ -55,7 +62,6 @@ abstract class Supplier Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset)); } - //dd($response->header('Content-Type'),$response->headers()); // Assume the supplier provides an ASCII output for text/html if (preg_match('#^text/html;#',$x=$response->header('Content-Type'))) { return collect(explode("\n",$response->body()))->filter(); diff --git a/app/Console/Commands/BroadbandTraffic.php b/app/Console/Commands/BroadbandTraffic.php index 20a5f15..d3180d2 100644 --- a/app/Console/Commands/BroadbandTraffic.php +++ b/app/Console/Commands/BroadbandTraffic.php @@ -5,7 +5,7 @@ namespace App\Console\Commands; use Illuminate\Console\Command; use App\Jobs\BroadbandTraffic as Job; -use App\Models\AdslSupplier; +use App\Models\Supplier; class BroadbandTraffic extends Command { @@ -14,7 +14,8 @@ class BroadbandTraffic extends Command * * @var string */ - protected $signature = 'broadband:traffic:import'; + protected $signature = 'broadband:traffic:import'. + ' {--s|supplier= : Supplier Name}'; /** * The console command description. @@ -30,7 +31,14 @@ class BroadbandTraffic extends Command */ public function handle() { - foreach (AdslSupplier::active()->get() as $o) + if ($this->option('supplier')) { + $o = Supplier::where('name','like',$this->option('supplier'))->singleOrFail(); + + Job::dispatch($o); + return; + } + + foreach (Supplier::active()->get() as $o) Job::dispatch($o); } } \ No newline at end of file diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 0d08c8d..a921282 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -5,8 +5,8 @@ namespace App\Console; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; -use App\Models\AdslSupplier; use App\Jobs\BroadbandTraffic; +use App\Models\Supplier; class Kernel extends ConsoleKernel { @@ -29,7 +29,7 @@ class Kernel extends ConsoleKernel { // @todo This needs to be more generic and dynamic // Exetel Traffic - $schedule->job(new BroadbandTraffic(AdslSupplier::find(1)))->timezone('Australia/Melbourne')->dailyAt('10:00'); + $schedule->job(new BroadbandTraffic(Supplier::find(1)))->timezone('Australia/Melbourne')->dailyAt('10:00'); } /** diff --git a/app/Interfaces/ServiceUsage.php b/app/Interfaces/ServiceUsage.php index 713fee0..392b40d 100644 --- a/app/Interfaces/ServiceUsage.php +++ b/app/Interfaces/ServiceUsage.php @@ -6,6 +6,13 @@ use Illuminate\Support\Collection; interface ServiceUsage { + /** + * Our model that holds traffic information + * + * @return mixed + */ + public function traffic(); + /** * This service provides usage information * diff --git a/app/Jobs/BroadbandTraffic.php b/app/Jobs/BroadbandTraffic.php index 03a90f7..e079369 100644 --- a/app/Jobs/BroadbandTraffic.php +++ b/app/Jobs/BroadbandTraffic.php @@ -9,14 +9,16 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Arr; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Mail; +use App\Classes\External\Supplier as ExternalSupplier; use App\Mail\TrafficMismatch; -use App\Models\Service\Adsl; -use App\Models\Service\AdslTraffic; -use App\Models\AdslSupplier; +use App\Models\Supplier; +use App\Models\Service\Broadband as ServiceBroadband; +use App\Models\Usage\Broadband as UsageBroadband; /** * Class BroadbandTraffic @@ -33,7 +35,9 @@ final class BroadbandTraffic implements ShouldQueue protected Model $o; // The supplier we are updating from private const class_prefix = 'App\Classes\External\Supplier\\'; - public function __construct(AdslSupplier $o) + private const traffic = 'broadband'; + + public function __construct(Supplier $o) { $this->o = $o; } @@ -43,12 +47,14 @@ final class BroadbandTraffic implements ShouldQueue * * @return void * @throws \Exception - * @todo The column stats_lastupdate is actually the "next" date that stats should be retrieved. Rename it. */ public function handle() { Log::info(sprintf('%s:Importing Broadband Traffic from [%s]',self::LOGKEY,$this->o->name)); + if ((! $connection=$this->o->detail->connections->get('broadband')) || (count(array_intersect(array_keys($connection),ExternalSupplier::traffic_connection_keys)) !== 3)) + throw new \Exception('No or missing connection details for:'.self::traffic); + $u = 0; // Load our class for this supplier @@ -57,21 +63,24 @@ final class BroadbandTraffic implements ShouldQueue $o = new $class($this->o); } else { - Log::error(sprintf('%s: Class doesnt exist: %d',get_class($this),$class)); + Log::error(sprintf('%s: Class doesnt exist: %s',self::LOGKEY,$class)); exit(1); } + $last_update = Carbon::create(Arr::get($connection,'last'))->addDay(); + Arr::set($connection,'last',$last_update->format('Y-m-d')); + // Repeat pull traffic data until yesterday - while ($this->o->stats_lastupdate < Carbon::now()->subDay()) { - Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$this->o->stats_lastupdate->format('Y-m-d'))); + while ($last_update < Carbon::now()->subDay()) { + Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$last_update->format('Y-m-d'))); // Delete traffic, since we'll refresh it. - AdslTraffic::where('supplier_id',$this->o->id) - ->where('date',$this->o->stats_lastupdate) + UsageBroadband::where('supplier_id',$this->o->id) + ->where('date',$last_update->format('Y-m-d')) ->delete(); $c = 0; - foreach ($o->fetch() as $line) { + foreach ($o->fetch($connection,self::traffic) as $line) { // The first row is our header if (! $c++) { $fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header())); @@ -79,28 +88,26 @@ final class BroadbandTraffic implements ShouldQueue } if (! $fields->count()) - abort(500,'? No fields in data exportupda'); + abort(500,'? No fields in data export'); $row = str_getcsv(trim($line)); try { - // @todo Put the date format in the DB. $date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]); - - // Find the right service dependant on the dates we supplied the service - $oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')]) - ->select(DB::raw('ab_service__adsl.*')) + // Find the right service dependent on the dates we supplied the service + $oo = ServiceBroadband::where('service_username',$row[$o->getColumnKey('Login')]) + ->select(DB::raw('service_broadband.*')) ->join('services','services.id','=','service_id') ->where('services.start_at','<=',$date) ->where(function($query) use ($date) { $query->whereNULL('services.stop_at') ->orWhere('services.stop_at','<=',$date); }) - ->get(); + ->single(); - $to = new AdslTraffic; - $to->site_id = 1; // @todo TO ADDRESS - $to->date = $this->o->stats_lastupdate; + $to = new UsageBroadband; + $to->site_id = $oo->site_id; + $to->date = $last_update; $to->supplier_id = $this->o->id; $to->up_peak = $row[$o->getColumnKey('Peak upload')]; $to->up_offpeak = $row[$o->getColumnKey('Off peak upload')]; @@ -111,30 +118,35 @@ final class BroadbandTraffic implements ShouldQueue $to->time = '24:00'; // @todo // If we have no records - if ($oo->count() != 1) { + if (! $oo) { Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['date'=>$date,'count'=>$oo->count()]); $to->service = $row[$o->getColumnKey('Login')]; - $to->save(); } else { - $oo->first()->traffic()->save($to); + $to->service_item_id = $oo->id; } - $u++; + if ($to->save()) + $u++; } catch (\Exception $e) { + dump($to); Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['row'=>$row,'line'=>$line]); throw new \Exception('Error while storing traffic date'); } } + Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$last_update->format('Y-m-d'))); - Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$this->o->stats_lastupdate->format('Y-m-d'))); + // Save our current progress. + $this->o->detail->connections = $this->o->detail->connections->put(self::traffic,array_merge($connection,['last'=>$last_update->format('Y-m-d')])); + $this->o->detail->save(); + + // Update our details for the next iteration. + $last_update = $last_update->addDay(); + Arr::set($connection,'last',$last_update->format('Y-m-d')); if ($u) { - $this->o->stats_lastupdate = $this->o->stats_lastupdate->addDay(); - $this->o->save(); - if ($this->o->trafficMismatch($date)->count()) Mail::to('deon@graytech.net.au') // @todo To change ->send(new TrafficMismatch($this->o,$date)); diff --git a/app/Mail/TrafficMismatch.php b/app/Mail/TrafficMismatch.php index 0941bf6..cea1d17 100644 --- a/app/Mail/TrafficMismatch.php +++ b/app/Mail/TrafficMismatch.php @@ -8,13 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Mail\Mailable; use Illuminate\Queue\SerializesModels; -use App\Models\{AdslSupplier,Site}; +use App\Models\{Supplier,Site}; class TrafficMismatch extends Mailable { use Queueable, SerializesModels; - public AdslSupplier $aso; + public Supplier $aso; public Carbon $date; /** @@ -22,7 +22,7 @@ class TrafficMismatch extends Mailable * * @return void */ - public function __construct(AdslSupplier $o,Carbon $date) + public function __construct(Supplier $o,Carbon $date) { $this->aso = $o; $this->date = $date; diff --git a/app/Models/AdslSupplier.php b/app/Models/AdslSupplier.php deleted file mode 100644 index cd246ed..0000000 --- a/app/Models/AdslSupplier.php +++ /dev/null @@ -1,48 +0,0 @@ -where('active',TRUE); - } - - /** METHODS **/ - - /** - * Return the traffic records, that were not matched to a service. - * - * @param Carbon $date - * @return Collection - */ - public function trafficMismatch(Carbon $date): Collection - { - return AdslTraffic::where('date',$date->format('Y-m-d')) - ->where('supplier_id',$this->id) - ->whereNULL('ab_service_adsl_id') - ->get(); - } -} \ No newline at end of file diff --git a/app/Models/Product/Broadband.php b/app/Models/Product/Broadband.php index c623bc9..0b359d6 100644 --- a/app/Models/Product/Broadband.php +++ b/app/Models/Product/Broadband.php @@ -22,7 +22,7 @@ final class Broadband extends Type implements ProductItem 'options.address'=>[ 'request'=>'options.address', 'key'=>'service_address', - 'validation'=>'required|string:10|unique:ab_service__adsl,service_address', + 'validation'=>'required|string:10|unique:service_broadband,service_address', 'validation_message'=>'Address is a required field.', ], 'options.notes'=>[ diff --git a/app/Models/Service/Broadband.php b/app/Models/Service/Broadband.php index d09620a..facf731 100644 --- a/app/Models/Service/Broadband.php +++ b/app/Models/Service/Broadband.php @@ -10,6 +10,7 @@ use App\Interfaces\ServiceUsage; use App\Models\Base\ServiceType; use App\Models\Supplier\Broadband as SupplierBroadband; use App\Models\Supplier\Type; +use App\Models\Usage\Broadband as UsageBroadband; /** * Class Broadband (Service) @@ -65,17 +66,15 @@ class Broadband extends ServiceType implements ServiceUsage return $this->service_number ?: ($this->service_address ?: '-'); } - /* RELATIONS */ - /** - * The accounts that this user manages + * The usage information for broadband * * @return \Illuminate\Database\Eloquent\Relations\HasMany */ public function traffic() { - // @todo Need to include site_id in this relation - return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id'); + return $this->hasMany(UsageBroadband::class,'service_item_id') + ->where('site_id',$this->site_id); } /* ATTRIBUTES */ @@ -128,8 +127,6 @@ class Broadband extends ServiceType implements ServiceUsage if (! $maxdate) return collect(); - Log::debug(sprintf('%s:Getting Usage data for [%d] days from [%s]',self::LOGKEY,$days,$maxdate->date->format('Y-m-d')),['m'=>__METHOD__]); - return $this->traffic() ->where('date','<=',$maxdate->date->format('Y-m-d')) ->where('date','>=',$maxdate->date->subDays($days)->format('Y-m-d')) @@ -139,9 +136,9 @@ class Broadband extends ServiceType implements ServiceUsage /** * Find the last date any traffic was recorded for a service * - * @return AdslTraffic|null + * @return UsageBroadband|null */ - private function usage_last_date(): ?AdslTraffic + private function usage_last_date(): ?UsageBroadband { return $this->traffic ->sortBy('date') diff --git a/app/Models/Supplier.php b/app/Models/Supplier.php index dbdf702..e658867 100644 --- a/app/Models/Supplier.php +++ b/app/Models/Supplier.php @@ -2,6 +2,7 @@ namespace App\Models; +use Carbon\Carbon; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Arr; use Illuminate\Support\Collection; @@ -62,7 +63,8 @@ class Supplier extends Model // EG: Crazy Domains, "domains" and "hosting". public function detail() { - return $this->hasOne(SupplierDetail::class); + return $this->hasOne(SupplierDetail::class) + ->withoutGlobalScope(\App\Models\Scopes\SiteScope::class); } /* METHODS */ @@ -103,4 +105,18 @@ class Supplier extends Model return $result; } + + /** + * Return the traffic records, that were not matched to a service. + * + * @param Carbon $date + * @return \Illuminate\Database\Eloquent\Collection + */ + public function trafficMismatch(Carbon $date): Collection + { + return Usage\Broadband::where('date',$date->format('Y-m-d')) + ->where('supplier_id',$this->id) + ->whereNULL('service_item_id') + ->get(); + } } \ No newline at end of file diff --git a/app/Models/Service/AdslTraffic.php b/app/Models/Usage/Broadband.php similarity index 82% rename from app/Models/Service/AdslTraffic.php rename to app/Models/Usage/Broadband.php index 697237d..7386b56 100644 --- a/app/Models/Service/AdslTraffic.php +++ b/app/Models/Usage/Broadband.php @@ -1,23 +1,29 @@ belongsTo(Adsl::class); + return $this->belongsTo(ServiceBroadband::class); } + /* ATTRIBUTES */ + public function getTotalAttribute() { return $this->up_peak+$this->down_peak+$this->up_offpeak+$this->down_offpeak; } diff --git a/database/migrations/2022_04_19_121452_rename_service_tables.php b/database/migrations/2022_04_19_121452_rename_service_tables.php index 6f42bd7..2eb2516 100644 --- a/database/migrations/2022_04_19_121452_rename_service_tables.php +++ b/database/migrations/2022_04_19_121452_rename_service_tables.php @@ -266,6 +266,8 @@ class RenameServiceTables extends Migration DB::statement('ALTER TABLE service_broadband MODIFY service_id int unsigned NOT NULL'); DB::statement('ALTER TABLE service_broadband MODIFY service_stats_collect tinyint(1) DEFAULT NULL'); DB::statement('ALTER TABLE service_broadband RENAME COLUMN service_stats_lastupdate TO service_stats_at'); + DB::statement('ALTER TABLE service_broadband RENAME COLUMN provided_adsl_plan_id TO provided_supplier_broadband_id'); + DB::statement('ALTER TABLE service_broadband MODIFY provided_supplier_broadband_id int unsigned DEFAULT NULL'); // @todo drop column provided_adsl_plan_id Schema::table('service_broadband', function (Blueprint $table) { @@ -281,6 +283,7 @@ class RenameServiceTables extends Migration $table->dropIndex('ab_service__adsl_id_site_id_index'); $table->foreign(['service_id','site_id'])->references(['id','site_id'])->on('services'); + $table->foreign(['provided_supplier_broadband_id','site_id'])->references(['id','site_id'])->on('supplier_broadband'); }); // Convert our dates diff --git a/database/migrations/2022_04_20_134953_rename_traffic.php b/database/migrations/2022_04_20_134953_rename_traffic.php new file mode 100644 index 0000000..0ca84b3 --- /dev/null +++ b/database/migrations/2022_04_20_134953_rename_traffic.php @@ -0,0 +1,42 @@ +unique(['service_item_id','site_id','date','time']); + $table->foreign(['service_item_id','site_id'])->references(['id','site_id'])->on('service_broadband'); + $table->foreign(['supplier_id'])->references(['id'])->on('suppliers'); + }); + + Schema::table('suppliers', function (Blueprint $table) { + $table->date('usage_last')->nullable(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + abort(500,'Cant go back'); + } +}