Switchout DB to CockroachDB

This commit is contained in:
Deon George 2022-01-01 16:59:35 +11:00
parent afaa7d8bc7
commit 424d6ef39d
28 changed files with 1342 additions and 865 deletions

View File

@ -7,12 +7,16 @@ APP_DEBUG=true
LOG_CHANNEL=stack LOG_CHANNEL=stack
LOG_LEVEL=info LOG_LEVEL=info
DB_CONNECTION=pgsql DB_CONNECTION=cockroach
DB_HOST=postgres DB_HOST=cockroach
DB_PORT=5432 DB_PORT=26257
DB_DATABASE=postgres DB_DATABASE=clrghouz
DB_USERNAME=postgres DB_USERNAME=dev
DB_PASSWORD=password DB_PASSWORD=
DB_SSLMODE=prefer
DB_SSLROOTCERT=/var/www/html/config/ssl/ca.crt
DB_SSLCERT=/var/www/html/config/ssl/client.crt
DB_SSLKEY=/var/www/html/config/ssl/client.key
DB_MONGO_HOST=mongo DB_MONGO_HOST=mongo
DB_MONGO_USERNAME=mongo DB_MONGO_USERNAME=mongo

View File

@ -93,7 +93,7 @@ class Message extends FTNBase
private string $subject; // Message subject private string $subject; // Message subject
private string $msgid; // MSG ID private string $msgid; // MSG ID
private string $reply; // Reply ID private string $replyid; // Reply ID
private string $echoarea; // FTS-0004.001 private string $echoarea; // FTS-0004.001
private string $intl; // Netmail details private string $intl; // Netmail details
@ -182,7 +182,7 @@ class Message extends FTNBase
$this->message = ''; $this->message = '';
$this->msgid = ''; $this->msgid = '';
$this->reply = ''; $this->replyid = '';
$this->echoarea = ''; $this->echoarea = '';
$this->intl = ''; $this->intl = '';
@ -372,7 +372,7 @@ class Message extends FTNBase
case 'echoarea': case 'echoarea':
case 'msgid': case 'msgid':
case 'reply': case 'replyid':
case 'message': case 'message':
case 'message_src': case 'message_src':
@ -412,7 +412,7 @@ class Message extends FTNBase
case 'subject': case 'subject':
case 'msgid': case 'msgid':
case 'reply': case 'replyid':
case 'echoarea': case 'echoarea':
case 'intl': case 'intl':
@ -486,8 +486,8 @@ class Message extends FTNBase
// Add some kludges // Add some kludges
$return .= sprintf("\01MSGID: %s\r",$this->msgid); $return .= sprintf("\01MSGID: %s\r",$this->msgid);
if ($this->reply) if ($this->replyid)
$return .= sprintf("\01REPLY: %s\r",$this->reply); $return .= sprintf("\01REPLY: %s\r",$this->replyid);
foreach ($this->_kludge as $k=>$v) { foreach ($this->_kludge as $k=>$v) {
if ($x=$this->kludge->get($k)) if ($x=$this->kludge->get($k))

View File

@ -26,6 +26,7 @@ final class Test extends Process
private const testing = ['test','testing']; private const testing = ['test','testing'];
// @todo add path and other kludges
public static function handle(Message $msg): bool public static function handle(Message $msg): bool
{ {
if ((strtolower($msg->user_to) !== 'all') || ! in_array(strtolower($msg->subject),self::testing)) if ((strtolower($msg->user_to) !== 'all') || ! in_array(strtolower($msg->subject),self::testing))
@ -66,7 +67,7 @@ final class Test extends Process
$o->datetime = Carbon::now(); $o->datetime = Carbon::now();
$o->tzoffset = $o->datetime->utcOffset(); $o->tzoffset = $o->datetime->utcOffset();
$o->echoarea_id = $eo?->id; $o->echoarea_id = $eo?->id;
$o->reply = $msg->msgid; $o->replyid = $msg->msgid;
$o->fftn_id = $ftns->id; $o->fftn_id = $ftns->id;
$o->flags = Message::FLAG_LOCAL; $o->flags = Message::FLAG_LOCAL;

View File

@ -50,7 +50,7 @@ final class Ping extends Process
$o->datetime = Carbon::now(); $o->datetime = Carbon::now();
$o->tzoffset = $o->datetime->utcOffset(); $o->tzoffset = $o->datetime->utcOffset();
$o->reply = $msg->msgid; $o->replyid = $msg->msgid;
$o->fftn_id = $ftns->id; $o->fftn_id = $ftns->id;
$o->tftn_id = ($x=$msg->fftn_o) ? $x->id : NULL; $o->tftn_id = ($x=$msg->fftn_o) ? $x->id : NULL;
$o->flags = Message::FLAG_LOCAL; $o->flags = Message::FLAG_LOCAL;

View File

@ -0,0 +1,160 @@
<?php
namespace App\Console\Commands;
use App\Models\Echoarea;
use App\Models\Echomail;
use App\Models\Netmail;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Notification;
class ConvertMongo extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'convert:mongo {start=0}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Convert Mongo to Cockroach';
/**
* Execute the console command.
*
* @return mixed
* @throws \Exception
*/
public function handle()
{
/*
// Netmails
foreach (DB::connection('mongodb')->collection('netmails')->cursor() as $o) {
$o['mid'] = (string)$o['_id'];
foreach([
'reply'=>'replyid',
'packet'=>'sent_pkt',
] as $key => $newkey) {
if (array_key_exists($key,$o)) {
$o[$newkey] = $o[$key];
unset($o[$key]);
}
}
unset($o['_id'],$o['sent'],$o['reply']);
foreach (['created_at','updated_at','sent_at','datetime','deleted_at'] as $key) {
if (Arr::get($o,$key))
$o[$key] = Carbon::create($o[$key]->toDatetime());
}
if (! Arr::get($o,'datetime'))
$o['datetime'] = $o['created_at'];
$oo = Netmail::withTrashed()->where('mid',$o['mid'])->firstOrNew();
$oo->forceFill($o);
try {
$oo->save(['timestamps' => false]);
} catch (\Exception $e) {
$this->warn(sprintf('Netmail didnt move over: %s (%s)',$o['mid'],$e->getMessage()));
}
}
*/
/**/
// Echomail
$c = 0;
foreach (DB::connection('mongodb')->collection('echomails')->cursor() as $o) {
if (++$c < $this->argument('start'))
continue;
if (! ($c%100))
Log::debug(sprintf('Processed : %d Echomails',$c));
$o['mid'] = (string)$o['_id'];
foreach([
'reply'=>'replyid',
'rogue_seen'=>'rogue_seenby',
] as $key => $newkey) {
if (array_key_exists($key,$o)) {
$o[$newkey] = $o[$key];
unset($o[$key]);
}
}
$path = (array_key_exists('path',$o) && $o['path']) ? $o['path'] : NULL;
$seenby = (array_key_exists('seenby',$o) && $o['seenby']) ? $o['seenby'] : NULL;
$packet = (array_key_exists('packet',$o) && $o['packet']) ? $o['packet'] : NULL;
unset($o['_id'],$o['reply'],$o['path'],$o['seenby'],$o['toexport'],$o['sent_at'],$o['packet'],$o['sent']);
foreach (['created_at','updated_at','datetime','deleted_at'] as $key) {
if (Arr::get($o,$key))
$o[$key] = Carbon::create($o[$key]->toDatetime());
}
if (! Arr::get($o,'datetime'))
$o['datetime'] = $o['created_at'];
if (Arr::get($o,'echoarea') && ! Arr::get($o,'echoarea_id')) {
$ea = Echoarea::where('name',$o['echoarea'])->single();
$o['echoarea_id'] = $ea->id;
unset($o['echoarea']);
}
if (! $o['echoarea_id']) {
Log::error(sprintf('Echomail didnt move over: %s [%d] - has no echoarea_id',$o['mid'],$c));
continue;
}
if (Arr::get($o,'msg_src') && ! Arr::get($o,'msg_crc')) {
$o['msg_crc'] = 'x'.md5($o['msg_src']);
}
$oo = Echomail::withTrashed()->where('mid',$o['mid'])->firstOrNew();
$oo->forceFill($o);
$oo->set_path = $path ? array_filter($path) : [];
$oo->set_seenby = $seenby ? array_filter($seenby): [];
$oo->set_packet = $packet;
try {
$oo->save(['timestamps'=>FALSE]);
} catch (\Exception $e) {
Log::error(sprintf('Echomail didnt move over: %s [%d] (%s@%d|%s)',$o['mid'],$c,$e->getFile(),$e->getLine(),$e->getMessage()));
dd(['e'=>$e,'o'=>$o,'oo'=>$oo]);
}
DB::connection('mongodb')->collection('echomails')->delete($o['mid']);
}
/**/
// Update old MID seenby with proper ID
foreach (DB::connection('cockroach')->table('echomail_seenby')->whereNotNull('mid')->whereNull('echomail_id')->cursor() as $o)
{
$eo = Echomail::where('mid',$o->mid)->get();
if ($eo->count() && $eo->count() == 1) {
DB::update('UPDATE echomail_seenby set echomail_id = ?, mid=NULL where echomail_id IS NULL AND mid = ? ',[
$eo->first()->id,
$o->mid,
]);
} elseif ($eo->count() > 1) {
Log::error(sprintf('Echomail [%s] has more than 1 record [%d] - skipped',$o->mid,$eo->count()));
}
}
}
}

View File

@ -121,6 +121,7 @@ class HomeController extends Controller
# Look for Messages # Look for Messages
foreach (Echomail::select(['id','fftn_id','from']) foreach (Echomail::select(['id','fftn_id','from'])
->where('msgid','like','%'.$request->query('term').'%') ->where('msgid','like','%'.$request->query('term').'%')
->orWhere('replyid','like','%'.$request->query('term').'%')
->get() as $o) ->get() as $o)
{ {
$result->push(['id'=>$o->id,'name'=>sprintf('%s (%s)',$o->from,$o->fftn->ftn3d),'value'=>url('echomail/view',[$o->id]),'category'=>'Echomail']); $result->push(['id'=>$o->id,'name'=>sprintf('%s (%s)',$o->from,$o->fftn->ftn3d),'value'=>url('echomail/view',[$o->id]),'category'=>'Echomail']);

View File

@ -112,7 +112,7 @@ class MessageProcess implements ShouldQueue
$o->fftn_id = ($x=$this->msg->tftn_o) ? $x->id : NULL; $o->fftn_id = ($x=$this->msg->tftn_o) ? $x->id : NULL;
$o->tftn_id = ($x=$this->msg->fftn_o) ? $x->id : NULL; $o->tftn_id = ($x=$this->msg->fftn_o) ? $x->id : NULL;
$o->reply = $this->msg->msgid; $o->replyid = $this->msg->msgid;
$o->msg = Process::format_msg($reply,$reject); $o->msg = Process::format_msg($reply,$reject);
$o->tagline = 'Do you think it was fate which brought us together? Nah, bad luck :('; $o->tagline = 'Do you think it was fate which brought us together? Nah, bad luck :(';

View File

@ -11,7 +11,7 @@ use Illuminate\Support\Facades\DB;
use App\Classes\FTN\Packet; use App\Classes\FTN\Packet;
use App\Http\Controllers\DomainController; use App\Http\Controllers\DomainController;
use App\Traits\{ScopeActive,UsePostgres}; use App\Traits\ScopeActive;
/** /**
* @todo Need to stop this from happening: * @todo Need to stop this from happening:
@ -24,7 +24,9 @@ use App\Traits\{ScopeActive,UsePostgres};
*/ */
class Address extends Model class Address extends Model
{ {
use ScopeActive,SoftDeletes,UsePostgres; use ScopeActive,SoftDeletes;
protected $with = ['zone'];
/* SCOPES */ /* SCOPES */
@ -337,6 +339,21 @@ class Address extends Model
return ($o && $o->system->active) ? $o : NULL; return ($o && $o->system->active) ? $o : NULL;
} }
/**
* Netmail waiting to be sent to this system
*
* @return Collection
*/
public function echomailWaiting(): Collection
{
return Echomail::select(['echomails.*'])
->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
->where('echomail_seenby.address_id',$this->id)
->whereNull('echomail_seenby.sent_at')
->whereNotNull('echomail_seenby.export_at')
->get();
}
/** /**
* Get echomail for this node * Get echomail for this node
* *
@ -346,23 +363,17 @@ class Address extends Model
{ {
$pkt = NULL; $pkt = NULL;
$echomail = DB::table('address_echomail') if (($x=$this->echomailWaiting())
->select('echomail_id')
->where('address_id',$this->id)
->whereNull('sent_date')
->get();
if (($x=Echomail::select('*')
->whereIn('_id',$echomail->pluck('echomail_id')))
->count()) ->count())
{ {
$pkt = $this->getPacket($x->get()); $pkt = $this->getPacket($x);
DB::table('address_echomail') DB::table('echomail_seenby')
->whereIn('echomail_id',$echomail->pluck('echomail_id')) ->whereIn('echomail_id',$x->pluck('id'))
->where('address_id',$this->id) ->where('address_id',$this->id)
->whereNull('sent_date') ->whereNull('sent_at')
->update(['sent_date'=>Carbon::now()]); ->whereNotNull('echomail_seenby.export_at')
->update(['sent_at'=>Carbon::now(),'packet'=>$pkt->name]);
} }
return $pkt; return $pkt;
@ -375,18 +386,19 @@ class Address extends Model
*/ */
public function getNetmail(): ?Packet public function getNetmail(): ?Packet
{ {
if (($x=Netmail::whereIn('tftn_id',(($x=$this->children) ? $x->pluck('id') : collect())->push($this->id)) $pkt = NULL;
->where(function($q) {
return $q->whereNull('sent') if (($x=$this->netmailWaiting())
->orWhere('sent',FALSE);
}))
->whereNull('local')
->count()) ->count())
{ {
return $this->getPacket($x->get()); $pkt = $this->getPacket($x);
DB::table('netmails')
->whereIn('id',$x->pluck('id'))
->update(['sent_at'=>Carbon::now(),'sent_pkt'=>$pkt->name]);
} }
return NULL; return $pkt;
} }
/** /**
@ -411,6 +423,18 @@ class Address extends Model
return $o; return $o;
} }
/**
* Netmail waiting to be sent to this system
*
* @return Collection
*/
public function netmailWaiting(): Collection
{
return Netmail::whereIn('tftn_id',(($x=$this->children) ? $x->pluck('id') : collect())->push($this->id))
->whereNull('sent_at')
->get();
}
/** /**
* Parse a string and split it out as an FTN array * Parse a string and split it out as an FTN array
* *

View File

@ -7,13 +7,15 @@ use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Collection; use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use MongoDB\BSON\UTCDateTime; use MongoDB\BSON\UTCDateTime;
use App\Traits\ScopeActive; use App\Traits\{QueryCacheableConfig,ScopeActive};
class Domain extends Model class Domain extends Model
{ {
use HasFactory,ScopeActive; use HasFactory,ScopeActive,QueryCacheableConfig;
private const CACHE_TIME = 3600; private const CACHE_TIME = 3600;
private const STATS_MONTHS = 6; private const STATS_MONTHS = 6;
@ -65,17 +67,19 @@ class Domain extends Model
$key = sprintf('%s_%d','daily_area_stats',$this->id); $key = sprintf('%s_%d','daily_area_stats',$this->id);
return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() { Cache::forget($key);
$where = [ return Cache::remember($key,self::CACHE_TIME,function() {
'echoarea_id'=>$this->echoareas->pluck('id')->toArray(), $gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime";
'datetime' => ['$gte',new UTCDateTime(Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())],
];
$echostats = Echomail::countGroupBy(['datetime',['datetime'=>'%Y-%m-%d']],$where); $echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')])
->whereIn('id',$this->echoareas->pluck('id')->toArray())
->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())
->groupBy('datetime')
->orderBy('datetime')
->get();
return $echostats return $echostats
->sortBy(function($item) { return $item->id->datetime; }) ->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; })
->map(function($item) { return ['x'=>Carbon::createFromFormat('Y-m-d',$item->id->datetime)->timestamp*1000,'y'=>$item->count]; })
->values(); ->values();
}); });
} }
@ -87,17 +91,19 @@ class Domain extends Model
$key = sprintf('%s_%d-%d','daily_echoarea_stats',$this->id,$o->id); $key = sprintf('%s_%d-%d','daily_echoarea_stats',$this->id,$o->id);
return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() use ($o) { Cache::forget($key);
$where = [ return Cache::remember($key,self::CACHE_TIME,function() use ($o) {
'echoarea_id'=>[$o->id], $gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime";
'datetime' => ['$gte',new UTCDateTime(Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())],
];
$echostats = Echomail::countGroupBy(['datetime',['datetime'=>'%Y-%m-%d']],$where); $echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')])
->whereIn('echoarea_id',[$o->id])
->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())
->groupBy('datetime')
->orderBy('datetime')
->get();
return $echostats return $echostats
->sortBy(function($item) { return $item->id->datetime; }) ->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; })
->map(function($item) { return ['x'=>Carbon::createFromFormat('Y-m-d',$item->id->datetime)->timestamp*1000,'y'=>$item->count]; })
->values(); ->values();
}); });
} }

View File

@ -3,14 +3,16 @@
namespace App\Models; namespace App\Models;
use Carbon\Carbon; use Carbon\Carbon;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Cache;
use App\Traits\{ScopeActive,UsePostgres};
use App\Traits\ScopeActive;
class Echoarea extends Model class Echoarea extends Model
{ {
use SoftDeletes,ScopeActive,UsePostgres; use SoftDeletes,ScopeActive;
private const CACHE_TIME = 3600; private const CACHE_TIME = 3600;
@ -28,28 +30,24 @@ class Echoarea extends Model
public function echomail() public function echomail()
{ {
return Echomail::select('*') return $this->hasMany(Echomail::class)
->where('echoarea_id',$this->id); ->orderBy('datetime','ASC');
} }
/* ATTRIBUTES */ /* ATTRIBUTES */
public function getLastMessageAttribute(): ?Carbon public function getLastMessageAttribute(): ?Carbon
{ {
$key = sprintf('%s_%d','echo_last_message',$this->id); return $this->echomail?->last()->datetime;
return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() {
return ($x=$this->echomail()->orderBy('datetime','DESC')->first()) ? $x->datetime : NULL;
});
} }
/* METHODS */ /* METHODS */
public function messages_count(int $period): int public function messages_count(int $period): int
{ {
$key = sprintf('%s_%d_%d','echo_mesages_count',$this->id,$period); $key = sprintf('%s_%d_%d','echo_messages_count',$this->id,$period);
return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() use ($period) { return Cache::remember($key,self::CACHE_TIME,function() use ($period) {
switch ($period) { switch ($period) {
case 1: // day case 1: // day
return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subDay())->count(); return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subDay())->count();
@ -62,4 +60,20 @@ class Echoarea extends Model
} }
}); });
} }
/**
* Number of messages waiting for address
*
* @param Address $ao
* @return Collection
*/
public function waiting(Address $ao): Collection
{
return $this->echomail()
->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
->whereNull('sent_at')
->whereNotNull('export_at')
->where('address_id',$ao->id)
->get();
}
} }

View File

@ -3,25 +3,29 @@
namespace App\Models; namespace App\Models;
use Carbon\Carbon; use Carbon\Carbon;
use Illuminate\Support\Collection; use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use Jenssegers\Mongodb\Eloquent\Model;
use Jenssegers\Mongodb\Eloquent\SoftDeletes;
use App\Classes\FTN\Message; use App\Classes\FTN\Message;
use App\Interfaces\Packet; use App\Interfaces\Packet;
use App\Traits\{EncodeUTF8,MsgID,UseMongo}; use App\Traits\{EncodeUTF8,MsgID};
final class Echomail extends Model implements Packet final class Echomail extends Model implements Packet
{ {
use SoftDeletes,MsgID,UseMongo,EncodeUTF8; use SoftDeletes,EncodeUTF8,MsgID;
private const LOGKEY = 'ME-'; private const LOGKEY = 'ME-';
private array $set_seenby = [];
private array $set_path = [];
private ?string $set_packet = NULL;
protected $collection = FALSE; protected $casts = [
'kludges' => 'json',
protected $casts = [ 'kludges' => 'json' ]; 'rogue_seenby' => 'json',
'rogue_path' => 'json',
];
private const cast_utf8 = [ private const cast_utf8 = [
'to', 'to',
@ -35,6 +39,20 @@ final class Echomail extends Model implements Packet
protected $dates = ['datetime']; protected $dates = ['datetime'];
public function __set($key, $value)
{
switch ($key) {
case 'set_path':
case 'set_packet':
case 'set_seenby':
$this->{$key} = $value;
break;
default:
parent::__set($key,$value);
}
}
public static function boot() public static function boot()
{ {
parent::boot(); parent::boot();
@ -46,29 +64,43 @@ final class Echomail extends Model implements Packet
return; return;
} }
// See if we need to export this message. // Save the seenby
$exportto = $model->echoarea->addresses->pluck('system')->diff($model->seenby->pluck('system')); foreach ($model->set_seenby as $aoid) {
DB::insert('INSERT INTO echomail_seenby (echomail_id,address_id,packet) VALUES (?,?,?)',[
$export_ao = collect(); $model->id,
foreach ($model->echoarea->domain->zones as $zo) { $aoid,
foreach ($exportto as $so) { $model->set_packet,
$export_ao = $export_ao->merge($so->match($zo));
}
}
// Add to export
foreach ($export_ao as $ao) {
Log::info(sprintf('%s:- Exporting message [%s] to [%s]',self::LOGKEY,$model->id,$ao->ftn));
DB::table('address_echomail')->insert([
'address_id'=>$ao->id,
'echomail_id'=>$model->id,
'export_date'=>Carbon::now()
]); ]);
} }
$model->seenby = $model->seenby->merge($export_ao)->pluck('id')->toArray(); // Save the Path
$model->save(); $ppoid = NULL;
foreach ($model->set_path as $aoid) {
$po = DB::select('INSERT INTO echomail_path (echomail_id,address_id,parent_id) VALUES (?,?,?) RETURNING id',[
$model->id,
$aoid,
$ppoid,
]);
$ppoid = $po[0]->id;
}
// See if we need to export this message.
$exportto = $model->echoarea->addresses->pluck('id')->diff($model->set_seenby);
if ($exportto->count()) {
Log::debug(sprintf('%s:- Exporting message [%s] to [%s]',self::LOGKEY,$model->id,$exportto->join(',')));
// Save the seenby for the exported systems
$export_at = Carbon::now();
foreach ($exportto as $aoid) {
DB::insert('INSERT INTO echomail_seenby (echomail_id,address_id,export_at) VALUES (?,?,?)',[
$model->id,
$aoid,
$export_at,
]);
}
}
}); });
} }
@ -81,35 +113,19 @@ final class Echomail extends Model implements Packet
public function fftn() public function fftn()
{ {
return $this return $this->belongsTo(Address::class)
->setConnection('pgsql')
->belongsTo(Address::class)
->withTrashed(); ->withTrashed();
} }
/* ATTRIBUTES */ public function seenby()
public function getKludgesAttribute(?string $value): Collection
{ {
return collect($this->castAttribute('kludges',$value)); return $this->belongsToMany(Address::class,'echomail_seenby')
->ftnOrder();
} }
public function getPathAttribute(?array $value): Collection public function path()
{ {
if (is_null($value)) return $this->belongsToMany(Address::class,'echomail_path');
return collect();
return Address::whereIn('id',$value)
->orderBy(DB::raw(sprintf("position (id::text in '(%s)')",join(',',$value))))
->get();
}
public function getSeenByAttribute(?array $value): Collection
{
if (is_null($value))
return collect();
return Address::whereIn('id',$value)->get();
} }
/* METHODS */ /* METHODS */
@ -149,11 +165,11 @@ final class Echomail extends Model implements Packet
if ($this->kludges) if ($this->kludges)
$o->kludge = $this->kludges; $o->kludge = $this->kludges;
$o->kludge->put('mid',$this->id); $o->kludge->put('dbid',$this->id);
$o->msgid = $this->msgid; $o->msgid = $this->msgid;
if ($this->reply) if ($this->replyid)
$o->reply = $this->reply; $o->replyid = $this->replyid;
$o->message = $this->msg; $o->message = $this->msg;

View File

@ -3,19 +3,19 @@
namespace App\Models; namespace App\Models;
use Carbon\Carbon; use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use Jenssegers\Mongodb\Eloquent\Model;
use Jenssegers\Mongodb\Eloquent\SoftDeletes;
use App\Classes\FTN\Message; use App\Classes\FTN\Message;
use App\Interfaces\Packet; use App\Interfaces\Packet;
use App\Traits\{EncodeUTF8,UseMongo}; use App\Traits\EncodeUTF8;
final class Netmail extends Model implements Packet final class Netmail extends Model implements Packet
{ {
private const LOGKEY = 'MN-'; private const LOGKEY = 'MN-';
use SoftDeletes,UseMongo,EncodeUTF8; use SoftDeletes,EncodeUTF8;
private const cast_utf8 = [ private const cast_utf8 = [
'to', 'to',
@ -80,8 +80,8 @@ final class Netmail extends Model implements Packet
$o->flags = $this->flags; $o->flags = $this->flags;
$o->msgid = sprintf('%s %08x',$this->fftn->ftn3d,crc32($this->id)); $o->msgid = sprintf('%s %08x',$this->fftn->ftn3d,crc32($this->id));
if ($this->reply) if ($this->replyid)
$o->reply = $this->reply; $o->replyid = $this->replyid;
$o->message = $this->msg; $o->message = $this->msg;
$o->tagline = $this->tagline; $o->tagline = $this->tagline;

View File

@ -8,10 +8,11 @@ use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\Auth;
use App\Http\Controllers\DomainController; use App\Http\Controllers\DomainController;
use App\Traits\QueryCacheableConfig;
class System extends Model class System extends Model
{ {
use HasFactory; use HasFactory,QueryCacheableConfig;
protected $dates = ['last_session']; protected $dates = ['last_session'];
@ -37,10 +38,7 @@ class System extends Model
public function addresses() public function addresses()
{ {
return $this->hasMany(Address::class) return $this->hasMany(Address::class)
->orderBy('region_id') ->FTNorder();
->orderBy('host_id')
->orderBy('node_id')
->orderBy('point_id');
} }
/** /**

View File

@ -4,11 +4,11 @@ namespace App\Models;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
use App\Traits\ScopeActive; use App\Traits\{QueryCacheableConfig,ScopeActive};
class Zone extends Model class Zone extends Model
{ {
use ScopeActive; use ScopeActive,QueryCacheableConfig;
/* SCOPES */ /* SCOPES */

View File

@ -7,13 +7,15 @@ namespace App\Traits;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;
use App\Classes\FTN\Message;
use App\Models\Setup; use App\Models\Setup;
trait MsgID trait MsgID
{ {
public function save(array $options = []) public function save(array $options = [])
{ {
if (! $this->exists) { // Only create a MSGID for locally generated conetnt
if ((! $this->exists) && ($this->flags & Message::FLAG_LOCAL)) {
$ftns = Setup::findOrFail(config('app.id'))->system->match($this->fftn->zone)->first(); $ftns = Setup::findOrFail(config('app.id'))->system->match($this->fftn->zone)->first();
if (is_null(Arr::get($this->attributes,'msgid'))) if (is_null(Arr::get($this->attributes,'msgid')))

View File

@ -0,0 +1,17 @@
<?php
/**
* Set defaults of QueryCacheable
*/
namespace App\Traits;
use Rennokki\QueryCache\Traits\QueryCacheable;
trait QueryCacheableConfig
{
use QueryCacheable;
public $cacheFor = 900; // cache time, in seconds
protected static $flushCacheOnUpdate = TRUE;
public $cacheDriver = 'memcached';
}

View File

@ -1,104 +0,0 @@
<?php
/**
* Models that store data in Mongo
*/
namespace App\Traits;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Arr;
trait UseMongo
{
/**
* Resolve a connection instance.
* We need to do this, because our relations are in pgsql and somehow loading a relation changes this models
* connection information. Using protected $connection is not enough.
*
* @param string|null $connection
*/
public static function resolveConnection($connection = null)
{
return static::$resolver->connection('mongodb');
}
/* ATTRIBUTES */
public function getMsgAttribute($value): string
{
return utf8_decode($value);
}
public function setMsgAttribute($value): void
{
$this->attributes['msg'] = utf8_encode($value);
}
public function getSubjectAttribute($value): string
{
return utf8_decode($value);
}
public function setSubjectAttribute($value): void
{
$this->attributes['subject'] = utf8_encode($value);
}
/* METHODS */
public static function countGroupBy(array $fields,array $where=[]): Collection
{
$query = collect();
if (count($where)) {
$where_condition = [];
foreach ($where as $key => $values) {
if (! is_array($values))
throw new \Exception('Where values must be an array.');
switch ($x=Arr::get($values,0)) {
case '$gt':
case '$gte':
$where_condition[$key] = [ $x => Arr::get($values,1)];
break;
case '$in':
default:
$where_condition[$key] = ['$in' => $values];
}
}
$query->push([
'$match' => [ '$and'=> [$where_condition]]
]);
}
$gb = collect();
foreach ($fields as $field)
if (is_array($field)) {
foreach ($field as $k=>$v) {
$gb->put('datetime',['$dateToString'=>['format'=>$v,'date'=>'$'.$k]]);
}
} else {
$gb->put($field,'$'.$field);
}
$query->push([
'$group' => [
'_id' => $gb->toArray(),
'count' => ['$sum' => 1]
]
]);
return (new self)
->groupBy($field)
->raw(function($collection) use ($query) {
return $collection->aggregate(
$query->toArray()
);
});
}
}

View File

@ -1,21 +0,0 @@
<?php
/**
* Models that store data in Postgres
*/
namespace App\Traits;
trait UsePostgres
{
/**
* Resolve a connection instance.
* We need to do this, because our relations are in pgsql and somehow loading a relation changes this models
* connection information. Using protected $connection is not enough.
*
* @param string|null $connection
*/
public static function resolveConnection($connection = null)
{
return static::$resolver->connection('pgsql');
}
}

View File

@ -17,6 +17,7 @@
"laravel/framework": "^8.0", "laravel/framework": "^8.0",
"laravel/passport": "^10.1", "laravel/passport": "^10.1",
"laravel/ui": "^3.2", "laravel/ui": "^3.2",
"leenooks/cockroachdb-laravel": "^0.1.0",
"rennokki/laravel-eloquent-query-cache": "^3.1", "rennokki/laravel-eloquent-query-cache": "^3.1",
"repat/laravel-job-models": "^0.5.1" "repat/laravel-job-models": "^0.5.1"
}, },
@ -54,6 +55,12 @@
"Tests\\": "tests/" "Tests\\": "tests/"
} }
}, },
"repositories": [
{
"type": "vcs",
"url": "https://github.com/leenooks/cockroachdb-laravel"
}
],
"minimum-stability": "dev", "minimum-stability": "dev",
"prefer-stable": true, "prefer-stable": true,
"scripts": { "scripts": {

1265
composer.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -67,7 +67,10 @@ return [
'prefix' => '', 'prefix' => '',
'prefix_indexes' => true, 'prefix_indexes' => true,
'schema' => 'public', 'schema' => 'public',
'sslmode' => 'prefer', 'sslmode' => 'disable', //depends on your security level https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
'sslrootcert' => env('DB_SSLROOTCERT', 'config/ssl/ca.crt'),
'sslcert' => env('DB_SSLCERT', 'config/ssl/client.crt'),
'sslkey' => env('DB_SSLKEY', 'config/ssl/client.key'),
], ],
'sqlsrv' => [ 'sqlsrv' => [
@ -92,13 +95,13 @@ return [
'charset' => 'utf8', 'charset' => 'utf8',
'prefix' => '', 'prefix' => '',
'schema' => 'public', 'schema' => 'public',
'sslmode' => 'prefer', 'sslmode' => env('DB_SSLMODE', 'prefer'),
// Only set these keys if you want to run en secure mode // Only set these keys if you want to run en secure mode
// otherwise you can them out of the configuration array // otherwise you can them out of the configuration array
#'sslcert' => env('DB_SSLCERT', 'client.crt'), 'sslcert' => env('DB_SSLCERT', 'config/ssl/client.crt'),
#'sslkey' => env('DB_SSLKEY', 'client.key'), 'sslkey' => env('DB_SSLKEY', 'config/ssl/client.key'),
#'sslrootcert' => env('DB_SSLROOTCERT', 'ca.crt'), 'sslrootcert' => env('DB_SSLROOTCERT', 'config/ssl/ca.crt'),
], ],
'mongodb' => [ 'mongodb' => [

View File

@ -16,11 +16,10 @@ class UpdateZones extends Migration
DB::statement('ALTER TABLE zones ALTER COLUMN domain_id SET NOT NULL'); DB::statement('ALTER TABLE zones ALTER COLUMN domain_id SET NOT NULL');
Schema::table('zones', function (Blueprint $table) { Schema::table('zones', function (Blueprint $table) {
$table->integer('system_id');
$table->dropColumn(['description','ipv4','ipv4_mask','ipv6','ipv6_mask','zt_id']); $table->dropColumn(['description','ipv4','ipv4_mask','ipv6','ipv6_mask','zt_id']);
$table->string('ztid')->nullable(); $table->string('ztid')->nullable();
$table->dropUnique(['domain_id']); $table->dropUnique(['domain_id']);
$table->integer('system_id');
$table->foreign('system_id')->references('id')->on('systems'); $table->foreign('system_id')->references('id')->on('systems');
}); });
} }

View File

@ -14,8 +14,8 @@ class AddSystemToSetups extends Migration
public function up() public function up()
{ {
Schema::table('setups', function (Blueprint $table) { Schema::table('setups', function (Blueprint $table) {
$table->integer('options');
$table->integer('system_id'); $table->integer('system_id');
$table->integer('options');
$table->foreign('system_id')->references('id')->on('systems'); $table->foreign('system_id')->references('id')->on('systems');
}); });
} }

View File

@ -0,0 +1,137 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateMail extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('echomails', function (Blueprint $table) {
$table->id();
$table->timestamps();
$table->softDeletes();
$table->string('mid')->nullable();
$table->string('to',128);
$table->string('from',128);
$table->string('subject',256);
$table->dateTimeTz('datetime');
$table->tinyInteger('tzoffset');
$table->string('msgid')->nullable();
$table->string('replyid')->nullable();
$table->text('msg');
$table->string('msg_src')->nullable();
$table->string('msg_crc')->nullable();
$table->string('tagline')->nullable();
$table->string('tearline')->nullable();
$table->string('origin')->nullable();
$table->tinyInteger('flags')->nullable();
$table->jsonb('rogue_path')->nullable();
$table->jsonb('rogue_seenby')->nullable();
$table->jsonb('kludges')->nullable();
$table->integer('echoarea_id');
$table->foreign('echoarea_id')->references('id')->on('echoareas');
$table->integer('fftn_id');
$table->foreign('fftn_id')->references('id')->on('addresses');
});
DB::statement("ALTER TABLE address_echomail RENAME COLUMN echomail_id TO mid");
DB::statement("ALTER TABLE address_echomail ALTER COLUMN mid DROP NOT NULL");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN export_date TO export_at");
DB::statement("ALTER TABLE address_echomail ALTER COLUMN export_at DROP NOT NULL");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN sent_date TO sent_at");
DB::statement("ALTER TABLE address_echomail RENAME TO echomail_seenby");
Schema::table('echomail_seenby', function (Blueprint $table) {
$table->integer('echomail_id')->nullable()->first();
$table->string('packet')->nullable();
$table->foreign('echomail_id')->references('id')->on('echomails');
});
Schema::create('echomail_path', function (Blueprint $table) {
$table->id();
$table->integer('address_id');
$table->foreign('address_id')->references('id')->on('addresses');
$table->unique(['id','echomail_id']);
$table->unique(['echomail_id','address_id','parent_id']);
$table->integer('parent_id')->nullable();
$table->foreign(['parent_id','echomail_id'])->references(['id','echomail_id'])->on('echomail_path');
$table->integer('echomail_id');
$table->foreign('echomail_id')->references('id')->on('echomails');
});
Schema::create('netmails', function (Blueprint $table) {
$table->id();
$table->timestamps();
$table->softDeletes();
$table->string('mid')->nullable();
$table->string('to',128);
$table->string('from',128);
$table->string('subject',256);
$table->dateTimeTz('datetime');
$table->tinyInteger('tzoffset')->nullable();
$table->tinyInteger('flags')->nullable();
$table->tinyInteger('cost')->nullable();
$table->string('msgid')->nullable();
$table->string('replyid')->nullable();
$table->text('msg');
$table->text('msg_src')->nullable();
$table->integer('msg_crc')->nullable();
$table->string('tagline')->nullable();
$table->string('tearline')->nullable();
$table->boolean('local')->default(FALSE);
$table->string('recv_pkt')->nullable();
$table->string('sent_pkt')->nullable();
$table->dateTimeTz('sent_at')->nullable();
$table->integer('fftn_id');
$table->integer('tftn_id');
$table->foreign('fftn_id')->references('id')->on('addresses');
$table->foreign('tftn_id')->references('id')->on('addresses');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('echomail_seenby', function (Blueprint $table) {
$table->dropForeign(['echomail_id']);
$table->dropColumn(['packet','echomail_id']);
});
DB::commit();
DB::statement("ALTER TABLE echomail_seenby RENAME TO address_echomail");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN mid TO echomail_id;");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN export_at TO export_date;");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN sent_at TO sent_date;");
Schema::dropIfExists('echomail_path');
Schema::dropIfExists('echomails');
Schema::dropIfExists('netmails');
}
}

View File

@ -261,7 +261,13 @@
}, },
}, },
legend: { legend: {
symbolWidth: 40 align: 'right',
layout: 'vertical',
symbolWidth: 40,
floating: false,
navigation: {
arrowSize: 10
},
}, },
plotOptions: { plotOptions: {
series: { series: {

View File

@ -111,5 +111,46 @@
</div> </div>
</div> </div>
</div> </div>
<div class="row">
<div class="col-12">
<h4>Info</h4>
<p>There are <strong class="highlight">{{ number_format($o->echomail->count()) }}</strong> messages in this area, and the oldest message in this area is <strong class="highlight">{{ $o->last_message ? $o->last_message->format('Y-m-d H:i:s') : 'None' }}</strong>.</p>
</div>
</div>
@can('admin',$o)
@if($o->addresses->count())
<div class="row">
<div class="col-12">
<h3>Exporting to the following adresses:</h3>
<table class="table monotable" id="system">
<thead>
<tr>
<th>System</th>
<th>Address</th>
<th>Last Session</th>
<th>Oldest Message</th>
<th class="text-right">Messages Waiting</th>
</tr>
</thead>
<tbody>
@foreach ($o->addresses as $ao)
<tr>
<td><a href="{{ url('ftn/system/addedit',[$ao->system_id]) }}">{{ $ao->system->full_name($ao) }}</a> @auth<span class="float-end"><small>@if($ao->session('sespass'))<sup>{{ $ao->session('default') ? '**' : '*' }}</sup>@elseif($ao->system->setup)<sup class="success">+</sup>@endif[{{ $ao->system_id }}]</small></span>@endauth</td>
<td>{{ $ao->ftn_3d }}</td>
<td>{{ $ao->system->last_session ? $ao->system->last_session->format('Y-m-d H:i') : '-' }}</td>
<td>{{ ($x=$o->waiting($ao))->count() ? $x->first()->datetime->format('Y-m-d H:i') : '-' }}</td>
<td class="text-right">{{ number_format($x->count()) }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
@endif
@endcan
</form> </form>
@endsection @endsection

View File

@ -217,7 +217,7 @@ use App\Http\Controllers\DomainController as DC;
<div class="accordion-item"> <div class="accordion-item">
<h3 class="accordion-header" id="routing" data-bs-toggle="collapse" data-bs-target="#collapse_routing" aria-expanded="false" aria-controls="collapse_routing">Mail Routing</h3> <h3 class="accordion-header" id="routing" data-bs-toggle="collapse" data-bs-target="#collapse_routing" aria-expanded="false" aria-controls="collapse_routing">Mail Routing</h3>
<div id="collapse_routing" class="accordion-collapse collapse {{ ($flash=='routing') ? 'show' : '' }}" aria-labelledby="addresses" data-bs-parent="#accordion_homepage"> <div id="collapse_routing" class="accordion-collapse collapse {{ ($flash=='routing') ? 'show' : '' }}" aria-labelledby="routing" data-bs-parent="#accordion_homepage">
<div class="accordion-body"> <div class="accordion-body">
<div class="row"> <div class="row">
<!-- Zone mail sent to --> <!-- Zone mail sent to -->
@ -288,6 +288,61 @@ use App\Http\Controllers\DomainController as DC;
</div> </div>
</div> </div>
</div> </div>
<!-- Waiting Mail -->
<div class="accordion-item">
<h3 class="accordion-header" id="mail" data-bs-toggle="collapse" data-bs-target="#collapse_mail" aria-expanded="false" aria-controls="collapse_mail">Mail Waiting</h3>
<div id="collapse_mail" class="accordion-collapse collapse {{ ($flash=='mail') ? 'show' : '' }}" aria-labelledby="mail" data-bs-parent="#accordion_homepage">
<div class="accordion-body">
<div class="row">
<!-- Netmail -->
<div class="col-6">
Netmails are waiting for these addresses:
<table class="table monotable">
<thead>
<tr>
<th>Address</th>
<th>Netmail</th>
</tr>
</thead>
<tbody>
@foreach ($o->addresses->sortBy('zone.zone_id') as $ao)
<tr>
<td>{{ $ao->ftn3d }}</td>
<td>{{ $ao->netmailWaiting()->count() }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
<!-- Echomail -->
<div class="col-6">
Echomail waiting for these addresses:
<table class="table monotable">
<thead>
<tr>
<th>Address</th>
<th>Echomail</th>
</tr>
</thead>
<tbody>
@foreach ($o->addresses->sortBy('zone.zone_id') as $ao)
<tr>
<td>{{ $ao->ftn3d }}</td>
<td>{{ $ao->echomailWaiting()->count() }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
@endif @endif
@else @else
@include('system.form-system') @include('system.form-system')

View File

@ -1,22 +1,22 @@
<div class="row"> <div class="row">
<div class="col-4">
TO: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->to) !!}</strong>
</div>
<div class="col-4"> <div class="col-4">
DATE: <strong class="highlight">{{ $msg->datetime->format('Y-m-d H:i:s') }}</strong> DATE: <strong class="highlight">{{ $msg->datetime->format('Y-m-d H:i:s') }}</strong>
</div> </div>
<div class="col-4">
MSGID: <strong class="highlight">{{ $msg->msgid }}</strong>@if($x=\App\Models\Echomail::where('reply',$msg->msgid)->count()) (<strong class="highlight">{{$x}}</strong> replies)@endif @if($msg->reply)<br>REPLY: <strong class="highlight">{{ $msg->reply }}</strong>@endif
</div>
</div> </div>
<div class="row pb-2"> <div class="row pt-1">
<div class="col-4"> <div class="col-4">
FROM: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->from) !!}</strong> (<strong class="highlight">{{ $msg->fftn->ftn }}</strong>) FROM: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->from) !!}</strong> (<strong class="highlight">{{ $msg->fftn->ftn }}</strong>)
</div> </div>
<div class="col-4"> <div class="col-4">
TO: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->to) !!}</strong> MSGID: <strong class="highlight">{{ $msg->msgid }}</strong>@if($x=\App\Models\Echomail::where('replyid',$msg->msgid)->count()) (<strong class="highlight">{{$x}}</strong> replies)@endif @if($msg->replyid)<br>REPLY: <strong class="highlight">{{ $msg->replyid }}</strong>@endif
</div> </div>
</div> </div>
<div class="row pb-2"> <div class="row pt-1 pb-2">
<div class="col-8"> <div class="col-8">
SUBJECT: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->subject) !!}</strong> SUBJECT: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->subject) !!}</strong>
</div> </div>