bbs/clrghouz
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Switchout DB to CockroachDB

Browse Source
This commit is contained in:
Deon George 2022-01-01 16:59:35 +11:00
parent afaa7d8bc7
commit 424d6ef39d
28 changed files with 1342 additions and 865 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.env.example
View File

@ -7,12 +7,16 @@ APP_DEBUG=true
LOG_CHANNEL=stack
LOG_LEVEL=info
DB_CONNECTION=pgsql
DB_HOST=postgres
DB_PORT=5432
DB_DATABASE=postgres
DB_USERNAME=postgres
DB_PASSWORD=password
DB_CONNECTION=cockroach
DB_HOST=cockroach
DB_PORT=26257
DB_DATABASE=clrghouz
DB_USERNAME=dev
DB_PASSWORD=
DB_SSLMODE=prefer
DB_SSLROOTCERT=/var/www/html/config/ssl/ca.crt
DB_SSLCERT=/var/www/html/config/ssl/client.crt
DB_SSLKEY=/var/www/html/config/ssl/client.key
DB_MONGO_HOST=mongo
DB_MONGO_USERNAME=mongo

12
app/Classes/FTN/Message.php
View File

@ -93,7 +93,7 @@ class Message extends FTNBase
private string $subject; // Message subject
private string $msgid; // MSG ID
private string $reply; // Reply ID
private string $replyid; // Reply ID
private string $echoarea; // FTS-0004.001
private string $intl; // Netmail details
@ -182,7 +182,7 @@ class Message extends FTNBase
$this->message = '';
$this->msgid = '';
$this->reply = '';
$this->replyid = '';
$this->echoarea = '';
$this->intl = '';
@ -372,7 +372,7 @@ class Message extends FTNBase
case 'echoarea':
case 'msgid':
case 'reply':
case 'replyid':
case 'message':
case 'message_src':
@ -412,7 +412,7 @@ class Message extends FTNBase
case 'subject':
case 'msgid':
case 'reply':
case 'replyid':
case 'echoarea':
case 'intl':
@ -486,8 +486,8 @@ class Message extends FTNBase
// Add some kludges
$return .= sprintf("\01MSGID: %s\r",$this->msgid);
if ($this->reply)
$return .= sprintf("\01REPLY: %s\r",$this->reply);
if ($this->replyid)
$return .= sprintf("\01REPLY: %s\r",$this->replyid);
foreach ($this->_kludge as $k=>$v) {
if ($x=$this->kludge->get($k))

3
app/Classes/FTN/Process/Echomail/Test.php
View File

@ -26,6 +26,7 @@ final class Test extends Process
private const testing = ['test','testing'];
// @todo add path and other kludges
public static function handle(Message $msg): bool
{
if ((strtolower($msg->user_to) !== 'all') || ! in_array(strtolower($msg->subject),self::testing))
@ -66,7 +67,7 @@ final class Test extends Process
$o->datetime = Carbon::now();
$o->tzoffset = $o->datetime->utcOffset();
$o->echoarea_id = $eo?->id;
$o->reply = $msg->msgid;
$o->replyid = $msg->msgid;
$o->fftn_id = $ftns->id;
$o->flags = Message::FLAG_LOCAL;

2
app/Classes/FTN/Process/Netmail/Ping.php
View File

@ -50,7 +50,7 @@ final class Ping extends Process
$o->datetime = Carbon::now();
$o->tzoffset = $o->datetime->utcOffset();
$o->reply = $msg->msgid;
$o->replyid = $msg->msgid;
$o->fftn_id = $ftns->id;
$o->tftn_id = ($x=$msg->fftn_o) ? $x->id : NULL;
$o->flags = Message::FLAG_LOCAL;

160
app/Console/Commands/ConvertMongo.php Normal file
View File

@ -0,0 +1,160 @@
<?php
namespace App\Console\Commands;
use App\Models\Echoarea;
use App\Models\Echomail;
use App\Models\Netmail;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Notification;
class ConvertMongo extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'convert:mongo {start=0}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Convert Mongo to Cockroach';
/**
* Execute the console command.
*
* @return mixed
* @throws \Exception
*/
public function handle()
{
/*
// Netmails
foreach (DB::connection('mongodb')->collection('netmails')->cursor() as $o) {
$o['mid'] = (string)$o['_id'];
foreach([
'reply'=>'replyid',
'packet'=>'sent_pkt',
] as $key => $newkey) {
if (array_key_exists($key,$o)) {
$o[$newkey] = $o[$key];
unset($o[$key]);
}
}
unset($o['_id'],$o['sent'],$o['reply']);
foreach (['created_at','updated_at','sent_at','datetime','deleted_at'] as $key) {
if (Arr::get($o,$key))
$o[$key] = Carbon::create($o[$key]->toDatetime());
}
if (! Arr::get($o,'datetime'))
$o['datetime'] = $o['created_at'];
$oo = Netmail::withTrashed()->where('mid',$o['mid'])->firstOrNew();
$oo->forceFill($o);
try {
$oo->save(['timestamps' => false]);
} catch (\Exception $e) {
$this->warn(sprintf('Netmail didnt move over: %s (%s)',$o['mid'],$e->getMessage()));
}
}
*/
/**/
// Echomail
$c = 0;
foreach (DB::connection('mongodb')->collection('echomails')->cursor() as $o) {
if (++$c < $this->argument('start'))
continue;
if (! ($c%100))
Log::debug(sprintf('Processed : %d Echomails',$c));
$o['mid'] = (string)$o['_id'];
foreach([
'reply'=>'replyid',
'rogue_seen'=>'rogue_seenby',
] as $key => $newkey) {
if (array_key_exists($key,$o)) {
$o[$newkey] = $o[$key];
unset($o[$key]);
}
}
$path = (array_key_exists('path',$o) && $o['path']) ? $o['path'] : NULL;
$seenby = (array_key_exists('seenby',$o) && $o['seenby']) ? $o['seenby'] : NULL;
$packet = (array_key_exists('packet',$o) && $o['packet']) ? $o['packet'] : NULL;
unset($o['_id'],$o['reply'],$o['path'],$o['seenby'],$o['toexport'],$o['sent_at'],$o['packet'],$o['sent']);
foreach (['created_at','updated_at','datetime','deleted_at'] as $key) {
if (Arr::get($o,$key))
$o[$key] = Carbon::create($o[$key]->toDatetime());
}
if (! Arr::get($o,'datetime'))
$o['datetime'] = $o['created_at'];
if (Arr::get($o,'echoarea') && ! Arr::get($o,'echoarea_id')) {
$ea = Echoarea::where('name',$o['echoarea'])->single();
$o['echoarea_id'] = $ea->id;
unset($o['echoarea']);
}
if (! $o['echoarea_id']) {
Log::error(sprintf('Echomail didnt move over: %s [%d] - has no echoarea_id',$o['mid'],$c));
continue;
}
if (Arr::get($o,'msg_src') && ! Arr::get($o,'msg_crc')) {
$o['msg_crc'] = 'x'.md5($o['msg_src']);
}
$oo = Echomail::withTrashed()->where('mid',$o['mid'])->firstOrNew();
$oo->forceFill($o);
$oo->set_path = $path ? array_filter($path) : [];
$oo->set_seenby = $seenby ? array_filter($seenby): [];
$oo->set_packet = $packet;
try {
$oo->save(['timestamps'=>FALSE]);
} catch (\Exception $e) {
Log::error(sprintf('Echomail didnt move over: %s [%d] (%s@%d|%s)',$o['mid'],$c,$e->getFile(),$e->getLine(),$e->getMessage()));
dd(['e'=>$e,'o'=>$o,'oo'=>$oo]);
}
DB::connection('mongodb')->collection('echomails')->delete($o['mid']);
}
/**/
// Update old MID seenby with proper ID
foreach (DB::connection('cockroach')->table('echomail_seenby')->whereNotNull('mid')->whereNull('echomail_id')->cursor() as $o)
{
$eo = Echomail::where('mid',$o->mid)->get();
if ($eo->count() && $eo->count() == 1) {
DB::update('UPDATE echomail_seenby set echomail_id = ?, mid=NULL where echomail_id IS NULL AND mid = ? ',[
$eo->first()->id,
$o->mid,
]);
} elseif ($eo->count() > 1) {
Log::error(sprintf('Echomail [%s] has more than 1 record [%d] - skipped',$o->mid,$eo->count()));
}
}
}
}

1
app/Http/Controllers/HomeController.php
View File

@ -121,6 +121,7 @@ class HomeController extends Controller
# Look for Messages
foreach (Echomail::select(['id','fftn_id','from'])
->where('msgid','like','%'.$request->query('term').'%')
->orWhere('replyid','like','%'.$request->query('term').'%')
->get() as $o)
{
$result->push(['id'=>$o->id,'name'=>sprintf('%s (%s)',$o->from,$o->fftn->ftn3d),'value'=>url('echomail/view',[$o->id]),'category'=>'Echomail']);

2
app/Jobs/MessageProcess.php
View File

@ -112,7 +112,7 @@ class MessageProcess implements ShouldQueue
$o->fftn_id = ($x=$this->msg->tftn_o) ? $x->id : NULL;
$o->tftn_id = ($x=$this->msg->fftn_o) ? $x->id : NULL;
$o->reply = $this->msg->msgid;
$o->replyid = $this->msg->msgid;
$o->msg = Process::format_msg($reply,$reject);
$o->tagline = 'Do you think it was fate which brought us together? Nah, bad luck :(';

70
app/Models/Address.php
View File

@ -11,7 +11,7 @@ use Illuminate\Support\Facades\DB;
use App\Classes\FTN\Packet;
use App\Http\Controllers\DomainController;
use App\Traits\{ScopeActive,UsePostgres};
use App\Traits\ScopeActive;
/**
* @todo Need to stop this from happening:
@ -24,7 +24,9 @@ use App\Traits\{ScopeActive,UsePostgres};
*/
class Address extends Model
{
use ScopeActive,SoftDeletes,UsePostgres;
use ScopeActive,SoftDeletes;
protected $with = ['zone'];
/* SCOPES */
@ -337,6 +339,21 @@ class Address extends Model
return ($o && $o->system->active) ? $o : NULL;
}
/**
* Netmail waiting to be sent to this system
*
* @return Collection
*/
public function echomailWaiting(): Collection
{
return Echomail::select(['echomails.*'])
->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
->where('echomail_seenby.address_id',$this->id)
->whereNull('echomail_seenby.sent_at')
->whereNotNull('echomail_seenby.export_at')
->get();
}
/**
* Get echomail for this node
*
@ -346,23 +363,17 @@ class Address extends Model
{
$pkt = NULL;
$echomail = DB::table('address_echomail')
->select('echomail_id')
->where('address_id',$this->id)
->whereNull('sent_date')
->get();
if (($x=Echomail::select('*')
->whereIn('_id',$echomail->pluck('echomail_id')))
if (($x=$this->echomailWaiting())
->count())
{
$pkt = $this->getPacket($x->get());
$pkt = $this->getPacket($x);
DB::table('address_echomail')
->whereIn('echomail_id',$echomail->pluck('echomail_id'))
DB::table('echomail_seenby')
->whereIn('echomail_id',$x->pluck('id'))
->where('address_id',$this->id)
->whereNull('sent_date')
->update(['sent_date'=>Carbon::now()]);
->whereNull('sent_at')
->whereNotNull('echomail_seenby.export_at')
->update(['sent_at'=>Carbon::now(),'packet'=>$pkt->name]);
}
return $pkt;
@ -375,18 +386,19 @@ class Address extends Model
*/
public function getNetmail(): ?Packet
{
if (($x=Netmail::whereIn('tftn_id',(($x=$this->children) ? $x->pluck('id') : collect())->push($this->id))
->where(function($q) {
return $q->whereNull('sent')
->orWhere('sent',FALSE);
}))
->whereNull('local')
$pkt = NULL;
if (($x=$this->netmailWaiting())
->count())
{
return $this->getPacket($x->get());
$pkt = $this->getPacket($x);
DB::table('netmails')
->whereIn('id',$x->pluck('id'))
->update(['sent_at'=>Carbon::now(),'sent_pkt'=>$pkt->name]);
}
return NULL;
return $pkt;
}
/**
@ -411,6 +423,18 @@ class Address extends Model
return $o;
}
/**
* Netmail waiting to be sent to this system
*
* @return Collection
*/
public function netmailWaiting(): Collection
{
return Netmail::whereIn('tftn_id',(($x=$this->children) ? $x->pluck('id') : collect())->push($this->id))
->whereNull('sent_at')
->get();
}
/**
* Parse a string and split it out as an FTN array
*

42
app/Models/Domain.php
View File

@ -7,13 +7,15 @@ use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use MongoDB\BSON\UTCDateTime;
use App\Traits\ScopeActive;
use App\Traits\{QueryCacheableConfig,ScopeActive};
class Domain extends Model
{
use HasFactory,ScopeActive;
use HasFactory,ScopeActive,QueryCacheableConfig;
private const CACHE_TIME = 3600;
private const STATS_MONTHS = 6;
@ -65,17 +67,19 @@ class Domain extends Model
$key = sprintf('%s_%d','daily_area_stats',$this->id);
return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() {
$where = [
'echoarea_id'=>$this->echoareas->pluck('id')->toArray(),
'datetime' => ['$gte',new UTCDateTime(Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())],
];
Cache::forget($key);
return Cache::remember($key,self::CACHE_TIME,function() {
$gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime";
$echostats = Echomail::countGroupBy(['datetime',['datetime'=>'%Y-%m-%d']],$where);
$echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')])
->whereIn('id',$this->echoareas->pluck('id')->toArray())
->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())
->groupBy('datetime')
->orderBy('datetime')
->get();
return $echostats
->sortBy(function($item) { return $item->id->datetime; })
->map(function($item) { return ['x'=>Carbon::createFromFormat('Y-m-d',$item->id->datetime)->timestamp*1000,'y'=>$item->count]; })
->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; })
->values();
});
}
@ -87,17 +91,19 @@ class Domain extends Model
$key = sprintf('%s_%d-%d','daily_echoarea_stats',$this->id,$o->id);
return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() use ($o) {
$where = [
'echoarea_id'=>[$o->id],
'datetime' => ['$gte',new UTCDateTime(Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())],
];
Cache::forget($key);
return Cache::remember($key,self::CACHE_TIME,function() use ($o) {
$gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime";
$echostats = Echomail::countGroupBy(['datetime',['datetime'=>'%Y-%m-%d']],$where);
$echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')])
->whereIn('echoarea_id',[$o->id])
->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())
->groupBy('datetime')
->orderBy('datetime')
->get();
return $echostats
->sortBy(function($item) { return $item->id->datetime; })
->map(function($item) { return ['x'=>Carbon::createFromFormat('Y-m-d',$item->id->datetime)->timestamp*1000,'y'=>$item->count]; })
->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; })
->values();
});
}

36
app/Models/Echoarea.php
View File

@ -3,14 +3,16 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Facades\Cache;
use App\Traits\{ScopeActive,UsePostgres};
use App\Traits\ScopeActive;
class Echoarea extends Model
{
use SoftDeletes,ScopeActive,UsePostgres;
use SoftDeletes,ScopeActive;
private const CACHE_TIME = 3600;
@ -28,28 +30,24 @@ class Echoarea extends Model
public function echomail()
{
return Echomail::select('*')
->where('echoarea_id',$this->id);
return $this->hasMany(Echomail::class)
->orderBy('datetime','ASC');
}
/* ATTRIBUTES */
public function getLastMessageAttribute(): ?Carbon
{
$key = sprintf('%s_%d','echo_last_message',$this->id);
return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() {
return ($x=$this->echomail()->orderBy('datetime','DESC')->first()) ? $x->datetime : NULL;
});
return $this->echomail?->last()->datetime;
}
/* METHODS */
public function messages_count(int $period): int
{
$key = sprintf('%s_%d_%d','echo_mesages_count',$this->id,$period);
$key = sprintf('%s_%d_%d','echo_messages_count',$this->id,$period);
return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() use ($period) {
return Cache::remember($key,self::CACHE_TIME,function() use ($period) {
switch ($period) {
case 1: // day
return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subDay())->count();
@ -62,4 +60,20 @@ class Echoarea extends Model
}
});
}
/**
* Number of messages waiting for address
*
* @param Address $ao
* @return Collection
*/
public function waiting(Address $ao): Collection
{
return $this->echomail()
->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
->whereNull('sent_at')
->whereNotNull('export_at')
->where('address_id',$ao->id)
->get();
}
}

122
app/Models/Echomail.php
View File

@ -3,25 +3,29 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Jenssegers\Mongodb\Eloquent\Model;
use Jenssegers\Mongodb\Eloquent\SoftDeletes;
use App\Classes\FTN\Message;
use App\Interfaces\Packet;
use App\Traits\{EncodeUTF8,MsgID,UseMongo};
use App\Traits\{EncodeUTF8,MsgID};
final class Echomail extends Model implements Packet
{
use SoftDeletes,MsgID,UseMongo,EncodeUTF8;
use SoftDeletes,EncodeUTF8,MsgID;
private const LOGKEY = 'ME-';
private array $set_seenby = [];
private array $set_path = [];
private ?string $set_packet = NULL;
protected $collection = FALSE;
protected $casts = [ 'kludges' => 'json' ];
protected $casts = [
'kludges' => 'json',
'rogue_seenby' => 'json',
'rogue_path' => 'json',
];
private const cast_utf8 = [
'to',
@ -35,6 +39,20 @@ final class Echomail extends Model implements Packet
protected $dates = ['datetime'];
public function __set($key, $value)
{
switch ($key) {
case 'set_path':
case 'set_packet':
case 'set_seenby':
$this->{$key} = $value;
break;
default:
parent::__set($key,$value);
}
}
public static function boot()
{
parent::boot();
@ -46,29 +64,43 @@ final class Echomail extends Model implements Packet
return;
}
// See if we need to export this message.
$exportto = $model->echoarea->addresses->pluck('system')->diff($model->seenby->pluck('system'));
$export_ao = collect();
foreach ($model->echoarea->domain->zones as $zo) {
foreach ($exportto as $so) {
$export_ao = $export_ao->merge($so->match($zo));
}
}
// Add to export
foreach ($export_ao as $ao) {
Log::info(sprintf('%s:- Exporting message [%s] to [%s]',self::LOGKEY,$model->id,$ao->ftn));
DB::table('address_echomail')->insert([
'address_id'=>$ao->id,
'echomail_id'=>$model->id,
'export_date'=>Carbon::now()
// Save the seenby
foreach ($model->set_seenby as $aoid) {
DB::insert('INSERT INTO echomail_seenby (echomail_id,address_id,packet) VALUES (?,?,?)',[
$model->id,
$aoid,
$model->set_packet,
]);
}
$model->seenby = $model->seenby->merge($export_ao)->pluck('id')->toArray();
$model->save();
// Save the Path
$ppoid = NULL;
foreach ($model->set_path as $aoid) {
$po = DB::select('INSERT INTO echomail_path (echomail_id,address_id,parent_id) VALUES (?,?,?) RETURNING id',[
$model->id,
$aoid,
$ppoid,
]);
$ppoid = $po[0]->id;
}
// See if we need to export this message.
$exportto = $model->echoarea->addresses->pluck('id')->diff($model->set_seenby);
if ($exportto->count()) {
Log::debug(sprintf('%s:- Exporting message [%s] to [%s]',self::LOGKEY,$model->id,$exportto->join(',')));
// Save the seenby for the exported systems
$export_at = Carbon::now();
foreach ($exportto as $aoid) {
DB::insert('INSERT INTO echomail_seenby (echomail_id,address_id,export_at) VALUES (?,?,?)',[
$model->id,
$aoid,
$export_at,
]);
}
}
});
}
@ -81,35 +113,19 @@ final class Echomail extends Model implements Packet
public function fftn()
{
return $this
->setConnection('pgsql')
->belongsTo(Address::class)
return $this->belongsTo(Address::class)
->withTrashed();
}
/* ATTRIBUTES */
public function getKludgesAttribute(?string $value): Collection
public function seenby()
{
return collect($this->castAttribute('kludges',$value));
return $this->belongsToMany(Address::class,'echomail_seenby')
->ftnOrder();
}
public function getPathAttribute(?array $value): Collection
public function path()
{
if (is_null($value))
return collect();
return Address::whereIn('id',$value)
->orderBy(DB::raw(sprintf("position (id::text in '(%s)')",join(',',$value))))
->get();
}
public function getSeenByAttribute(?array $value): Collection
{
if (is_null($value))
return collect();
return Address::whereIn('id',$value)->get();
return $this->belongsToMany(Address::class,'echomail_path');
}
/* METHODS */
@ -149,11 +165,11 @@ final class Echomail extends Model implements Packet
if ($this->kludges)
$o->kludge = $this->kludges;
$o->kludge->put('mid',$this->id);
$o->kludge->put('dbid',$this->id);
$o->msgid = $this->msgid;
if ($this->reply)
$o->reply = $this->reply;
if ($this->replyid)
$o->replyid = $this->replyid;
$o->message = $this->msg;

12
app/Models/Netmail.php
View File

@ -3,19 +3,19 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Facades\Log;
use Jenssegers\Mongodb\Eloquent\Model;
use Jenssegers\Mongodb\Eloquent\SoftDeletes;
use App\Classes\FTN\Message;
use App\Interfaces\Packet;
use App\Traits\{EncodeUTF8,UseMongo};
use App\Traits\EncodeUTF8;
final class Netmail extends Model implements Packet
{
private const LOGKEY = 'MN-';
use SoftDeletes,UseMongo,EncodeUTF8;
use SoftDeletes,EncodeUTF8;
private const cast_utf8 = [
'to',
@ -80,8 +80,8 @@ final class Netmail extends Model implements Packet
$o->flags = $this->flags;
$o->msgid = sprintf('%s %08x',$this->fftn->ftn3d,crc32($this->id));
if ($this->reply)
$o->reply = $this->reply;
if ($this->replyid)
$o->replyid = $this->replyid;
$o->message = $this->msg;
$o->tagline = $this->tagline;

8
app/Models/System.php
View File

@ -8,10 +8,11 @@ use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Auth;
use App\Http\Controllers\DomainController;
use App\Traits\QueryCacheableConfig;
class System extends Model
{
use HasFactory;
use HasFactory,QueryCacheableConfig;
protected $dates = ['last_session'];
@ -37,10 +38,7 @@ class System extends Model
public function addresses()
{
return $this->hasMany(Address::class)
->orderBy('region_id')
->orderBy('host_id')
->orderBy('node_id')
->orderBy('point_id');
->FTNorder();
}
/**

4
app/Models/Zone.php
View File

@ -4,11 +4,11 @@ namespace App\Models;
use Illuminate\Database\Eloquent\Model;
use App\Traits\ScopeActive;
use App\Traits\{QueryCacheableConfig,ScopeActive};
class Zone extends Model
{
use ScopeActive;
use ScopeActive,QueryCacheableConfig;
/* SCOPES */

4
app/Traits/MsgID.php
View File

@ -7,13 +7,15 @@ namespace App\Traits;
use Illuminate\Support\Arr;
use App\Classes\FTN\Message;
use App\Models\Setup;
trait MsgID
{
public function save(array $options = [])
{
if (! $this->exists) {
// Only create a MSGID for locally generated conetnt
if ((! $this->exists) && ($this->flags & Message::FLAG_LOCAL)) {
$ftns = Setup::findOrFail(config('app.id'))->system->match($this->fftn->zone)->first();
if (is_null(Arr::get($this->attributes,'msgid')))

17
app/Traits/QueryCacheableConfig.php Normal file
View File

@ -0,0 +1,17 @@
<?php
/**
* Set defaults of QueryCacheable
*/
namespace App\Traits;
use Rennokki\QueryCache\Traits\QueryCacheable;
trait QueryCacheableConfig
{
use QueryCacheable;
public $cacheFor = 900; // cache time, in seconds
protected static $flushCacheOnUpdate = TRUE;
public $cacheDriver = 'memcached';
}

104
app/Traits/UseMongo.php
View File

@ -1,104 +0,0 @@
<?php
/**
* Models that store data in Mongo
*/
namespace App\Traits;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Support\Arr;
trait UseMongo
{
/**
* Resolve a connection instance.
* We need to do this, because our relations are in pgsql and somehow loading a relation changes this models
* connection information. Using protected $connection is not enough.
*
* @param string|null $connection
*/
public static function resolveConnection($connection = null)
{
return static::$resolver->connection('mongodb');
}
/* ATTRIBUTES */
public function getMsgAttribute($value): string
{
return utf8_decode($value);
}
public function setMsgAttribute($value): void
{
$this->attributes['msg'] = utf8_encode($value);
}
public function getSubjectAttribute($value): string
{
return utf8_decode($value);
}
public function setSubjectAttribute($value): void
{
$this->attributes['subject'] = utf8_encode($value);
}
/* METHODS */
public static function countGroupBy(array $fields,array $where=[]): Collection
{
$query = collect();
if (count($where)) {
$where_condition = [];
foreach ($where as $key => $values) {
if (! is_array($values))
throw new \Exception('Where values must be an array.');
switch ($x=Arr::get($values,0)) {
case '$gt':
case '$gte':
$where_condition[$key] = [ $x => Arr::get($values,1)];
break;
case '$in':
default:
$where_condition[$key] = ['$in' => $values];
}
}
$query->push([
'$match' => [ '$and'=> [$where_condition]]
]);
}
$gb = collect();
foreach ($fields as $field)
if (is_array($field)) {
foreach ($field as $k=>$v) {
$gb->put('datetime',['$dateToString'=>['format'=>$v,'date'=>'$'.$k]]);
}
} else {
$gb->put($field,'$'.$field);
}
$query->push([
'$group' => [
'_id' => $gb->toArray(),
'count' => ['$sum' => 1]
]
]);
return (new self)
->groupBy($field)
->raw(function($collection) use ($query) {
return $collection->aggregate(
$query->toArray()
);
});
}
}

21
app/Traits/UsePostgres.php
View File

@ -1,21 +0,0 @@
<?php
/**
* Models that store data in Postgres
*/
namespace App\Traits;
trait UsePostgres
{
/**
* Resolve a connection instance.
* We need to do this, because our relations are in pgsql and somehow loading a relation changes this models
* connection information. Using protected $connection is not enough.
*
* @param string|null $connection
*/
public static function resolveConnection($connection = null)
{
return static::$resolver->connection('pgsql');
}
}

7
composer.json
View File

@ -17,6 +17,7 @@
"laravel/framework": "^8.0",
"laravel/passport": "^10.1",
"laravel/ui": "^3.2",
"leenooks/cockroachdb-laravel": "^0.1.0",
"rennokki/laravel-eloquent-query-cache": "^3.1",
"repat/laravel-job-models": "^0.5.1"
},
@ -54,6 +55,12 @@
"Tests\\": "tests/"
}
},
"repositories": [
{
"type": "vcs",
"url": "https://github.com/leenooks/cockroachdb-laravel"
}
],
"minimum-stability": "dev",
"prefer-stable": true,
"scripts": {

1265
composer.lock generated
View File

File diff suppressed because it is too large Load Diff

39
config/database.php
View File

@ -67,7 +67,10 @@ return [
'prefix' => '',
'prefix_indexes' => true,
'schema' => 'public',
'sslmode' => 'prefer',
'sslmode' => 'disable', //depends on your security level https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
'sslrootcert' => env('DB_SSLROOTCERT', 'config/ssl/ca.crt'),
'sslcert' => env('DB_SSLCERT', 'config/ssl/client.crt'),
'sslkey' => env('DB_SSLKEY', 'config/ssl/client.key'),
],
'sqlsrv' => [
@ -92,29 +95,29 @@ return [
'charset' => 'utf8',
'prefix' => '',
'schema' => 'public',
'sslmode' => 'prefer',
'sslmode' => env('DB_SSLMODE', 'prefer'),
// Only set these keys if you want to run en secure mode
// otherwise you can them out of the configuration array
#'sslcert' => env('DB_SSLCERT', 'client.crt'),
#'sslkey' => env('DB_SSLKEY', 'client.key'),
#'sslrootcert' => env('DB_SSLROOTCERT', 'ca.crt'),
'sslcert' => env('DB_SSLCERT', 'config/ssl/client.crt'),
'sslkey' => env('DB_SSLKEY', 'config/ssl/client.key'),
'sslrootcert' => env('DB_SSLROOTCERT', 'config/ssl/ca.crt'),
],
'mongodb' => [
'driver' => 'mongodb',
'host' => env('DB_MONGO_HOST', '127.0.0.1'),
'port' => env('DB_MONGO_PORT', 27017),
'database' => env('DB_MONGO_DATABASE', 'fido'),
'username' => env('DB_MONGO_USERNAME', 'mongo'),
'password' => env('DB_MONGO_PASSWORD', 'password'),
'options' => [
// here you can pass more settings to the Mongo Driver Manager
// see https://www.php.net/manual/en/mongodb-driver-manager.construct.php under "Uri Options" for a list of complete parameters that you can use
'mongodb' => [
'driver' => 'mongodb',
'host' => env('DB_MONGO_HOST', '127.0.0.1'),
'port' => env('DB_MONGO_PORT', 27017),
'database' => env('DB_MONGO_DATABASE', 'fido'),
'username' => env('DB_MONGO_USERNAME', 'mongo'),
'password' => env('DB_MONGO_PASSWORD', 'password'),
'options' => [
// here you can pass more settings to the Mongo Driver Manager
// see https://www.php.net/manual/en/mongodb-driver-manager.construct.php under "Uri Options" for a list of complete parameters that you can use
'database' => env('DB_AUTHENTICATION_DATABASE', 'admin'), // required with Mongo 3+
],
],
'database' => env('DB_AUTHENTICATION_DATABASE', 'admin'), // required with Mongo 3+
],
],
],
/*

3
database/migrations/2021_06_18_112022_update_zones.php
View File

@ -16,11 +16,10 @@ class UpdateZones extends Migration
DB::statement('ALTER TABLE zones ALTER COLUMN domain_id SET NOT NULL');
Schema::table('zones', function (Blueprint $table) {
$table->integer('system_id');
$table->dropColumn(['description','ipv4','ipv4_mask','ipv6','ipv6_mask','zt_id']);
$table->string('ztid')->nullable();
$table->dropUnique(['domain_id']);
$table->integer('system_id');
$table->foreign('system_id')->references('id')->on('systems');
});
}

2
database/migrations/2021_06_21_121910_add_system_to_setups.php
View File

@ -14,8 +14,8 @@ class AddSystemToSetups extends Migration
public function up()
{
Schema::table('setups', function (Blueprint $table) {
$table->integer('options');
$table->integer('system_id');
$table->integer('options');
$table->foreign('system_id')->references('id')->on('systems');
});
}

137
database/migrations/2022_01_01_235631_create_mail.php Normal file
View File

@ -0,0 +1,137 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateMail extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('echomails', function (Blueprint $table) {
$table->id();
$table->timestamps();
$table->softDeletes();
$table->string('mid')->nullable();
$table->string('to',128);
$table->string('from',128);
$table->string('subject',256);
$table->dateTimeTz('datetime');
$table->tinyInteger('tzoffset');
$table->string('msgid')->nullable();
$table->string('replyid')->nullable();
$table->text('msg');
$table->string('msg_src')->nullable();
$table->string('msg_crc')->nullable();
$table->string('tagline')->nullable();
$table->string('tearline')->nullable();
$table->string('origin')->nullable();
$table->tinyInteger('flags')->nullable();
$table->jsonb('rogue_path')->nullable();
$table->jsonb('rogue_seenby')->nullable();
$table->jsonb('kludges')->nullable();
$table->integer('echoarea_id');
$table->foreign('echoarea_id')->references('id')->on('echoareas');
$table->integer('fftn_id');
$table->foreign('fftn_id')->references('id')->on('addresses');
});
DB::statement("ALTER TABLE address_echomail RENAME COLUMN echomail_id TO mid");
DB::statement("ALTER TABLE address_echomail ALTER COLUMN mid DROP NOT NULL");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN export_date TO export_at");
DB::statement("ALTER TABLE address_echomail ALTER COLUMN export_at DROP NOT NULL");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN sent_date TO sent_at");
DB::statement("ALTER TABLE address_echomail RENAME TO echomail_seenby");
Schema::table('echomail_seenby', function (Blueprint $table) {
$table->integer('echomail_id')->nullable()->first();
$table->string('packet')->nullable();
$table->foreign('echomail_id')->references('id')->on('echomails');
});
Schema::create('echomail_path', function (Blueprint $table) {
$table->id();
$table->integer('address_id');
$table->foreign('address_id')->references('id')->on('addresses');
$table->unique(['id','echomail_id']);
$table->unique(['echomail_id','address_id','parent_id']);
$table->integer('parent_id')->nullable();
$table->foreign(['parent_id','echomail_id'])->references(['id','echomail_id'])->on('echomail_path');
$table->integer('echomail_id');
$table->foreign('echomail_id')->references('id')->on('echomails');
});
Schema::create('netmails', function (Blueprint $table) {
$table->id();
$table->timestamps();
$table->softDeletes();
$table->string('mid')->nullable();
$table->string('to',128);
$table->string('from',128);
$table->string('subject',256);
$table->dateTimeTz('datetime');
$table->tinyInteger('tzoffset')->nullable();
$table->tinyInteger('flags')->nullable();
$table->tinyInteger('cost')->nullable();
$table->string('msgid')->nullable();
$table->string('replyid')->nullable();
$table->text('msg');
$table->text('msg_src')->nullable();
$table->integer('msg_crc')->nullable();
$table->string('tagline')->nullable();
$table->string('tearline')->nullable();
$table->boolean('local')->default(FALSE);
$table->string('recv_pkt')->nullable();
$table->string('sent_pkt')->nullable();
$table->dateTimeTz('sent_at')->nullable();
$table->integer('fftn_id');
$table->integer('tftn_id');
$table->foreign('fftn_id')->references('id')->on('addresses');
$table->foreign('tftn_id')->references('id')->on('addresses');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('echomail_seenby', function (Blueprint $table) {
$table->dropForeign(['echomail_id']);
$table->dropColumn(['packet','echomail_id']);
});
DB::commit();
DB::statement("ALTER TABLE echomail_seenby RENAME TO address_echomail");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN mid TO echomail_id;");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN export_at TO export_date;");
DB::statement("ALTER TABLE address_echomail RENAME COLUMN sent_at TO sent_date;");
Schema::dropIfExists('echomail_path');
Schema::dropIfExists('echomails');
Schema::dropIfExists('netmails');
}
}

8
resources/views/domain/view.blade.php
View File

@ -261,7 +261,13 @@
},
},
legend: {
symbolWidth: 40
align: 'right',
layout: 'vertical',
symbolWidth: 40,
floating: false,
navigation: {
arrowSize: 10
},
},
plotOptions: {
series: {

41
resources/views/echoarea/addedit.blade.php
View File

@ -111,5 +111,46 @@
</div>
</div>
</div>
<div class="row">
<div class="col-12">
<h4>Info</h4>
<p>There are <strong class="highlight">{{ number_format($o->echomail->count()) }}</strong> messages in this area, and the oldest message in this area is <strong class="highlight">{{ $o->last_message ? $o->last_message->format('Y-m-d H:i:s') : 'None' }}</strong>.</p>
</div>
</div>
@can('admin',$o)
@if($o->addresses->count())
<div class="row">
<div class="col-12">
<h3>Exporting to the following adresses:</h3>
<table class="table monotable" id="system">
<thead>
<tr>
<th>System</th>
<th>Address</th>
<th>Last Session</th>
<th>Oldest Message</th>
<th class="text-right">Messages Waiting</th>
</tr>
</thead>
<tbody>
@foreach ($o->addresses as $ao)
<tr>
<td><a href="{{ url('ftn/system/addedit',[$ao->system_id]) }}">{{ $ao->system->full_name($ao) }}</a> @auth<span class="float-end"><small>@if($ao->session('sespass'))<sup>{{ $ao->session('default') ? '**' : '*' }}</sup>@elseif($ao->system->setup)<sup class="success">+</sup>@endif[{{ $ao->system_id }}]</small></span>@endauth</td>
<td>{{ $ao->ftn_3d }}</td>
<td>{{ $ao->system->last_session ? $ao->system->last_session->format('Y-m-d H:i') : '-' }}</td>
<td>{{ ($x=$o->waiting($ao))->count() ? $x->first()->datetime->format('Y-m-d H:i') : '-' }}</td>
<td class="text-right">{{ number_format($x->count()) }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
@endif
@endcan
</form>
@endsection

57
resources/views/system/addedit.blade.php
View File

@ -217,7 +217,7 @@ use App\Http\Controllers\DomainController as DC;
<div class="accordion-item">
<h3 class="accordion-header" id="routing" data-bs-toggle="collapse" data-bs-target="#collapse_routing" aria-expanded="false" aria-controls="collapse_routing">Mail Routing</h3>
<div id="collapse_routing" class="accordion-collapse collapse {{ ($flash=='routing') ? 'show' : '' }}" aria-labelledby="addresses" data-bs-parent="#accordion_homepage">
<div id="collapse_routing" class="accordion-collapse collapse {{ ($flash=='routing') ? 'show' : '' }}" aria-labelledby="routing" data-bs-parent="#accordion_homepage">
<div class="accordion-body">
<div class="row">
<!-- Zone mail sent to -->
@ -288,6 +288,61 @@ use App\Http\Controllers\DomainController as DC;
</div>
</div>
</div>
<!-- Waiting Mail -->
<div class="accordion-item">
<h3 class="accordion-header" id="mail" data-bs-toggle="collapse" data-bs-target="#collapse_mail" aria-expanded="false" aria-controls="collapse_mail">Mail Waiting</h3>
<div id="collapse_mail" class="accordion-collapse collapse {{ ($flash=='mail') ? 'show' : '' }}" aria-labelledby="mail" data-bs-parent="#accordion_homepage">
<div class="accordion-body">
<div class="row">
<!-- Netmail -->
<div class="col-6">
Netmails are waiting for these addresses:
<table class="table monotable">
<thead>
<tr>
<th>Address</th>
<th>Netmail</th>
</tr>
</thead>
<tbody>
@foreach ($o->addresses->sortBy('zone.zone_id') as $ao)
<tr>
<td>{{ $ao->ftn3d }}</td>
<td>{{ $ao->netmailWaiting()->count() }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
<!-- Echomail -->
<div class="col-6">
Echomail waiting for these addresses:
<table class="table monotable">
<thead>
<tr>
<th>Address</th>
<th>Echomail</th>
</tr>
</thead>
<tbody>
@foreach ($o->addresses->sortBy('zone.zone_id') as $ao)
<tr>
<td>{{ $ao->ftn3d }}</td>
<td>{{ $ao->echomailWaiting()->count() }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
@endif
@else
@include('system.form-system')

12
resources/views/widgets/message.blade.php
View File

@ -1,22 +1,22 @@
<div class="row">
<div class="col-4">
TO: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->to) !!}</strong>
</div>
<div class="col-4">
DATE: <strong class="highlight">{{ $msg->datetime->format('Y-m-d H:i:s') }}</strong>
</div>
<div class="col-4">
MSGID: <strong class="highlight">{{ $msg->msgid }}</strong>@if($x=\App\Models\Echomail::where('reply',$msg->msgid)->count()) (<strong class="highlight">{{$x}}</strong> replies)@endif @if($msg->reply)<br>REPLY: <strong class="highlight">{{ $msg->reply }}</strong>@endif
</div>
</div>
<div class="row pb-2">
<div class="row pt-1">
<div class="col-4">
FROM: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->from) !!}</strong> (<strong class="highlight">{{ $msg->fftn->ftn }}</strong>)
</div>
<div class="col-4">
TO: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->to) !!}</strong>
MSGID: <strong class="highlight">{{ $msg->msgid }}</strong>@if($x=\App\Models\Echomail::where('replyid',$msg->msgid)->count()) (<strong class="highlight">{{$x}}</strong> replies)@endif @if($msg->replyid)<br>REPLY: <strong class="highlight">{{ $msg->replyid }}</strong>@endif
</div>
</div>
<div class="row pb-2">
<div class="row pt-1 pb-2">
<div class="col-8">
SUBJECT: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->subject) !!}</strong>
</div>