Change the way we figure out zones in packets, some packet testing, fix Echomail import

This commit is contained in:
Deon George 2021-08-29 23:58:12 +10:00
parent 271f066667
commit 9fb6d191d0
11 changed files with 218 additions and 48 deletions

View File

@ -2,11 +2,11 @@
namespace App\Classes; namespace App\Classes;
use App\Models\{Address,Domain}; use App\Models\{Address,Zone};
abstract class FTN abstract class FTN
{ {
protected ?Domain $domain; // Domain the packet is from protected ?Zone $zone; // Zone the packet is from
public function __get($key) public function __get($key)
{ {
@ -17,7 +17,7 @@ abstract class FTN
$this->fn, $this->fn,
$this->ff, $this->ff,
$this->fp, $this->fp,
).($this->domain ? sprintf('@%s',$this->domain->name) : ''); ).($this->zone ? sprintf('@%s',$this->zone->domain->name) : '');
case 'tftn': case 'tftn':
return sprintf('%d:%d/%d.%d', return sprintf('%d:%d/%d.%d',
@ -25,7 +25,7 @@ abstract class FTN
$this->tn, $this->tn,
$this->tf, $this->tf,
$this->tp, $this->tp,
).($this->domain ? sprintf('@%s',$this->domain->name) : ''); ).($this->zone ? sprintf('@%s',$this->zone->domain->name) : '');
case 'fftn_o': case 'fftn_o':
return Address::findFTN($this->fftn); return Address::findFTN($this->fftn);

View File

@ -10,7 +10,7 @@ use Illuminate\Support\Facades\Validator;
use Illuminate\Validation\Validator as ValidatorResult; use Illuminate\Validation\Validator as ValidatorResult;
use App\Classes\FTN as FTNBase; use App\Classes\FTN as FTNBase;
use App\Models\{Address,Domain}; use App\Models\{Address,Zone};
use App\Rules\TwoByteInteger; use App\Rules\TwoByteInteger;
use App\Traits\EncodeUTF8; use App\Traits\EncodeUTF8;
@ -163,9 +163,9 @@ class Message extends FTNBase
0xfc => 0x207f, 0xfd => 0x00b2, 0xfe => 0x25a0, 0xff => 0x00a0, 0xfc => 0x207f, 0xfd => 0x00b2, 0xfe => 0x25a0, 0xff => 0x00a0,
]; ];
public function __construct(Domain $domain=NULL) public function __construct(Zone $zone=NULL)
{ {
$this->domain = $domain; $this->zone = $zone;
$this->header = []; $this->header = [];
$this->kludge = collect(); $this->kludge = collect();
@ -204,9 +204,9 @@ class Message extends FTNBase
* @return Message * @return Message
* @throws InvalidPacketException * @throws InvalidPacketException
*/ */
public static function parseMessage(string $msg,Domain $domain=NULL): self public static function parseMessage(string $msg,Zone $zone=NULL): self
{ {
$o = new self($domain); $o = new self($zone);
try { try {
$o->header = unpack(self::unpackheader(self::header),substr($msg,0,self::HEADER_LEN)); $o->header = unpack(self::unpackheader(self::header),substr($msg,0,self::HEADER_LEN));
@ -246,7 +246,7 @@ class Message extends FTNBase
$o->unpackMessage(substr($msg,self::HEADER_LEN+$ptr)); $o->unpackMessage(substr($msg,self::HEADER_LEN+$ptr));
if (($x=$o->validate($domain))->fails()) { if (($x=$o->validate())->fails()) {
Log::debug('Message fails validation',['result'=>$x->errors()]); Log::debug('Message fails validation',['result'=>$x->errors()]);
//throw new \Exception('Message validation fails:'.join(' ',$x->errors()->all())); //throw new \Exception('Message validation fails:'.join(' ',$x->errors()->all()));
} }
@ -651,7 +651,7 @@ class Message extends FTNBase
* Extract information out of the message text. * Extract information out of the message text.
* *
* @param string $message * @param string $message
* @throws InvalidPacketException * @throws \Exception
*/ */
public function unpackMessage(string $message): void public function unpackMessage(string $message): void
{ {
@ -690,11 +690,13 @@ class Message extends FTNBase
preg_match('/^.*\((.*)\)$/',$this->origin,$matches); preg_match('/^.*\((.*)\)$/',$this->origin,$matches);
// Double check we have an address in the origin line // Double check we have an address in the origin line
if (! Arr::get($matches,1)) if (! Arr::get($matches,1)) {
throw new InvalidPacketException('No address in Origin?'); Log::error(sprintf('%s:! Origin line doesnt have an address',self::LOGKEY));
} else {
// Double check, our src and origin match // Double check, our src and origin match
$this->src = Address::parseFTN($matches[1]); $this->src = Address::parseFTN($matches[1]);
}
// We'll double check our FTN // We'll double check our FTN
if ($this->isNetmail() && (($this->src['n'] !== $this->fn) || ($this->src['f'] !== $this->ff))) { if ($this->isNetmail() && (($this->src['n'] !== $this->fn) || ($this->src['f'] !== $this->ff))) {
@ -758,6 +760,9 @@ class Message extends FTNBase
elseif ($t = $this->kludge('PATH: ',$v)) elseif ($t = $this->kludge('PATH: ',$v))
$this->path->push($t); $this->path->push($t);
elseif ($t = $this->kludge('SEEN-BY: ',$v))
$this->seenby->push($t);
// To Point: <SOH>TOPT <point number><CR> // To Point: <SOH>TOPT <point number><CR>
elseif ($t = $this->kludge('TOPT ',$v)) elseif ($t = $this->kludge('TOPT ',$v))
$this->point['dst'] = $t; $this->point['dst'] = $t;
@ -777,6 +782,14 @@ class Message extends FTNBase
$m = []; $m = [];
if ($this->msgid && preg_match('#([0-9]+:[0-9]+/[0-9]+)?\.?([0-9]+)?@?([A-Za-z-_~]+)?\ +#',$this->msgid,$m)) { if ($this->msgid && preg_match('#([0-9]+:[0-9]+/[0-9]+)?\.?([0-9]+)?@?([A-Za-z-_~]+)?\ +#',$this->msgid,$m)) {
$this->src = Address::parseFTN($m[1].((isset($m[2]) && $m[2] != '') ? '.'.$m[2] : '').(isset($m[3]) ? '@'.$m[3] : '')); $this->src = Address::parseFTN($m[1].((isset($m[2]) && $m[2] != '') ? '.'.$m[2] : '').(isset($m[3]) ? '@'.$m[3] : ''));
// Without a MSGID, get our domain from the origin
} elseif ($this->origin && preg_match('#\(([0-9]+:[0-9]+/[0-9]+)?\.?([0-9]+)?@?([A-Za-z-_~]+)?\)$#',$this->origin,$m)) {
$this->src = Address::parseFTN($m[1].((isset($m[2]) && $m[2] != '') ? '.'.$m[2] : '').(isset($m[3]) ? '@'.$m[3] : ''));
// Otherwise get it from our zone object and packet header
} elseif ($this->zone) {
$this->src = Address::parseFTN(sprintf('%d:%d/%d.%d',$this->zone->zone_id,$this->fn,$this->ff,$this->fp));
} }
// Parse SEEN-BY // Parse SEEN-BY
@ -793,7 +806,7 @@ class Message extends FTNBase
* *
* @return \Illuminate\Contracts\Validation\Validator * @return \Illuminate\Contracts\Validation\Validator
*/ */
public function validate(Domain $domain=NULL): ValidatorResult public function validate(): ValidatorResult
{ {
// Check lengths // Check lengths
$validator = Validator::make([ $validator = Validator::make([
@ -820,18 +833,16 @@ class Message extends FTNBase
'flags' => 'required|numeric', 'flags' => 'required|numeric',
'cost' => 'required|numeric', 'cost' => 'required|numeric',
'echoarea' => 'nullable|max:'.self::AREATAG_LEN, 'echoarea' => 'nullable|max:'.self::AREATAG_LEN,
'ozone' => ['required',$this->domain ? 'in:'.$x=$this->domain->zones->pluck('zone_id')->join(','): ''], 'ozone' => ['required'],
'dzone' => ['required',$this->domain ? 'in:'.$x : ''] 'dzone' => ['required']
]); ]);
if ($domain) {
$validator->after(function($validator) { $validator->after(function($validator) {
if (! $this->fboss_o) if (! $this->fboss_o)
$validator->errors()->add('from',sprintf('Undefined Node [%s] sent message.',$this->fboss)); $validator->errors()->add('from',sprintf('Undefined Node [%s] sent message.',$this->fboss));
if (! $this->tboss_o) if (! $this->tboss_o)
$validator->errors()->add('to',sprintf('Undefined Node [%s] for destination.',$this->tboss)); $validator->errors()->add('to',sprintf('Undefined Node [%s] for destination.',$this->tboss));
}); });
}
if ($validator->fails()) if ($validator->fails())
$this->errors = $validator; $this->errors = $validator;

View File

@ -10,7 +10,7 @@ use Illuminate\Support\Facades\Redis;
use Symfony\Component\HttpFoundation\File\File; use Symfony\Component\HttpFoundation\File\File;
use App\Classes\FTN as FTNBase; use App\Classes\FTN as FTNBase;
use App\Models\{Address,Domain,Setup,Software}; use App\Models\{Address,Setup,Software,Zone};
class Packet extends FTNBase implements \Iterator, \Countable class Packet extends FTNBase implements \Iterator, \Countable
{ {
@ -110,11 +110,12 @@ class Packet extends FTNBase implements \Iterator, \Countable
* Open a packet file * Open a packet file
* *
* @param File $file * @param File $file
* @param Domain|null $domain * @param Zone|null $zone
* @param bool $use_redis
* @return Packet * @return Packet
* @throws InvalidPacketException * @throws InvalidPacketException
*/ */
public static function open(File $file,Domain $domain=NULL): self public static function open(File $file,Zone $zone=NULL,bool $use_redis=TRUE): self
{ {
Log::debug(sprintf('%s:+ Opening Packet [%s]',self::LOGKEY,$file)); Log::debug(sprintf('%s:+ Opening Packet [%s]',self::LOGKEY,$file));
@ -134,6 +135,8 @@ class Packet extends FTNBase implements \Iterator, \Countable
throw new InvalidPacketException('Not a type 2 packet: '.$version); throw new InvalidPacketException('Not a type 2 packet: '.$version);
$o = new self; $o = new self;
$o->zone = $zone;
$o->use_redis = $use_redis;
$o->name = (string)$file; $o->name = (string)$file;
$o->header = unpack(self::unpackheader(self::v2header),$header); $o->header = unpack(self::unpackheader(self::v2header),$header);
@ -151,6 +154,10 @@ class Packet extends FTNBase implements \Iterator, \Countable
else if (! strlen($x)) else if (! strlen($x))
throw new InvalidPacketException('No message in packet: '.bin2hex($x)); throw new InvalidPacketException('No message in packet: '.bin2hex($x));
// If zone is null, we'll take the zone from the packet
if (! $o->zone)
$o->zone = Zone::where('zone_id',$o->fz)->where('default',TRUE)->single();
$buf_ptr = 0; $buf_ptr = 0;
$message = ''; $message = '';
$readbuf = ''; $readbuf = '';
@ -174,7 +181,7 @@ class Packet extends FTNBase implements \Iterator, \Countable
$last .= substr($readbuf,0,2); $last .= substr($readbuf,0,2);
if (($end=strpos($last,"\x00\x02\x00",$buf_ptr)) !== FALSE) { if (($end=strpos($last,"\x00\x02\x00",$buf_ptr)) !== FALSE) {
$o->parseMessage(substr($message,0,$end-2),$domain); $o->parseMessage(substr($message,0,$end-2));
$last = ''; $last = '';
$message = ''; $message = '';
$buf_ptr = 1+$end; $buf_ptr = 1+$end;
@ -217,13 +224,13 @@ class Packet extends FTNBase implements \Iterator, \Countable
} }
// Look for the next message // Look for the next message
$o->parseMessage($message,$domain); $o->parseMessage($message);
$message = ''; $message = '';
} }
// If our message is still set, then we have an unprocessed message // If our message is still set, then we have an unprocessed message
if ($message) if ($message)
$o->parseMessage($message,$domain); $o->parseMessage($message);
return $o; return $o;
} }
@ -413,12 +420,11 @@ class Packet extends FTNBase implements \Iterator, \Countable
* Parse a message in a mail packet * Parse a message in a mail packet
* *
* @param string $message * @param string $message
* @param Domain|null $domain
* @throws InvalidPacketException * @throws InvalidPacketException
*/ */
private function parseMessage(string $message,Domain $domain=NULL): void private function parseMessage(string $message): void
{ {
$msg = Message::parseMessage($message,$domain); $msg = Message::parseMessage($message,$this->zone);
// If the message is invalid, we'll ignore it // If the message is invalid, we'll ignore it
if ($msg->errors && ( if ($msg->errors && (
@ -432,8 +438,9 @@ class Packet extends FTNBase implements \Iterator, \Countable
} else { } else {
if ($this->use_redis) { if ($this->use_redis) {
Redis::set($msg->msgid ?: sprintf('%s %s',$msg->fftn,Carbon::now()->timestamp),serialize($msg)); $key = $msg->msgid ?: sprintf('%s %s',$msg->fftn,Carbon::now()->timestamp);
$this->messages->push($msg->msgid); Redis::set($key,serialize($msg));
$this->messages->push($key);
} else { } else {
$this->messages->push($msg); $this->messages->push($msg);

View File

@ -0,0 +1,35 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Models\Echomail;
class EchomailDump extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'echomail:dump {id}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Echomail Dump';
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
dump(Echomail::findOrFail($this->argument('id')));
}
}

View File

@ -6,8 +6,7 @@ use Illuminate\Console\Command;
use Symfony\Component\HttpFoundation\File\File; use Symfony\Component\HttpFoundation\File\File;
use App\Classes\FTN\Packet; use App\Classes\FTN\Packet;
use App\Jobs\ProcessPacket as Job; use App\Models\Zone;
use App\Models\Domain;
class PacketInfo extends Command class PacketInfo extends Command
{ {
@ -16,7 +15,7 @@ class PacketInfo extends Command
* *
* @var string * @var string
*/ */
protected $signature = 'packet:info {pkt : Packet to process} {domain : Domain the packet is from}'; protected $signature = 'packet:info {pkt : Packet to process} {zone? : Zone the packet is from}';
/** /**
* The console command description. * The console command description.
@ -34,9 +33,9 @@ class PacketInfo extends Command
public function handle() public function handle()
{ {
$f = new File($this->argument('pkt')); $f = new File($this->argument('pkt'));
$d = Domain::where('name',$this->argument('domain'))->singleOrFail(); $z = $this->argument('zone') ? Zone::where('zone_id',$this->argument('zone'))->singleOrFail() : NULL;
$pkt = Packet::open($f,$d); $pkt = Packet::open($f,$z);
$this->info(sprintf('Packet Type: %s',$pkt->type)); $this->info(sprintf('Packet Type: %s',$pkt->type));
$this->info(sprintf('From: %s to %s',$pkt->fftn,$pkt->tftn)); $this->info(sprintf('From: %s to %s',$pkt->fftn,$pkt->tftn));

View File

@ -16,7 +16,7 @@ class ProcessPacket extends Command
* *
* @var string * @var string
*/ */
protected $signature = 'packet:process {pkt : Packet to process} {domain : Domain the packet is from}'; protected $signature = 'packet:process {pkt : Packet to process} {domain? : Domain the packet is from}';
/** /**
* The console command description. * The console command description.
@ -34,14 +34,14 @@ class ProcessPacket extends Command
public function handle() public function handle()
{ {
$f = new File($this->argument('pkt')); $f = new File($this->argument('pkt'));
$d = Domain::where('name',$this->argument('domain'))->singleOrFail(); $d = $this->argument('domain') ? Domain::where('name',$this->argument('domain'))->singleOrFail() : NULL;
foreach (Packet::open($f,$d) as $msg) { foreach (Packet::open($f,$d) as $msg) {
// @todo Quick check that the packet should be processed by us. // @todo Quick check that the packet should be processed by us.
// @todo validate that the packet's zone is in the domain. // @todo validate that the packet's zone is in the domain.
// Dispatch job. // Dispatch job.
Job::dispatch($msg); Job::dispatchSync($msg);
} }
} }
} }

View File

@ -200,10 +200,10 @@ class ProcessPacket implements ShouldQueue
$o->msgid = $this->msg->msgid; $o->msgid = $this->msg->msgid;
$o->msg = $this->msg->message_src; $o->msg = $this->msg->message_src;
$o->path = $this->msg->pathaddress->pluck('id')->jsonSerialize(); $o->path = $this->msg->pathaddress->jsonSerialize();
$o->rogue_path = $this->msg->rogue_path->jsonSerialize(); $o->rogue_path = $this->msg->rogue_path->jsonSerialize();
$o->seenby = $this->msg->seenaddress->pluck('id')->jsonSerialize(); $o->seenby = $this->msg->seenaddress->jsonSerialize();
$o->rogue_seen = $this->msg->rogue_path->jsonSerialize(); $o->rogue_seen = $this->msg->rogue_seen->jsonSerialize();
$o->toexport = TRUE; $o->toexport = TRUE;
$o->save(); $o->save();

View File

@ -0,0 +1,118 @@
<?php
namespace Tests\Feature;
use Symfony\Component\HttpFoundation\File\File;
use Tests\TestCase;
use App\Classes\FTN\Packet;
use App\Models\{Address,Domain,System,Zone};
class PacketTest extends TestCase
{
private System $so;
private Domain $do;
private function init()
{
System::unguard();
Domain::unguard();
$this->so = System::firstOrCreate(['name'=>'test','sysop'=>'sysop','location'=>'location','active'=>TRUE]);
$this->do = Domain::firstOrCreate(['name'=>'test','active'=>TRUE]);
}
public function test_nomsgid_origin()
{
$this->init();
Zone::unguard();
Address::unguard();
$zo = Zone::firstOrCreate(['zone_id'=>21,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>3,'node_id'=>151,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
$hub = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>1,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
$ao = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>1,'node_id'=>4,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
// This packet has an incorrect zone in the Origin
$f = new File(__DIR__.'/data/test_nomsgid_origin.pkt');
$pkt = Packet::open($f);
$this->assertEquals(1,$pkt->count());
$messages = FALSE;
foreach ($pkt as $msg) {
$messages = TRUE;
$this->assertNotTrue($msg->isNetmail());
$this->assertNotFalse($msg->pathaddress->search($hub->id));
$this->assertCount(1,$msg->rogue_path);
$this->assertNotFalse($msg->rogue_path->search('21:999/1'));
$this->assertCount(3,$msg->rogue_seen);
$this->assertNotFalse($msg->rogue_seen->search('21:999/1'));
$this->assertNotFalse($msg->rogue_seen->search('21:999/999'));
}
$this->assertTrue($messages);
}
public function test_nomsgid_noorigin()
{
$this->init();
Zone::unguard();
Address::unguard();
$zo = Zone::firstOrCreate(['zone_id'=>10,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>999,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
// This packet has an incorrect zone in the Origin
$f = new File(__DIR__.'/data/test_nomsgid_noorigin.pkt');
$pkt = Packet::open($f,$zo,TRUE);
$this->assertEquals(1,$pkt->count());
$messages = FALSE;
foreach ($pkt as $msg) {
$messages = TRUE;
$this->assertNotTrue($msg->isNetmail());
$this->assertCount(1,$msg->rogue_path);
$this->assertNotFalse($msg->rogue_path->search('10:1/1'));
$this->assertCount(0,$msg->rogue_seen);
}
$this->assertTrue($messages);
}
public function test_msgid_origin()
{
$this->init();
Zone::unguard();
Address::unguard();
$zo = Zone::firstOrCreate(['zone_id'=>10,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>999,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
// This packet has an incorrect zone in the Origin
$f = new File(__DIR__.'/data/test_msgid_origin.pkt');
$pkt = Packet::open($f,$zo,TRUE);
$this->assertEquals(1,$pkt->count());
$messages = FALSE;
foreach ($pkt as $msg) {
$messages = TRUE;
$this->assertNotTrue($msg->isNetmail());
$this->assertCount(0,$msg->rogue_path);
$this->assertCount(2,$msg->rogue_seen);
$this->assertNotFalse($msg->rogue_seen->search('10:1/1'));
$this->assertNotFalse($msg->rogue_seen->search('10:999/999'));
$this->assertNotFalse($msg->seenaddress->search($src->id));
}
$this->assertTrue($messages);
}
}

Binary file not shown.

Binary file not shown.

Binary file not shown.