From 9fb6d191d0e5316d719910b8b2c9503c18cfd603 Mon Sep 17 00:00:00 2001 From: Deon George Date: Sun, 29 Aug 2021 23:58:12 +1000 Subject: [PATCH] Change the way we figure out zones in packets, some packet testing, fix Echomail import --- app/Classes/FTN.php | 8 +- app/Classes/FTN/Message.php | 55 +++++---- app/Classes/FTN/Packet.php | 29 +++-- app/Console/Commands/EchomailDump.php | 35 ++++++ app/Console/Commands/PacketInfo.php | 9 +- app/Console/Commands/ProcessPacket.php | 6 +- app/Jobs/ProcessPacket.php | 6 +- tests/Feature/PacketTest.php | 118 +++++++++++++++++++ tests/Feature/data/test_msgid_origin.pkt | Bin 0 -> 1027 bytes tests/Feature/data/test_nomsgid_noorigin.pkt | Bin 0 -> 403 bytes tests/Feature/data/test_nomsgid_origin.pkt | Bin 0 -> 495 bytes 11 files changed, 218 insertions(+), 48 deletions(-) create mode 100644 app/Console/Commands/EchomailDump.php create mode 100644 tests/Feature/PacketTest.php create mode 100644 tests/Feature/data/test_msgid_origin.pkt create mode 100644 tests/Feature/data/test_nomsgid_noorigin.pkt create mode 100644 tests/Feature/data/test_nomsgid_origin.pkt diff --git a/app/Classes/FTN.php b/app/Classes/FTN.php index 97033a1..568a8c9 100644 --- a/app/Classes/FTN.php +++ b/app/Classes/FTN.php @@ -2,11 +2,11 @@ namespace App\Classes; -use App\Models\{Address,Domain}; +use App\Models\{Address,Zone}; abstract class FTN { - protected ?Domain $domain; // Domain the packet is from + protected ?Zone $zone; // Zone the packet is from public function __get($key) { @@ -17,7 +17,7 @@ abstract class FTN $this->fn, $this->ff, $this->fp, - ).($this->domain ? sprintf('@%s',$this->domain->name) : ''); + ).($this->zone ? sprintf('@%s',$this->zone->domain->name) : ''); case 'tftn': return sprintf('%d:%d/%d.%d', @@ -25,7 +25,7 @@ abstract class FTN $this->tn, $this->tf, $this->tp, - ).($this->domain ? sprintf('@%s',$this->domain->name) : ''); + ).($this->zone ? sprintf('@%s',$this->zone->domain->name) : ''); case 'fftn_o': return Address::findFTN($this->fftn); diff --git a/app/Classes/FTN/Message.php b/app/Classes/FTN/Message.php index 2839541..5b37734 100644 --- a/app/Classes/FTN/Message.php +++ b/app/Classes/FTN/Message.php @@ -10,7 +10,7 @@ use Illuminate\Support\Facades\Validator; use Illuminate\Validation\Validator as ValidatorResult; use App\Classes\FTN as FTNBase; -use App\Models\{Address,Domain}; +use App\Models\{Address,Zone}; use App\Rules\TwoByteInteger; use App\Traits\EncodeUTF8; @@ -163,9 +163,9 @@ class Message extends FTNBase 0xfc => 0x207f, 0xfd => 0x00b2, 0xfe => 0x25a0, 0xff => 0x00a0, ]; - public function __construct(Domain $domain=NULL) + public function __construct(Zone $zone=NULL) { - $this->domain = $domain; + $this->zone = $zone; $this->header = []; $this->kludge = collect(); @@ -204,9 +204,9 @@ class Message extends FTNBase * @return Message * @throws InvalidPacketException */ - public static function parseMessage(string $msg,Domain $domain=NULL): self + public static function parseMessage(string $msg,Zone $zone=NULL): self { - $o = new self($domain); + $o = new self($zone); try { $o->header = unpack(self::unpackheader(self::header),substr($msg,0,self::HEADER_LEN)); @@ -246,7 +246,7 @@ class Message extends FTNBase $o->unpackMessage(substr($msg,self::HEADER_LEN+$ptr)); - if (($x=$o->validate($domain))->fails()) { + if (($x=$o->validate())->fails()) { Log::debug('Message fails validation',['result'=>$x->errors()]); //throw new \Exception('Message validation fails:'.join(' ',$x->errors()->all())); } @@ -651,7 +651,7 @@ class Message extends FTNBase * Extract information out of the message text. * * @param string $message - * @throws InvalidPacketException + * @throws \Exception */ public function unpackMessage(string $message): void { @@ -690,11 +690,13 @@ class Message extends FTNBase preg_match('/^.*\((.*)\)$/',$this->origin,$matches); // Double check we have an address in the origin line - if (! Arr::get($matches,1)) - throw new InvalidPacketException('No address in Origin?'); + if (! Arr::get($matches,1)) { + Log::error(sprintf('%s:! Origin line doesnt have an address',self::LOGKEY)); - // Double check, our src and origin match - $this->src = Address::parseFTN($matches[1]); + } else { + // Double check, our src and origin match + $this->src = Address::parseFTN($matches[1]); + } // We'll double check our FTN if ($this->isNetmail() && (($this->src['n'] !== $this->fn) || ($this->src['f'] !== $this->ff))) { @@ -758,6 +760,9 @@ class Message extends FTNBase elseif ($t = $this->kludge('PATH: ',$v)) $this->path->push($t); + elseif ($t = $this->kludge('SEEN-BY: ',$v)) + $this->seenby->push($t); + // To Point: TOPT elseif ($t = $this->kludge('TOPT ',$v)) $this->point['dst'] = $t; @@ -777,6 +782,14 @@ class Message extends FTNBase $m = []; if ($this->msgid && preg_match('#([0-9]+:[0-9]+/[0-9]+)?\.?([0-9]+)?@?([A-Za-z-_~]+)?\ +#',$this->msgid,$m)) { $this->src = Address::parseFTN($m[1].((isset($m[2]) && $m[2] != '') ? '.'.$m[2] : '').(isset($m[3]) ? '@'.$m[3] : '')); + + // Without a MSGID, get our domain from the origin + } elseif ($this->origin && preg_match('#\(([0-9]+:[0-9]+/[0-9]+)?\.?([0-9]+)?@?([A-Za-z-_~]+)?\)$#',$this->origin,$m)) { + $this->src = Address::parseFTN($m[1].((isset($m[2]) && $m[2] != '') ? '.'.$m[2] : '').(isset($m[3]) ? '@'.$m[3] : '')); + + // Otherwise get it from our zone object and packet header + } elseif ($this->zone) { + $this->src = Address::parseFTN(sprintf('%d:%d/%d.%d',$this->zone->zone_id,$this->fn,$this->ff,$this->fp)); } // Parse SEEN-BY @@ -793,7 +806,7 @@ class Message extends FTNBase * * @return \Illuminate\Contracts\Validation\Validator */ - public function validate(Domain $domain=NULL): ValidatorResult + public function validate(): ValidatorResult { // Check lengths $validator = Validator::make([ @@ -820,18 +833,16 @@ class Message extends FTNBase 'flags' => 'required|numeric', 'cost' => 'required|numeric', 'echoarea' => 'nullable|max:'.self::AREATAG_LEN, - 'ozone' => ['required',$this->domain ? 'in:'.$x=$this->domain->zones->pluck('zone_id')->join(','): ''], - 'dzone' => ['required',$this->domain ? 'in:'.$x : ''] + 'ozone' => ['required'], + 'dzone' => ['required'] ]); - if ($domain) { - $validator->after(function($validator) { - if (! $this->fboss_o) - $validator->errors()->add('from',sprintf('Undefined Node [%s] sent message.',$this->fboss)); - if (! $this->tboss_o) - $validator->errors()->add('to',sprintf('Undefined Node [%s] for destination.',$this->tboss)); - }); - } + $validator->after(function($validator) { + if (! $this->fboss_o) + $validator->errors()->add('from',sprintf('Undefined Node [%s] sent message.',$this->fboss)); + if (! $this->tboss_o) + $validator->errors()->add('to',sprintf('Undefined Node [%s] for destination.',$this->tboss)); + }); if ($validator->fails()) $this->errors = $validator; diff --git a/app/Classes/FTN/Packet.php b/app/Classes/FTN/Packet.php index 30fe157..629880a 100644 --- a/app/Classes/FTN/Packet.php +++ b/app/Classes/FTN/Packet.php @@ -10,7 +10,7 @@ use Illuminate\Support\Facades\Redis; use Symfony\Component\HttpFoundation\File\File; use App\Classes\FTN as FTNBase; -use App\Models\{Address,Domain,Setup,Software}; +use App\Models\{Address,Setup,Software,Zone}; class Packet extends FTNBase implements \Iterator, \Countable { @@ -110,11 +110,12 @@ class Packet extends FTNBase implements \Iterator, \Countable * Open a packet file * * @param File $file - * @param Domain|null $domain + * @param Zone|null $zone + * @param bool $use_redis * @return Packet * @throws InvalidPacketException */ - public static function open(File $file,Domain $domain=NULL): self + public static function open(File $file,Zone $zone=NULL,bool $use_redis=TRUE): self { Log::debug(sprintf('%s:+ Opening Packet [%s]',self::LOGKEY,$file)); @@ -134,6 +135,8 @@ class Packet extends FTNBase implements \Iterator, \Countable throw new InvalidPacketException('Not a type 2 packet: '.$version); $o = new self; + $o->zone = $zone; + $o->use_redis = $use_redis; $o->name = (string)$file; $o->header = unpack(self::unpackheader(self::v2header),$header); @@ -151,6 +154,10 @@ class Packet extends FTNBase implements \Iterator, \Countable else if (! strlen($x)) throw new InvalidPacketException('No message in packet: '.bin2hex($x)); + // If zone is null, we'll take the zone from the packet + if (! $o->zone) + $o->zone = Zone::where('zone_id',$o->fz)->where('default',TRUE)->single(); + $buf_ptr = 0; $message = ''; $readbuf = ''; @@ -174,7 +181,7 @@ class Packet extends FTNBase implements \Iterator, \Countable $last .= substr($readbuf,0,2); if (($end=strpos($last,"\x00\x02\x00",$buf_ptr)) !== FALSE) { - $o->parseMessage(substr($message,0,$end-2),$domain); + $o->parseMessage(substr($message,0,$end-2)); $last = ''; $message = ''; $buf_ptr = 1+$end; @@ -217,13 +224,13 @@ class Packet extends FTNBase implements \Iterator, \Countable } // Look for the next message - $o->parseMessage($message,$domain); + $o->parseMessage($message); $message = ''; } // If our message is still set, then we have an unprocessed message if ($message) - $o->parseMessage($message,$domain); + $o->parseMessage($message); return $o; } @@ -413,12 +420,11 @@ class Packet extends FTNBase implements \Iterator, \Countable * Parse a message in a mail packet * * @param string $message - * @param Domain|null $domain * @throws InvalidPacketException */ - private function parseMessage(string $message,Domain $domain=NULL): void + private function parseMessage(string $message): void { - $msg = Message::parseMessage($message,$domain); + $msg = Message::parseMessage($message,$this->zone); // If the message is invalid, we'll ignore it if ($msg->errors && ( @@ -432,8 +438,9 @@ class Packet extends FTNBase implements \Iterator, \Countable } else { if ($this->use_redis) { - Redis::set($msg->msgid ?: sprintf('%s %s',$msg->fftn,Carbon::now()->timestamp),serialize($msg)); - $this->messages->push($msg->msgid); + $key = $msg->msgid ?: sprintf('%s %s',$msg->fftn,Carbon::now()->timestamp); + Redis::set($key,serialize($msg)); + $this->messages->push($key); } else { $this->messages->push($msg); diff --git a/app/Console/Commands/EchomailDump.php b/app/Console/Commands/EchomailDump.php new file mode 100644 index 0000000..815e808 --- /dev/null +++ b/app/Console/Commands/EchomailDump.php @@ -0,0 +1,35 @@ +argument('id'))); + } +} diff --git a/app/Console/Commands/PacketInfo.php b/app/Console/Commands/PacketInfo.php index 6dffaff..febbf22 100644 --- a/app/Console/Commands/PacketInfo.php +++ b/app/Console/Commands/PacketInfo.php @@ -6,8 +6,7 @@ use Illuminate\Console\Command; use Symfony\Component\HttpFoundation\File\File; use App\Classes\FTN\Packet; -use App\Jobs\ProcessPacket as Job; -use App\Models\Domain; +use App\Models\Zone; class PacketInfo extends Command { @@ -16,7 +15,7 @@ class PacketInfo extends Command * * @var string */ - protected $signature = 'packet:info {pkt : Packet to process} {domain : Domain the packet is from}'; + protected $signature = 'packet:info {pkt : Packet to process} {zone? : Zone the packet is from}'; /** * The console command description. @@ -34,9 +33,9 @@ class PacketInfo extends Command public function handle() { $f = new File($this->argument('pkt')); - $d = Domain::where('name',$this->argument('domain'))->singleOrFail(); + $z = $this->argument('zone') ? Zone::where('zone_id',$this->argument('zone'))->singleOrFail() : NULL; - $pkt = Packet::open($f,$d); + $pkt = Packet::open($f,$z); $this->info(sprintf('Packet Type: %s',$pkt->type)); $this->info(sprintf('From: %s to %s',$pkt->fftn,$pkt->tftn)); diff --git a/app/Console/Commands/ProcessPacket.php b/app/Console/Commands/ProcessPacket.php index 9b21ab8..362ee9c 100644 --- a/app/Console/Commands/ProcessPacket.php +++ b/app/Console/Commands/ProcessPacket.php @@ -16,7 +16,7 @@ class ProcessPacket extends Command * * @var string */ - protected $signature = 'packet:process {pkt : Packet to process} {domain : Domain the packet is from}'; + protected $signature = 'packet:process {pkt : Packet to process} {domain? : Domain the packet is from}'; /** * The console command description. @@ -34,14 +34,14 @@ class ProcessPacket extends Command public function handle() { $f = new File($this->argument('pkt')); - $d = Domain::where('name',$this->argument('domain'))->singleOrFail(); + $d = $this->argument('domain') ? Domain::where('name',$this->argument('domain'))->singleOrFail() : NULL; foreach (Packet::open($f,$d) as $msg) { // @todo Quick check that the packet should be processed by us. // @todo validate that the packet's zone is in the domain. // Dispatch job. - Job::dispatch($msg); + Job::dispatchSync($msg); } } } diff --git a/app/Jobs/ProcessPacket.php b/app/Jobs/ProcessPacket.php index 7b2a1c3..2589f4c 100644 --- a/app/Jobs/ProcessPacket.php +++ b/app/Jobs/ProcessPacket.php @@ -200,10 +200,10 @@ class ProcessPacket implements ShouldQueue $o->msgid = $this->msg->msgid; $o->msg = $this->msg->message_src; - $o->path = $this->msg->pathaddress->pluck('id')->jsonSerialize(); + $o->path = $this->msg->pathaddress->jsonSerialize(); $o->rogue_path = $this->msg->rogue_path->jsonSerialize(); - $o->seenby = $this->msg->seenaddress->pluck('id')->jsonSerialize(); - $o->rogue_seen = $this->msg->rogue_path->jsonSerialize(); + $o->seenby = $this->msg->seenaddress->jsonSerialize(); + $o->rogue_seen = $this->msg->rogue_seen->jsonSerialize(); $o->toexport = TRUE; $o->save(); diff --git a/tests/Feature/PacketTest.php b/tests/Feature/PacketTest.php new file mode 100644 index 0000000..2eb7a8e --- /dev/null +++ b/tests/Feature/PacketTest.php @@ -0,0 +1,118 @@ +so = System::firstOrCreate(['name'=>'test','sysop'=>'sysop','location'=>'location','active'=>TRUE]); + $this->do = Domain::firstOrCreate(['name'=>'test','active'=>TRUE]); + } + + public function test_nomsgid_origin() + { + $this->init(); + + Zone::unguard(); + Address::unguard(); + $zo = Zone::firstOrCreate(['zone_id'=>21,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]); + $src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>3,'node_id'=>151,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]); + $hub = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>1,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]); + $ao = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>1,'node_id'=>4,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]); + + // This packet has an incorrect zone in the Origin + $f = new File(__DIR__.'/data/test_nomsgid_origin.pkt'); + $pkt = Packet::open($f); + + $this->assertEquals(1,$pkt->count()); + + $messages = FALSE; + foreach ($pkt as $msg) { + $messages = TRUE; + $this->assertNotTrue($msg->isNetmail()); + + $this->assertNotFalse($msg->pathaddress->search($hub->id)); + + $this->assertCount(1,$msg->rogue_path); + $this->assertNotFalse($msg->rogue_path->search('21:999/1')); + + $this->assertCount(3,$msg->rogue_seen); + $this->assertNotFalse($msg->rogue_seen->search('21:999/1')); + $this->assertNotFalse($msg->rogue_seen->search('21:999/999')); + } + + $this->assertTrue($messages); + } + + public function test_nomsgid_noorigin() + { + $this->init(); + + Zone::unguard(); + Address::unguard(); + $zo = Zone::firstOrCreate(['zone_id'=>10,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]); + $src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>999,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]); + + // This packet has an incorrect zone in the Origin + $f = new File(__DIR__.'/data/test_nomsgid_noorigin.pkt'); + $pkt = Packet::open($f,$zo,TRUE); + + $this->assertEquals(1,$pkt->count()); + + $messages = FALSE; + foreach ($pkt as $msg) { + $messages = TRUE; + $this->assertNotTrue($msg->isNetmail()); + + $this->assertCount(1,$msg->rogue_path); + $this->assertNotFalse($msg->rogue_path->search('10:1/1')); + + $this->assertCount(0,$msg->rogue_seen); + } + + $this->assertTrue($messages); + } + + public function test_msgid_origin() + { + $this->init(); + + Zone::unguard(); + Address::unguard(); + $zo = Zone::firstOrCreate(['zone_id'=>10,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]); + $src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>999,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]); + + // This packet has an incorrect zone in the Origin + $f = new File(__DIR__.'/data/test_msgid_origin.pkt'); + $pkt = Packet::open($f,$zo,TRUE); + + $this->assertEquals(1,$pkt->count()); + + $messages = FALSE; + foreach ($pkt as $msg) { + $messages = TRUE; + $this->assertNotTrue($msg->isNetmail()); + $this->assertCount(0,$msg->rogue_path); + + $this->assertCount(2,$msg->rogue_seen); + $this->assertNotFalse($msg->rogue_seen->search('10:1/1')); + $this->assertNotFalse($msg->rogue_seen->search('10:999/999')); + + $this->assertNotFalse($msg->seenaddress->search($src->id)); + } + + $this->assertTrue($messages); + } +} diff --git a/tests/Feature/data/test_msgid_origin.pkt b/tests/Feature/data/test_msgid_origin.pkt new file mode 100644 index 0000000000000000000000000000000000000000..c657fad42865d3019c380bc52c7a22a35d890991 GIT binary patch literal 1027 zcmb7D%TC)s6m`l1m3E~5K&7)IDYfIV4aAlr#R+bLL>`JwtFl2uGL2n~ZP}9m62FUB zRBAt=9XphVP#2wrH8c16%(>^>TT|{fZZ|iT_sU!4jZ#+><(0g-+xT6ZxW4~oJU!S_ zw&bl+d-u8~ch$Z;%e-2ZdACuX3>)rp0fPe6a42=mmNMsQq70U2n5Ji`c-&#ivosfq zJM~;=@;NvSJU<|{q2E6|a6om(w(Tax9yJ)B%^9f$C&xjrjOe;Nlu2PGnsIUks@a;Z z&Xze0-Jq&&skB9&bJ1JQu8EQh>Jpnatp+ahRB(q7?b*74+V%Q}BMd#?cl(}-n`d#2 zbe=Nl+!Jn5Q_EK3RPaJ{)-S-=wRK~6&mf&iN!PzlW?`NtT)@<*&D5hPSzh6S6@ur@ z*0+7zGFx*4ZYB${s#?(R_0ZBxO_#4$7~QVVXJJ|*SpNyo8z1>nKhq{1iBh7uM}aqW zM*->d22&a9O)PUClJ967cn&`K-aHa%u6-}Cq8imL63Fr(QY?7Fb0#>7^nxQ}vvV#A zjmRa%`tkGj7iqWK@L#ee*uglD7E$7$0vqz8U<(fE%p{LiCFTj$A6z27=Z$u{UnOzb z4HnF%j;b@HsBSPQuPx9Q9zyj-2aguM@DQXPg!0jf35y{(%MnKjN5~4Cah@ROS|}dkmY;?OXd(yf2*vEigPRrnHVKibY0#WZ5to%%iaDLauzQO)OQ+MgQ%*Chf#4b zm;`FZ^B7^8o|6Y8>02y2h9v?+7R7_(E||b|D*M0S`6`<6O1(;@Mx~$nSHJa4$uE(U LU8=X1QjYE)H3$#~ literal 0 HcmV?d00001 diff --git a/tests/Feature/data/test_nomsgid_noorigin.pkt b/tests/Feature/data/test_nomsgid_noorigin.pkt new file mode 100644 index 0000000000000000000000000000000000000000..ccdc0a875a8bc56e2cb9df18f56e606b0284dade GIT binary patch literal 403 zcmXX?T~C8B6fN#y(WJ&_pRVr$Eg#F^!N3M=OE$JZanUR>u%TOmU|2Ep2b=v2Eojm< z=bU?*bMKYVx%b`gqg!;1?hru;T?pme`;k^|9Iuy=cZIHmilm!MNuUn5T==BUdG57L zGr?`%K%*e8t&1~2ZZv*R4+WYw8AMf9akQ{E%Y==VN$71%OB>R_(3Q1Mgw=QL)gyah ze$24Mo)b2$1VI2E@D36}RG%5uHbd`W* SSptg}i!HP1U#SzZSmiG&iC^{r literal 0 HcmV?d00001 diff --git a/tests/Feature/data/test_nomsgid_origin.pkt b/tests/Feature/data/test_nomsgid_origin.pkt new file mode 100644 index 0000000000000000000000000000000000000000..eb22c6c3f5fbd90c5bcdb5546b7c45d73e136b5b GIT binary patch literal 495 zcmYk3O>f#j5Qf*~5EW^qo^r}zE|nmP@$P~N_MyTQD@Ni=*raMN)f%>7)v+Uc9hDy( z@iUsi1Syx*JKxXj806S^y}BYdHJkB+l`ZIqrC*I2U+~s5R zJMzPb8qu$H^xXx{7(-(+c47altu*@wyyJA3xv2^eOmLI+Og^CNOvb4Kox@h8$%`sr zuJn^kQi^5YAk2&6a0hy?AXtweDWS@w)+*boZHLm~WFAcs*X>$3@LkxP4(q|QL#dMu zj7A~O<^sH4pAKi?!wKqKV{`Z;xf9#>5^K)nOd$E!z@IKDHBA#fba}b7Ic>l!Hn2n= zS}R#T#|k-NnOI%Xrf>_3BH1OG3uAfEc_wRy7H+mzMJlsx*@o7IpiM*HpPQqf_-Pi0 X9&{}Rj^m)lm*NGZ;2f%*6wmYq=U{m$ literal 0 HcmV?d00001