From ebd1cf873220c60681dbdc68f05c6c55981bb5f0 Mon Sep 17 00:00:00 2001 From: Deon George Date: Wed, 24 Nov 2021 22:34:40 +1100 Subject: [PATCH] Enable duplicates to update seen-by's before being discarded --- app/Classes/FTN/Packet.php | 6 ++++-- app/Classes/File/Receive.php | 2 +- app/Console/Commands/PacketInfo.php | 8 +++---- app/Console/Commands/PacketProcess.php | 6 +++--- app/Jobs/MessageProcess.php | 29 ++++++++++++++++++++++++-- tests/Feature/PacketTest.php | 4 ++-- 6 files changed, 41 insertions(+), 14 deletions(-) diff --git a/app/Classes/FTN/Packet.php b/app/Classes/FTN/Packet.php index d63b2a4..886d7d3 100644 --- a/app/Classes/FTN/Packet.php +++ b/app/Classes/FTN/Packet.php @@ -59,6 +59,7 @@ class Packet extends FTNBase implements \Iterator, \Countable public Collection $messages; // Messages in the Packet public Collection $errors; // Messages that fail validation private string $name; // Packet name + private ?System $system; // System the packet is from public bool $use_redis = TRUE; // Use redis for messages. private int $index; // Our array index @@ -116,7 +117,7 @@ class Packet extends FTNBase implements \Iterator, \Countable * @return Packet * @throws InvalidPacketException */ - public static function open(File $file,Zone $zone=NULL,bool $use_redis=TRUE): self + public static function open(File $file,System $system=NULL,bool $use_redis=TRUE): self { Log::debug(sprintf('%s:+ Opening Packet [%s]',self::LOGKEY,$file)); @@ -136,7 +137,6 @@ class Packet extends FTNBase implements \Iterator, \Countable throw new InvalidPacketException('Not a type 2 packet: '.$version); $o = new self; - $o->zone = $zone; $o->use_redis = $use_redis; $o->name = (string)$file; $o->header = unpack(self::unpackheader(self::v2header),$header); @@ -155,6 +155,8 @@ class Packet extends FTNBase implements \Iterator, \Countable else if (! strlen($x)) throw new InvalidPacketException('No message in packet: '.bin2hex($x)); + $o->zone = $system?->zones->firstWhere('zone_id',$o->fz); + // If zone is null, we'll take the zone from the packet if (! $o->zone) $o->zone = Zone::where('zone_id',$o->fz)->where('default',TRUE)->single(); diff --git a/app/Classes/File/Receive.php b/app/Classes/File/Receive.php index 375372b..4f57963 100644 --- a/app/Classes/File/Receive.php +++ b/app/Classes/File/Receive.php @@ -112,7 +112,7 @@ final class Receive extends Item Log::info(sprintf('%s: - Processing mail packet [%s]',self::LOGKEY,$this->file)); try { - $po = Packet::open(new File($this->file),$this->ao->zone); + $po = Packet::open(new File($this->file),$this->ao->system); } catch (InvalidPacketException $e) { Log::error(sprintf('%s: - Not deleting packet [%s], as it generated an exception',self::LOGKEY,$this->file)); diff --git a/app/Console/Commands/PacketInfo.php b/app/Console/Commands/PacketInfo.php index fdee03a..b017ec1 100644 --- a/app/Console/Commands/PacketInfo.php +++ b/app/Console/Commands/PacketInfo.php @@ -6,7 +6,7 @@ use Illuminate\Console\Command; use Symfony\Component\HttpFoundation\File\File; use App\Classes\FTN\Packet; -use App\Models\Zone; +use App\Models\System; class PacketInfo extends Command { @@ -17,7 +17,7 @@ class PacketInfo extends Command */ protected $signature = 'packet:info' .' {pkt : Packet to process}' - .' {zone? : Zone the packet is from}'; + .' {system? : Zone the packet is from}'; /** * The console command description. @@ -35,9 +35,9 @@ class PacketInfo extends Command public function handle() { $f = new File($this->argument('pkt')); - $z = $this->argument('zone') ? Zone::where('zone_id',$this->argument('zone'))->singleOrFail() : NULL; + $s = $this->argument('system') ? System::where('name',$this->argument('zone'))->singleOrFail() : NULL; - $pkt = Packet::open($f,$z); + $pkt = Packet::open($f,$s); $this->info(sprintf('Packet Type: %s',$pkt->type)); $this->info(sprintf('From: %s to %s',$pkt->fftn,$pkt->tftn)); diff --git a/app/Console/Commands/PacketProcess.php b/app/Console/Commands/PacketProcess.php index 6472a23..2e923da 100644 --- a/app/Console/Commands/PacketProcess.php +++ b/app/Console/Commands/PacketProcess.php @@ -19,7 +19,7 @@ class PacketProcess extends Command protected $signature = 'packet:process' .' {pkt : Packet to process}' .' {--N|nobot : Dont process bots}' - .' {zone? : Zone the packet is from}'; + .' {system? : Zone the packet is from}'; /** * The console command description. @@ -37,9 +37,9 @@ class PacketProcess extends Command public function handle() { $f = new File($this->argument('pkt')); - $z = $this->argument('zone') ? Zone::where('zone_id',$this->argument('zone'))->singleOrFail() : NULL; + $s = $this->argument('system') ? Zone::where('name',$this->argument('system'))->singleOrFail() : NULL; - foreach (Packet::open($f,$z) as $msg) { + foreach (Packet::open($f,$s) as $msg) { // @todo Quick check that the packet should be processed by us. // @todo validate that the packet's zone is in the domain. diff --git a/app/Jobs/MessageProcess.php b/app/Jobs/MessageProcess.php index aa9707e..1329ea6 100644 --- a/app/Jobs/MessageProcess.php +++ b/app/Jobs/MessageProcess.php @@ -142,16 +142,20 @@ class MessageProcess implements ShouldQueue ->where('domain_id',$this->msg->fboss_o->zone->domain_id) ->single(); + Log::debug(sprintf('%s: - Processing echomail [%s].',self::LOGKEY,$this->msg->msgid)); + // Check for duplicate messages // FTS-0009.001 if ($this->msg->msgid) { $o = Echomail::where('msgid',$this->msg->msgid) ->where('fftn_id',($x=$this->msg->fboss_o) ? $x->id : NULL) - ->where('datetime',Carbon::now()->subYears(3)) + ->where('datetime','>',Carbon::now()->subYears(3)) ->single(); + Log::debug(sprintf('%s: - Checking for duplicate from host id [%d].',self::LOGKEY,($x=$this->msg->fboss_o) ? $x->id : NULL)); + if ($o) { - Log::alert(sprintf('%s:! Ignoring duplicate echomail [%s] in [%s] from (%s) [%s] to (%s).', + Log::alert(sprintf('%s:! Duplicate echomail [%s] in [%s] from (%s) [%s] to (%s) - updating seenby.', self::LOGKEY, $this->msg->msgid, $this->msg->echoarea, @@ -159,10 +163,31 @@ class MessageProcess implements ShouldQueue $this->msg->user_from, )); + if (! $o->msg_crc) + $o->msg_crc = md5($this->msg->message); + + $o->seenby = collect($o->getRawOriginal('seenby'))->merge($this->msg->seenaddress)->filter()->toArray(); + $o->save(); + return; } } + // Find another message with the same msg_crc + if ($this->msg->message) { + $o = Echomail::where('msg_crc',$xx=md5($this->msg->message)) + ->where('fftn_id',($x=$this->msg->fboss_o) ? $x->id : NULL) + ->where('datetime','>',Carbon::now()->subWeek()) + ->get(); + + if ($o->count()) + Log::alert(sprintf('%s:! Duplicate message CRC [%s] in [%s].', + self::LOGKEY, + $xx, + $o->pluck('id')->join('|') + )); + } + // @todo Can the sender create it if it doesnt exist? // @todo Can the sender send messages to this area? // - Create it, or diff --git a/tests/Feature/PacketTest.php b/tests/Feature/PacketTest.php index d0ba9d3..d7eb6d7 100644 --- a/tests/Feature/PacketTest.php +++ b/tests/Feature/PacketTest.php @@ -67,7 +67,7 @@ class PacketTest extends TestCase // This packet has an incorrect zone in the Origin $f = new File(__DIR__.'/data/test_nomsgid_noorigin.pkt'); - $pkt = Packet::open($f,$zo,FALSE); + $pkt = Packet::open($f,$this->so,FALSE); $this->assertEquals(1,$pkt->count()); @@ -96,7 +96,7 @@ class PacketTest extends TestCase // This packet has an incorrect zone in the Origin $f = new File(__DIR__.'/data/test_msgid_origin.pkt'); - $pkt = Packet::open($f,$zo,FALSE); + $pkt = Packet::open($f,$this->so,FALSE); $this->assertEquals(1,$pkt->count());