From 188fd1a2cfcffa947eacd3ac99b5cc4fa2dbfe0d Mon Sep 17 00:00:00 2001 From: Deon George Date: Mon, 6 May 2019 22:29:29 +1000 Subject: [PATCH] Initial netmail import --- app/Classes/FTNMessage.php | 67 +++++-- app/Console/Commands/FtnPkt.php | 10 +- app/Console/Commands/ImportPacket.php | 177 +++++++++++++----- app/Models/Echomail.php | 1 + app/Models/Netmail.php | 21 +++ app/Traits/ParseNodes.php | 3 +- app/Traits/ParseZNFPDomain.php | 26 +++ .../2019_04_26_112307_create_netmail.php | 60 ++++++ ...019_04_26_113128_create_netmail_tables.php | 50 +++++ 9 files changed, 344 insertions(+), 71 deletions(-) create mode 100644 app/Models/Netmail.php create mode 100644 app/Traits/ParseZNFPDomain.php create mode 100644 database/migrations/2019_04_26_112307_create_netmail.php create mode 100644 database/migrations/2019_04_26_113128_create_netmail_tables.php diff --git a/app/Classes/FTNMessage.php b/app/Classes/FTNMessage.php index 1226188..fad537b 100644 --- a/app/Classes/FTNMessage.php +++ b/app/Classes/FTNMessage.php @@ -31,7 +31,7 @@ class FTNMessage extends FTN private $unknown = []; private $fqfa = NULL; // Fully qualified fidonet source where packet originated - private $fqfd = NULL; // Fully qualified fidonet destination address (Netmail) + private $fqda = NULL; // Fully qualified fidonet destination address (Netmail) // Single value kludge items private $_kludge = [ @@ -65,28 +65,45 @@ class FTNMessage extends FTN $result = unpack($this->unpackheader($struct),$header); - $this->fn = array_get($result,'onet'); - $this->ff = array_get($result,'onode'); + // For Echomail this is the packet src. + $this->psn = array_get($result,'onet'); + $this->psf = array_get($result,'onode'); $this->src = sprintf('%s/%s', - $this->fn, - $this->ff + $this->psn, + $this->psf ); - $this->tn = array_get($result,'dnet'); - $this->tf = array_get($result,'dnode'); + // For Echomail this is the packet dst. + $this->pdn = array_get($result,'dnet'); + $this->pdf = array_get($result,'dnode'); $this->dst = sprintf('%s/%s', - $this->tn, - $this->tf + $this->pdn, + $this->pdf ); + $this->flags = array_get($result,'flags'); $this->cost = array_get($result,'cost'); } public function __get($k) { - return isset($this->{$k}) ? $this->{$k} : NULL; + switch ($k) + { + case 'fz': return $this->znfp($this->fqfa,'z'); + case 'fn': return $this->znfp($this->fqfa,'n'); + case 'ff': return $this->znfp($this->fqfa,'f'); + case 'fp': return $this->znfp($this->fqfa,'p'); + + case 'tz': return $this->znfp($this->fqda,'z'); + case 'tn': return $this->znfp($this->fqda,'n'); + case 'tf': return $this->znfp($this->fqda,'f'); + case 'tp': return $this->znfp($this->fqda,'p'); + + default: + return isset($this->{$k}) ? $this->{$k} : NULL; + } } public function __set($k,$v) @@ -109,6 +126,26 @@ class FTNMessage extends FTN } } + private function znfp(string $data,string $key) + { + switch ($key) { + case 'z': + return substr($data,0,strpos($data,':')); + case 'n': + $x = strpos($data,':')+1; + return substr($data,$x,strpos($data,'/')-$x); + case 'f': + $x = strpos($data,'/')+1; + return substr($data,$x,strpos($data,'.') ?: strlen($data)-$x); + case 'p': + $x = strpos($data,'.'); + return $x ? substr($data,$x+1) : 0; + + default: + abort(500,'Unknown key: '.$key); + } + } + public function parsemessage(string $message) { // Split out the lines @@ -169,7 +206,7 @@ class FTNMessage extends FTN elseif ($t = $this->kludge('INTL ',$v)) { $this->intl = $t; - list($this->fqfd,$this->fqfa) = explode(' ',$t); + list($this->fqda,$this->fqfa) = explode(' ',$t); } elseif ($t = $this->kludge('MSGID: ',$v)) @@ -244,8 +281,8 @@ class FTNMessage extends FTN { switch ($this->type()) { - case 'Echomail': return sprintf('Echomail: '.$this->echoarea); - case 'Netmail': return sprintf('Netmail: %s->%s',$this->fqfa,$this->fqfd); + case 'echomail': return sprintf('Echomail: '.$this->echoarea); + case 'netmail': return sprintf('Netmail: %s->%s',$this->fqfa,$this->fqda); default: return 'UNKNOWN'; } @@ -254,10 +291,10 @@ class FTNMessage extends FTN public function type() { if ($this->echoarea) - return 'Echomail'; + return 'echomail'; if ($this->intl) - return 'Netmail'; + return 'netmail'; return 'UNKNOWN'; } diff --git a/app/Console/Commands/FtnPkt.php b/app/Console/Commands/FtnPkt.php index 87b7f8b..8835706 100644 --- a/app/Console/Commands/FtnPkt.php +++ b/app/Console/Commands/FtnPkt.php @@ -54,11 +54,11 @@ class FtnPkt extends Command $this->warn(sprintf('-- From: %s(%s)->%s(%s), Type: %s, Size: %d, FQFA: %s', $o->from, $o->src, - $o->to, - $o->dst, - $o->description(), - strlen($o->message), - $o->fqfa + $o->to, + $o->dst, + $o->description(), + strlen($o->message), + $o->fqfa )); if ($o->unknown->count()) diff --git a/app/Console/Commands/ImportPacket.php b/app/Console/Commands/ImportPacket.php index 43aa2a8..0a4d542 100644 --- a/app/Console/Commands/ImportPacket.php +++ b/app/Console/Commands/ImportPacket.php @@ -5,13 +5,13 @@ namespace App\Console\Commands; use Carbon\Carbon; use Illuminate\Console\Command; -use App\Traits\{GetNode,ParseNodes}; +use App\Traits\{GetNode,ParseNodes,ParseZNFPDomain}; use App\Classes\FTNPacket; -use App\Models\{Echomail,Kludge,Seenby,Zone}; +use App\Models\{Echomail,Netmail,Zone}; class ImportPacket extends Command { - use GetNode,ParseNodes; + use GetNode,ParseNodes,ParseZNFPDomain; /** * The name and signature of the console command. @@ -41,6 +41,7 @@ class ImportPacket extends Command * Execute the console command. * * @return mixed + * @throws \Exception */ public function handle() { @@ -48,57 +49,133 @@ class ImportPacket extends Command foreach ($pkt->messages as $o) { - $eo = new Echomail; - - $eo->pkt_from = $this->get_node($pkt->sz,$pkt->sn,$pkt->sf,$pkt->sp)->id; - $eo->pkt_to = $this->get_node($pkt->dz,$pkt->dn,$pkt->df,$pkt->dp)->id; - $eo->pkt = $pkt->filename; - $eo->pkt_date = $pkt->date; - - $eo->flags = $o->flags; - $eo->cost = $o->cost; - $eo->from_user = $o->from; - $eo->from_ftn = $this->get_node($pkt->sz,$o->fn,$o->ff,$pkt->sp)->id; - $eo->to_user = $o->to; - $eo->subject = $o->subject; - $eo->date = Carbon::createFromFormat('d M y H:i:s',$o->date,($o->tzutc >= 0 ? '+' : '').substr_replace($o->tzutc,':',2,0)); - $eo->tz = $o->tzutc; - - $eo->area = $o->echoarea; - $eo->msgid = $o->msgid; - $eo->replyid = $o->replyid; - $eo->message = $o->message; - - $eo->origin = $o->origin; - //$eo->original = (string)$o; - - $eo->save(); - - foreach ($o->kludge as $k=>$v) + switch ($o->type()) { - $eo->kludges()->attach($k,['value'=>json_encode($v)]); - } + case 'echomail': + $date = Carbon::createFromFormat('d M y H:i:s',$o->date,($o->tzutc >= 0 ? '+' : '').substr_replace($o->tzutc,':',2,0)); - foreach ($o->unknown as $v) - { - $eo->kludges()->attach('UNKNOWN',['value'=>json_encode($v)]); - } + // See if we already have this message. + $eo = Echomail::firstOrNew([ + 'date'=>$date, + 'from_ftn'=>$this->get_node($o->fz,$o->fn,$o->ff,$o->fp)->id, + 'msgid'=>$o->msgid, + ]); - foreach ($o->seenby as $v) - { - foreach ($this->parse_nodes(Zone::findOrFail($pkt->sz),$v) as $no) - { - $eo->seenbys()->attach($no->id); - } - } + if (md5(utf8_decode($eo->message)) == md5($o->message)) + { + $this->warn(sprintf('Duplicate message: %s@%s with id: %s',$o->from,$o->fqfa,$o->msgid)); + continue; + } - $seq = 0; - foreach ($o->path as $v) - { - foreach ($this->parse_nodes(Zone::findOrFail($pkt->sz),$v) as $no) - { - $eo->paths()->attach($no->id,['sequence'=>$seq++]); - } + $eo->pkt_from = $this->get_node($pkt->sz,$pkt->sn,$pkt->sf,$pkt->sp)->id; + $eo->pkt_to = $this->get_node($pkt->dz,$pkt->dn,$pkt->df,$pkt->dp)->id; + $eo->pkt = $pkt->filename; + $eo->pkt_date = $pkt->date; + + $eo->flags = $o->flags; + $eo->cost = $o->cost; + $eo->from_user = utf8_encode($o->from); + $eo->to_user = utf8_encode($o->to); + $eo->subject = utf8_encode($o->subject); + $eo->tz = $o->tzutc; + + $eo->area = $o->echoarea; + $eo->replyid = $o->replyid; + $eo->message = utf8_encode($o->message); + + $eo->origin = utf8_encode($o->origin); + //$eo->original = (string)$o; + + $eo->save(); + + foreach ($o->kludge as $k=>$v) + { + $eo->kludges()->attach($k,['value'=>json_encode($v)]); + } + + foreach ($o->unknown as $v) + { + $eo->kludges()->attach('UNKNOWN',['value'=>json_encode($v)]); + } + + foreach ($o->seenby as $v) + { + foreach ($this->parse_nodes(Zone::findOrFail($pkt->sz),$v) as $no) + { + $eo->seenbys()->attach($no->id); + } + } + + $seq = 0; + foreach ($o->path as $v) + { + foreach ($this->parse_nodes(Zone::findOrFail($pkt->sz),$v) as $no) + { + $eo->paths()->attach($no->id,['sequence'=>$seq++]); + } + } + break; + + case 'netmail': + $date = Carbon::createFromFormat('d M y H:i:s',$o->date,($o->tzutc >= 0 ? '+' : '').substr_replace($o->tzutc,':',2,0)); + + // See if we already have this message. + $no = Netmail::firstOrNew([ + 'date'=>$date, + 'from_ftn'=>$this->get_node($o->fz,$o->fn,$o->ff,$o->fp)->id, + 'msgid'=>$o->msgid, + ]); + + if (md5(utf8_decode($no->message)) == md5($o->message)) + { + $this->warn(sprintf('Duplicate message: %s@%s with id: %s',$o->from,$o->fqfa,$o->msgid)); + //continue; + } + + $no->pkt_from = $this->get_node($pkt->sz,$pkt->sn,$pkt->sf,$pkt->sp)->id; + $no->pkt_to = $this->get_node($pkt->dz,$pkt->dn,$pkt->df,$pkt->dp)->id; + $no->pkt = $pkt->filename; + $no->pkt_date = $pkt->date; + + $no->flags = $o->flags; + $no->cost = $o->cost; + $no->from_user = utf8_encode($o->from); + $no->to_user = utf8_encode($o->to); + $no->to_ftn = $this->get_node($o->tz,$o->tn,$o->tf,$o->tp)->id; + $no->subject = utf8_encode($o->subject); + $no->tz = $o->tzutc; + + $no->replyid = $o->replyid; + $no->message = utf8_encode($o->message); + + $no->origin = utf8_encode($o->origin); + + $no->save(); + + foreach ($o->kludge as $k=>$v) + { + $no->kludges()->attach($k,['value'=>json_encode($v)]); + } + + foreach ($o->unknown as $v) + { + $no->kludges()->attach('UNKNOWN',['value'=>json_encode($v)]); + } + + $seq = 0; + foreach ($o->via as $v) + { + dump($v); + $data = preg_split('/\s/',$v); + $ftno = $this->parse_znfp_domain($data[0]); + unset($data[0]); + + $no->paths()->attach($ftno->id,['sequence'=>$seq++,'value'=>json_encode($data)]); + } + break; + + default: + abort(500,'Unknown type: '.$o->type()); } } } diff --git a/app/Models/Echomail.php b/app/Models/Echomail.php index 90112df..37f2da0 100644 --- a/app/Models/Echomail.php +++ b/app/Models/Echomail.php @@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Model; class Echomail extends Model { protected $dates = ['date']; + protected $fillable = ['date','msgid','from_ftn']; public function kludges() { diff --git a/app/Models/Netmail.php b/app/Models/Netmail.php new file mode 100644 index 0000000..eeec940 --- /dev/null +++ b/app/Models/Netmail.php @@ -0,0 +1,21 @@ +belongsToMany(Kludge::class); + } + + public function paths() + { + return $this->belongsToMany(Path::class,NULL,NULL,'node_id'); + } +} \ No newline at end of file diff --git a/app/Traits/ParseNodes.php b/app/Traits/ParseNodes.php index 832d97a..5a464d1 100644 --- a/app/Traits/ParseNodes.php +++ b/app/Traits/ParseNodes.php @@ -13,6 +13,7 @@ trait ParseNodes { $net = FALSE; $result = collect(); + foreach (explode(' ',$line) as $node) { if (preg_match('#/#',$node)) @@ -23,7 +24,7 @@ trait ParseNodes if (! $net) throw new \Exception('Missing Net?',$node); - $result->push($this->get_node($zone->id,$net,$node,0),$create); + $result->push($this->get_node($zone->id,$net,$node,0,$create)); } return $result; diff --git a/app/Traits/ParseZNFPDomain.php b/app/Traits/ParseZNFPDomain.php new file mode 100644 index 0000000..6821803 --- /dev/null +++ b/app/Traits/ParseZNFPDomain.php @@ -0,0 +1,26 @@ +get_node($z,$n,$f,$p,$create); + } +} \ No newline at end of file diff --git a/database/migrations/2019_04_26_112307_create_netmail.php b/database/migrations/2019_04_26_112307_create_netmail.php new file mode 100644 index 0000000..46c4a05 --- /dev/null +++ b/database/migrations/2019_04_26_112307_create_netmail.php @@ -0,0 +1,60 @@ +increments('id'); + $table->timestamps(); + + $table->integer('pkt_from'); + $table->integer('pkt_to'); + $table->datetime('pkt_date'); + $table->string('pkt'); + + $table->string('flags'); + $table->integer('cost'); + $table->string('from_user'); + $table->integer('from_ftn'); + $table->string('to_user'); + $table->integer('to_ftn'); + $table->string('subject'); + $table->datetime('date'); + $table->string('tz')->nullable(); + $table->string('msgid')->nullable(); + $table->string('replyid')->nullable(); + $table->text('message'); + $table->string('origin'); + $table->text('original')->nullable(); + + $table->index('pkt_from'); + $table->index('pkt_to'); + $table->index('from_ftn'); + $table->index('to_ftn'); + $table->foreign('pkt_from')->references('id')->on('nodes'); + $table->foreign('pkt_to')->references('id')->on('nodes'); + $table->foreign('from_ftn')->references('id')->on('nodes'); + $table->foreign('to_ftn')->references('id')->on('nodes'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('netmails'); + } +} diff --git a/database/migrations/2019_04_26_113128_create_netmail_tables.php b/database/migrations/2019_04_26_113128_create_netmail_tables.php new file mode 100644 index 0000000..f280f76 --- /dev/null +++ b/database/migrations/2019_04_26_113128_create_netmail_tables.php @@ -0,0 +1,50 @@ +integer('netmail_id'); + $table->string('kludge_id')->nullable(); + $table->json('value'); + + $table->index('netmail_id'); + $table->foreign('netmail_id')->references('id')->on('netmails'); + }); + + Schema::create('netmail_path', function (Blueprint $table) { + $table->integer('netmail_id'); + $table->integer('node_id'); + $table->integer('sequence'); + $table->json('value'); + + $table->unique(['netmail_id','sequence']); + + $table->index('netmail_id'); + $table->foreign('netmail_id')->references('id')->on('netmails'); + $table->index('node_id'); + $table->foreign('node_id')->references('id')->on('nodes'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('netmail_path'); + Schema::dropIfExists('kludge_netmail'); + } +}