Initial netmail import

app/Classes/FTNMessage.php

@@ -31,7 +31,7 @@ class FTNMessage extends FTN
 	private $unknown = [];
 
 	private $fqfa = NULL;       // Fully qualified fidonet source where packet originated
-	private $fqfd = NULL;       // Fully qualified fidonet destination address (Netmail)
+	private $fqda = NULL;       // Fully qualified fidonet destination address (Netmail)
 
 	// Single value kludge items
 	private $_kludge = [
@@ -65,29 +65,46 @@ class FTNMessage extends FTN
 
 		$result = unpack($this->unpackheader($struct),$header);
 
-		$this->fn = array_get($result,'onet');
+		// For Echomail this is the packet src.
-		$this->ff = array_get($result,'onode');
+		$this->psn = array_get($result,'onet');
+		$this->psf = array_get($result,'onode');
 
 		$this->src = sprintf('%s/%s',
-			$this->fn,
+			$this->psn,
-			$this->ff
+			$this->psf
 		);
 
-		$this->tn = array_get($result,'dnet');
+		// For Echomail this is the packet dst.
-		$this->tf = array_get($result,'dnode');
+		$this->pdn = array_get($result,'dnet');
+		$this->pdf = array_get($result,'dnode');
 
 		$this->dst = sprintf('%s/%s',
-			$this->tn,
+			$this->pdn,
-			$this->tf
+			$this->pdf
 		);
+
 		$this->flags = array_get($result,'flags');
 		$this->cost = array_get($result,'cost');
 	}
 
 	public function __get($k)
 	{
+		switch ($k)
+		{
+			case 'fz': return $this->znfp($this->fqfa,'z');
+			case 'fn': return $this->znfp($this->fqfa,'n');
+			case 'ff': return $this->znfp($this->fqfa,'f');
+			case 'fp': return $this->znfp($this->fqfa,'p');
+
+			case 'tz': return $this->znfp($this->fqda,'z');
+			case 'tn': return $this->znfp($this->fqda,'n');
+			case 'tf': return $this->znfp($this->fqda,'f');
+			case 'tp': return $this->znfp($this->fqda,'p');
+
+			default:
 				return isset($this->{$k}) ? $this->{$k} : NULL;
 		}
+	}
 
 	public function __set($k,$v)
 	{
@@ -109,6 +126,26 @@ class FTNMessage extends FTN
 		}
 	}
 
+	private function znfp(string $data,string $key)
+	{
+		switch ($key) {
+			case 'z':
+				return substr($data,0,strpos($data,':'));
+			case 'n':
+				$x = strpos($data,':')+1;
+				return substr($data,$x,strpos($data,'/')-$x);
+			case 'f':
+				$x = strpos($data,'/')+1;
+				return substr($data,$x,strpos($data,'.') ?: strlen($data)-$x);
+			case 'p':
+				$x = strpos($data,'.');
+				return $x ? substr($data,$x+1) : 0;
+
+			default:
+				abort(500,'Unknown key: '.$key);
+		}
+	}
+
 	public function parsemessage(string $message)
 	{
 		// Split out the <SOH> lines
@@ -169,7 +206,7 @@ class FTNMessage extends FTN
 			elseif ($t = $this->kludge('INTL ',$v))
 			{
 				$this->intl = $t;
-				list($this->fqfd,$this->fqfa) = explode(' ',$t);
+				list($this->fqda,$this->fqfa) = explode(' ',$t);
 			}
 
 			elseif ($t = $this->kludge('MSGID: ',$v))
@@ -244,8 +281,8 @@ class FTNMessage extends FTN
 	{
 		switch ($this->type())
 		{
-			case 'Echomail': return sprintf('Echomail: '.$this->echoarea);
+			case 'echomail': return sprintf('Echomail: '.$this->echoarea);
-			case 'Netmail': return sprintf('Netmail: %s->%s',$this->fqfa,$this->fqfd);
+			case 'netmail': return sprintf('Netmail: %s->%s',$this->fqfa,$this->fqda);
 			default:
 				return 'UNKNOWN';
 		}
@@ -254,10 +291,10 @@ class FTNMessage extends FTN
 	public function type()
 	{
 		if ($this->echoarea)
-			return 'Echomail';
+			return 'echomail';
 
 		if ($this->intl)
-			return 'Netmail';
+			return 'netmail';
 
 		return 'UNKNOWN';
 	}

app/Console/Commands/ImportPacket.php

@@ -5,13 +5,13 @@ namespace App\Console\Commands;
 use Carbon\Carbon;
 use Illuminate\Console\Command;
 
-use App\Traits\{GetNode,ParseNodes};
+use App\Traits\{GetNode,ParseNodes,ParseZNFPDomain};
 use App\Classes\FTNPacket;
-use App\Models\{Echomail,Kludge,Seenby,Zone};
+use App\Models\{Echomail,Netmail,Zone};
 
 class ImportPacket extends Command
 {
-	use GetNode,ParseNodes;
+	use GetNode,ParseNodes,ParseZNFPDomain;
 
 	/**
 	 * The name and signature of the console command.
@@ -41,6 +41,7 @@ class ImportPacket extends Command
 	 * Execute the console command.
 	 *
 	 * @return mixed
+	 * @throws \Exception
 	 */
 	public function handle()
 	{
@@ -48,7 +49,23 @@ class ImportPacket extends Command
 
 		foreach ($pkt->messages as $o)
 		{
-			$eo = new Echomail;
+			switch ($o->type())
+			{
+				case 'echomail':
+					$date = Carbon::createFromFormat('d M y  H:i:s',$o->date,($o->tzutc >= 0 ? '+' : '').substr_replace($o->tzutc,':',2,0));
+
+					// See if we already have this message.
+					$eo = Echomail::firstOrNew([
+						'date'=>$date,
+						'from_ftn'=>$this->get_node($o->fz,$o->fn,$o->ff,$o->fp)->id,
+						'msgid'=>$o->msgid,
+					]);
+
+					if (md5(utf8_decode($eo->message)) == md5($o->message))
+					{
+						$this->warn(sprintf('Duplicate message: %s@%s with id: %s',$o->from,$o->fqfa,$o->msgid));
+						continue;
+					}
 
 					$eo->pkt_from = $this->get_node($pkt->sz,$pkt->sn,$pkt->sf,$pkt->sp)->id;
 					$eo->pkt_to = $this->get_node($pkt->dz,$pkt->dn,$pkt->df,$pkt->dp)->id;
@@ -57,19 +74,16 @@ class ImportPacket extends Command
 
 					$eo->flags = $o->flags;
 					$eo->cost = $o->cost;
-			$eo->from_user = $o->from;
+					$eo->from_user = utf8_encode($o->from);
-			$eo->from_ftn = $this->get_node($pkt->sz,$o->fn,$o->ff,$pkt->sp)->id;
+					$eo->to_user = utf8_encode($o->to);
-			$eo->to_user = $o->to;
+					$eo->subject = utf8_encode($o->subject);
-			$eo->subject = $o->subject;
-			$eo->date = Carbon::createFromFormat('d M y  H:i:s',$o->date,($o->tzutc >= 0 ? '+' : '').substr_replace($o->tzutc,':',2,0));
 					$eo->tz = $o->tzutc;
 
 					$eo->area = $o->echoarea;
-			$eo->msgid = $o->msgid;
 					$eo->replyid = $o->replyid;
-			$eo->message = $o->message;
+					$eo->message = utf8_encode($o->message);
 
-			$eo->origin = $o->origin;
+					$eo->origin = utf8_encode($o->origin);
 					//$eo->original = (string)$o;
 
 					$eo->save();
@@ -100,6 +114,69 @@ class ImportPacket extends Command
 							$eo->paths()->attach($no->id,['sequence'=>$seq++]);
 						}
 					}
+				break;
+
+				case 'netmail':
+					$date = Carbon::createFromFormat('d M y  H:i:s',$o->date,($o->tzutc >= 0 ? '+' : '').substr_replace($o->tzutc,':',2,0));
+
+					// See if we already have this message.
+					$no = Netmail::firstOrNew([
+						'date'=>$date,
+						'from_ftn'=>$this->get_node($o->fz,$o->fn,$o->ff,$o->fp)->id,
+						'msgid'=>$o->msgid,
+					]);
+
+					if (md5(utf8_decode($no->message)) == md5($o->message))
+					{
+						$this->warn(sprintf('Duplicate message: %s@%s with id: %s',$o->from,$o->fqfa,$o->msgid));
+						//continue;
+					}
+
+					$no->pkt_from = $this->get_node($pkt->sz,$pkt->sn,$pkt->sf,$pkt->sp)->id;
+					$no->pkt_to = $this->get_node($pkt->dz,$pkt->dn,$pkt->df,$pkt->dp)->id;
+					$no->pkt = $pkt->filename;
+					$no->pkt_date = $pkt->date;
+
+					$no->flags = $o->flags;
+					$no->cost = $o->cost;
+					$no->from_user = utf8_encode($o->from);
+					$no->to_user = utf8_encode($o->to);
+					$no->to_ftn = $this->get_node($o->tz,$o->tn,$o->tf,$o->tp)->id;
+					$no->subject = utf8_encode($o->subject);
+					$no->tz = $o->tzutc;
+
+					$no->replyid = $o->replyid;
+					$no->message = utf8_encode($o->message);
+
+					$no->origin = utf8_encode($o->origin);
+
+					$no->save();
+
+					foreach ($o->kludge as $k=>$v)
+					{
+						$no->kludges()->attach($k,['value'=>json_encode($v)]);
+					}
+
+					foreach ($o->unknown as $v)
+					{
+						$no->kludges()->attach('UNKNOWN',['value'=>json_encode($v)]);
+					}
+
+					$seq = 0;
+					foreach ($o->via as $v)
+					{
+						dump($v);
+						$data = preg_split('/\s/',$v);
+						$ftno = $this->parse_znfp_domain($data[0]);
+						unset($data[0]);
+
+						$no->paths()->attach($ftno->id,['sequence'=>$seq++,'value'=>json_encode($data)]);
+					}
+					break;
+
+				default:
+					abort(500,'Unknown type: '.$o->type());
+			}
 		}
 	}
 }

app/Models/Echomail.php

@@ -7,6 +7,7 @@ use Illuminate\Database\Eloquent\Model;
 class Echomail extends Model
 {
 	protected $dates = ['date'];
+	protected $fillable = ['date','msgid','from_ftn'];
 
 	public function kludges()
 	{

app/Models/Netmail.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+
+class Netmail extends Model
+{
+	protected $dates = ['date'];
+	protected $fillable = ['date','msgid','from_ftn'];
+
+	public function kludges()
+	{
+		return $this->belongsToMany(Kludge::class);
+	}
+
+	public function paths()
+	{
+		return $this->belongsToMany(Path::class,NULL,NULL,'node_id');
+	}
+}

app/Traits/ParseNodes.php

@@ -13,6 +13,7 @@ trait ParseNodes
 	{
 		$net = FALSE;
 		$result = collect();
+
 		foreach (explode(' ',$line) as $node)
 		{
 			if (preg_match('#/#',$node))
@@ -23,7 +24,7 @@ trait ParseNodes
 			if (! $net)
 				throw new \Exception('Missing Net?',$node);
 
-			$result->push($this->get_node($zone->id,$net,$node,0),$create);
+			$result->push($this->get_node($zone->id,$net,$node,0,$create));
 		}
 
 		return $result;

app/Traits/ParseZNFPDomain.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace App\Traits;
+
+trait ParseZNFPDomain
+{
+	private function parse_znfp_domain(string $data,$create=TRUE)
+	{
+		$z = substr($data,0,strpos($data,':'));
+
+		$x = strpos($data,':')+1;
+		$n = substr($data,$x,strpos($data,'/')-$x);
+
+		$x = strpos($data,'/')+1;
+		$f = substr($data,$x,(strpos($data,'.') OR strpos($data,'@')) ?: strlen($data)-$x);
+
+		$x = strpos($data,'.');
+		$p = $x ? substr($data,$x+1,strpos($data,'@') ?: strlen($data)-$x) : 0;
+
+		// @todo We dont handle domain yet.
+		$x = strpos($data,'@');
+		$d = $x ? substr($data,$x+1) : 0;
+
+		return $this->get_node($z,$n,$f,$p,$create);
+	}
+}

database/migrations/2019_04_26_112307_create_netmail.php

@@ -0,0 +1,60 @@
+<?php
+
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class CreateNetmail extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('netmails', function (Blueprint $table) {
+            $table->increments('id');
+            $table->timestamps();
+
+	        $table->integer('pkt_from');
+	        $table->integer('pkt_to');
+	        $table->datetime('pkt_date');
+	        $table->string('pkt');
+
+	        $table->string('flags');
+	        $table->integer('cost');
+	        $table->string('from_user');
+	        $table->integer('from_ftn');
+	        $table->string('to_user');
+	        $table->integer('to_ftn');
+	        $table->string('subject');
+	        $table->datetime('date');
+	        $table->string('tz')->nullable();
+	        $table->string('msgid')->nullable();
+	        $table->string('replyid')->nullable();
+	        $table->text('message');
+	        $table->string('origin');
+	        $table->text('original')->nullable();
+
+	        $table->index('pkt_from');
+	        $table->index('pkt_to');
+	        $table->index('from_ftn');
+	        $table->index('to_ftn');
+	        $table->foreign('pkt_from')->references('id')->on('nodes');
+	        $table->foreign('pkt_to')->references('id')->on('nodes');
+	        $table->foreign('from_ftn')->references('id')->on('nodes');
+	        $table->foreign('to_ftn')->references('id')->on('nodes');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('netmails');
+    }
+}

database/migrations/2019_04_26_113128_create_netmail_tables.php

@@ -0,0 +1,50 @@
+<?php
+
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class CreateNetmailTables extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('kludge_netmail', function (Blueprint $table) {
+            $table->integer('netmail_id');
+            $table->string('kludge_id')->nullable();
+            $table->json('value');
+
+            $table->index('netmail_id');
+            $table->foreign('netmail_id')->references('id')->on('netmails');
+        });
+
+        Schema::create('netmail_path', function (Blueprint $table) {
+            $table->integer('netmail_id');
+            $table->integer('node_id');
+            $table->integer('sequence');
+	        $table->json('value');
+
+            $table->unique(['netmail_id','sequence']);
+
+            $table->index('netmail_id');
+            $table->foreign('netmail_id')->references('id')->on('netmails');
+            $table->index('node_id');
+            $table->foreign('node_id')->references('id')->on('nodes');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('netmail_path');
+        Schema::dropIfExists('kludge_netmail');
+    }
+}