From 3d0f28ecd0fda0501ce36db6359d0b9261ae0bea Mon Sep 17 00:00:00 2001 From: Michiel Broek Date: Sun, 14 Aug 2005 12:50:19 +0000 Subject: [PATCH] Reinstalled charset translation on the gateway --- ChangeLog | 6 ++ TODO | 2 - mbfido/Makefile | 2 +- mbfido/ftn2rfc.c | 156 +++++++++++++++++----------------------------- mbfido/newspost.c | 3 +- mbfido/rfc2ftn.c | 87 ++++++++++++++++++++++++-- mbfido/rfc2ftn.h | 3 + mbfido/scan.c | 9 +-- 8 files changed, 154 insertions(+), 114 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1aecdcbd..0078e79f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,9 +12,15 @@ v0.71.4 12-Aug-2005 Fixed a segfault when echomail.jam or netmail.jam is corrupted when scanning for outgoing mail. Fixed fido style from address for posting news. + Reinstalled charset translation on the gateway. Changed the + way how FTN kludges are translated to RFC headers. mbsebbs: Fixed headerlines for posting news. + Fixed headerlines for posting email. + + mbnntp: + Improved charset detection. v0.71.3 13-Jun-2005 - 12-Aug-2005 diff --git a/TODO b/TODO index a50f6091..0f833e7e 100644 --- a/TODO +++ b/TODO @@ -88,8 +88,6 @@ mbfido: N: After readalias we need to free the alias list when mbfido ends. - N: Reimplement characterset conversion again on the gateway. - N: Hatch statistics are not updated. N: Copy personal mail to a personal message area. diff --git a/mbfido/Makefile b/mbfido/Makefile index 66af9d49..15f5e6f1 100644 --- a/mbfido/Makefile +++ b/mbfido/Makefile @@ -156,7 +156,7 @@ mbfrearc.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h mbfutil. utic.o: ../config.h ../lib/mbselib.h tic.h mover.h tic.h utic.h announce.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h ../lib/msg.h ../lib/msgtext.h ../lib/diesel.h grlist.h msgutil.h toberep.h announce.h fflist.o: ../config.h ../lib/mbselib.h ../lib/msg.h fflist.h -ftn2rfc.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h rollover.h aliasdb.h postemail.h backalias.h msgflags.h ftn2rfc.h +ftn2rfc.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h rollover.h aliasdb.h postemail.h backalias.h msgflags.h rfc2ftn.h ftn2rfc.h makestat.o: ../config.h ../lib/mbselib.h ../lib/diesel.h ../lib/msg.h mgrutil.h makestat.h mbindex.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h mbindex.h msgutil.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h ../lib/msg.h ../lib/msgtext.h ../lib/diesel.h msgutil.h diff --git a/mbfido/ftn2rfc.c b/mbfido/ftn2rfc.c index 502d39b4..25cba3eb 100644 --- a/mbfido/ftn2rfc.c +++ b/mbfido/ftn2rfc.c @@ -37,6 +37,7 @@ #include "postemail.h" #include "backalias.h" #include "msgflags.h" +#include "rfc2ftn.h" #include "ftn2rfc.h" @@ -268,8 +269,8 @@ void Send(int newsmode, char *outstr) char *p; unsigned long crc; - fwrite(outstr, 1, strlen(outstr), nfp); -// Syslog('m', "+ %s\n", printable(outstr, 0)); + charwrite(outstr, nfp); + Syslog('m', "+ %s", printable(outstr, 0)); if (newsmode) { Striplf(outstr); @@ -302,13 +303,10 @@ void Send(int newsmode, char *outstr) */ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int flags, FILE *pkt) { - int rrq, result = 1, modtype = 0; - int waskludge = FALSE, badkludge; - int bNeedToGetAddressFromMsgid = (int)NULL; - int newsmode = 0, lines, pass, count, first; - char *newsgroup = NULL, *distribution = NULL, *moderator = NULL; - char *temp, *p, *q, *r, *l, *b; - char *To = NULL, buf[4096], c; + int rrq, result = 1, waskludge = FALSE, badkludge; + int bNeedToGetAddressFromMsgid = (int)NULL, newsmode = 0, lines, pass, count, first; + char *newsgroup = NULL, *charset = NULL; + char *temp, *p, *q, *r, *l, *b, *To = NULL, buf[4096], c, MailFrom[128], MailTo[128]; time_t now; rfcmsg *kmsg = NULL, **tmsg, *qmsg, *msg = NULL; off_t endmsg_off, tear_off, orig_off, via_off; @@ -316,7 +314,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl FILE *fp; fa_list *rlist, *tfa, *ftnpath = NULL; struct utsname utsbuf; - char MailFrom[128], MailTo[128]; temp = calloc(32768, sizeof(char)); tmsg = &kmsg; @@ -421,7 +418,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl (*tmsg)->val=xstrcpy(buf+4); tmsg=&((*tmsg)->next); } - Syslog('M', "tearline \"%s\" at offset %ld", buf,(long)tear_off); } else if (!strncmp(buf," * Origin:",10)) { orig_off = ftell(fp); *tmsg = (rfcmsg *)malloc(sizeof(rfcmsg)); @@ -506,42 +502,45 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl endmsg_off = orig_off; if ((via_off) && (via_off < endmsg_off)) endmsg_off = via_off; - Syslog('M', "end message offset %ld",(long)endmsg_off); rewind(fp); msg = parsrfc(fp); bestaka = bestaka_s(f); rewind(fp); + /* + * Get characterset encoding from kludge or rfc header + */ p = hdr((char *)"CHRS", kmsg); if (p == NULL) p = hdr((char *)"CHARSET", kmsg); if (p == NULL) p = hdr((char *)"CODEPAGE", kmsg); - if (p) -// outcode = readchrs(p); - Syslog('m', "outcode from ftn kludge: %s", printable(p, 0)); - else { + if (p) { + while (*p && isspace(*p)) + p++; + Striplf(p); + charset = xstrcpy(p); + Syslog('m', "Charset from ftn \"%s\"", printable(charset, 0)); + } else { p=hdr((char *)"Content-Type",msg); if (p == NULL) p=hdr((char *)"RFC-Content-Type",kmsg); if (p == NULL) p=hdr((char *)"Content-Type",kmsg); - if (p) -// outcode=readcharset(p); - Syslog('m', "outcode from rfc header: %s", printable(p, 0)); - else { -// q = rfcmsgid(hdr((char *)"MSGID",kmsg),bestaka); -// if ((hdr((char *)"Message-ID",msg)) || (hdr((char *)"RFC-Message-ID",kmsg)) || -// (hdr((char *)"Message-ID",kmsg)) || (hdr((char *)"RFCID",kmsg)) || -// (hdr((char *)"ORIGID",kmsg)) || ((hdr((char *)"MSGID",kmsg)) && (!chkftnmsgid(q)))) -// outcode = defaultrfcchar; -// else -// outcode = defaultftnchar; -// if (q) -// free(q); -// q = NULL; - Syslog('m', "outcode will use default"); + if (p) { + q = strtok(p, " \n\0"); + q = strtok(NULL, " \n\0"); + while (*q && isspace(*q)) + q++; + Syslog('m', "charset part: %s", printable(q, 0)); + if (q && (strncasecmp(q, "charset=", 8) == 0)) { + charset = xstrcpy(q + 8); + Syslog('m', "Charset from rfc \"%s\"", printable(charset, 0)); + } + } else { + charset = xstrcpy((char *)"iso-8859-1"); + Syslog('m', "No charset, setting default to iso-8859-1"); } } @@ -550,33 +549,12 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl * The msgs record is already loaded. */ newsgroup = xstrcpy(msgs.Newsgroup); - if (strlen(msgs.Distribution)) - distribution = xstrcpy(msgs.Distribution); - Syslog('M', "newsgroup %s, distribution %s", printable(newsgroup, 0), printable(distribution, 0)); - newsmode = TRUE; - if ((modtype == 1) && (!hdr((char *)"Approved",msg)) && - (!hdr((char *)"RFC-Approved",kmsg)) && (!hdr((char *)"Approved",kmsg))) - newsmode = TRUE; + Syslog('M', "newsgroup %s", printable(newsgroup, 0)); + newsmode = TRUE; } else newsmode = FALSE; Syslog('m', "Got %s message", newsmode?"echo":"netmail"); -// if ((outcode == CHRS_NOTSET) && (hdr((char *)"MSGID", kmsg))) { -// p = rfcmsgid(hdr((char *)"MSGID",kmsg),bestaka); -// if ((hdr((char *)"Message-ID",msg)) || (hdr((char *)"RFC-Message-ID",kmsg)) || -// (hdr((char *)"Message-ID",kmsg)) || (hdr((char *)"RFCID",kmsg)) || -// (hdr((char *)"ORIGID",kmsg)) || ((hdr((char *)"MSGID",kmsg)) && (!chkftnmsgid(p)))) -// outcode = defaultrfcchar; -// else -// outcode = defaultftnchar; -// free(p); -// } -// if (pgpsigned) -// incode = outcode; -// else if (incode == CHRS_NOTSET) -// incode = getincode(outcode); - - /* * fsc-0038 defines "^aDOMAIN: othernet 99:12/34 fidonet 2:293/2219" */ @@ -669,9 +647,7 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl strncpy(buf, p, sizeof(buf) -1); if (*(p = buf + strlen(buf) -1) == '\n') *p='\0'; - } else if (modtype == 1) - sprintf(buf,"%s",moderator); - else + } else sprintf(buf,"%s",ascinode(t,0x7f)); substitute(buf); Syslog('+', "mail from %s to %s",ascfnode(f,0x7f),buf); @@ -712,11 +688,13 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl } Syslog('m', "Prepare is ready"); - - if (modtype == 1) - newsmode = TRUE; } + /* + * Setup charset conversion + */ + charset_set_in_out(charset, getrfcchrs(msgs.Charset)); + if (newsmode) { /* * Open temporary newsfile, append messages if it already exists. @@ -811,20 +789,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl Send(newsmode, temp); } - if ((p=hdr((char *)"Distribution",msg))) { - sprintf(temp,"Distribution:%s",p); - Send(newsmode, temp); - } else if ((p=hdr((char *)"RFC-Distribution",kmsg))) { - sprintf(temp,"Distribution: %s",p); - Send(newsmode, temp); - } else if ((p=hdr((char *)"Distribution",kmsg))) { - sprintf(temp,"Distribution: %s",p); - Send(newsmode, temp); - } else if (distribution) { - sprintf(temp,"Distribution: %s\n",distribution); - Send(newsmode, temp); - } - p = hdr((char *)"Comment-To",msg); if (p == NULL) p=hdr((char *)"X-Comment-To",msg); @@ -861,9 +825,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl } else if ((p=hdr((char *)"Approved",kmsg))) { sprintf(temp,"Approved: %s",p); Send(newsmode, temp); - } else if (modtype==2) { - sprintf(temp,"Approved: %s\n",moderator); - Send(newsmode, temp); } } else { /* if newsmode */ @@ -931,9 +892,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl Syslog('m', "2"); sprintf(temp,"To: %s\n",p); Send(FALSE, temp); - } else if (modtype == 1) { - sprintf(temp,"To: %s\n",moderator); - Send(FALSE, temp); } else if (is_local(t)) { Syslog('m', "3"); sprintf(temp, "To: %s <%s>\n", t->name, buf); @@ -1225,14 +1183,20 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl Send(newsmode, temp); } + /* + * We have setup translation, send the right charset name + */ if ((p=hdr((char *)"Content-Type",msg))) { - sprintf(temp,"Content-Type:%s",p); + sprintf(temp, "Content-Type: text/plain; charset=%s\n", getrfcchrs(msgs.Charset)); +// sprintf(temp,"Content-Type:%s",p); Send(newsmode, temp); } else if ((p=hdr((char *)"RFC-Content-Type",kmsg))) { - sprintf(temp,"Content-Type: %s",p); + sprintf(temp, "Content-Type: text/plain; charset=%s\n", getrfcchrs(msgs.Charset)); +// sprintf(temp,"Content-Type: %s",p); Send(newsmode, temp); } else if ((p=hdr((char *)"Content-Type",kmsg))) { - sprintf(temp,"Content-Type: %s",p); + sprintf(temp, "Content-Type: text/plain; charset=%s\n", getrfcchrs(msgs.Charset)); +// sprintf(temp,"Content-Type: %s",p); Send(newsmode, temp); } @@ -1247,15 +1211,16 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl Send(newsmode, temp); } - temp[0] = '\0'; - if ((p=hdr((char *)"Content-Transfer-Encoding",msg))) + if ((p=hdr((char *)"Content-Transfer-Encoding",msg))) { sprintf(temp,"Content-Transfer-Encoding:%s",p); - else if ((p=hdr((char *)"RFC-Content-Transfer-Encoding",kmsg))) - sprintf(temp,"Content-Transfer-Encoding: %s",p); - else if ((p=hdr((char *)"Content-Transfer-Encoding",kmsg))) - sprintf(temp,"Content-Transfer-Encoding: %s",p); - if (temp[0]) Send(newsmode, temp); + } else if ((p=hdr((char *)"RFC-Content-Transfer-Encoding",kmsg))) { + sprintf(temp,"Content-Transfer-Encoding: %s",p); + Send(newsmode, temp); + } else if ((p=hdr((char *)"Content-Transfer-Encoding",kmsg))) { + sprintf(temp,"Content-Transfer-Encoding: %s",p); + Send(newsmode, temp); + } if (newsmode) { if ((p=hdr((char *)"X-Newsreader",msg))) { @@ -1322,7 +1287,7 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl strcasecmp(qmsg->key,"Supersedes") && strcasecmp(qmsg->key,"Mime-Version") && strcasecmp(qmsg->key,"Content-Type") && - strcasecmp(qmsg->key,"Content-Lenght") && + strcasecmp(qmsg->key,"Content-Length") && strcasecmp(qmsg->key,"Content-Transfer-Encoding") && strcasecmp(qmsg->key,"Lines") && strcasecmp(qmsg->key,"Path") && @@ -1558,9 +1523,8 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl } Syslog('m', "End sending message body"); - if ((modtype==1) && (!hdr((char *)"Approved",msg)) && - (!hdr((char *)"RFC-Approved",kmsg)) && (!hdr((char *)"Approved",kmsg))) - newsmode = FALSE; + if (charset) + free(charset); tidyrfc(msg); fclose(fp); @@ -1580,10 +1544,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl if (newsgroup) free(newsgroup); - if (distribution) - free(distribution); - if (moderator) - free(moderator); rbuf = NULL; free(temp); return result; diff --git a/mbfido/newspost.c b/mbfido/newspost.c index 5c89b9fd..1a815f8c 100644 --- a/mbfido/newspost.c +++ b/mbfido/newspost.c @@ -79,7 +79,7 @@ int newspost(void) while (fgets(buf, 10240, nfp)) { if (start) { if (nntp_cmd((char *)"POST\r\n", 340) != 0) { - WriteError("NNTP POST refused"); + WriteError("NNTP: POST refused"); free(buf); return TRUE; } @@ -88,6 +88,7 @@ int newspost(void) if (!strcmp(buf, ".\n")) { if (nntp_cmd((char *)".\r\n", 240) == 0) { news_out++; + Syslog('+', "NTTP: article %d accepted", news_out); } else { WriteError("NNTP: refused article %d", news_out+1); news_bad++; diff --git a/mbfido/rfc2ftn.c b/mbfido/rfc2ftn.c index 905ebe43..51ae280c 100644 --- a/mbfido/rfc2ftn.c +++ b/mbfido/rfc2ftn.c @@ -59,6 +59,7 @@ static int removeref; static int removeinreply; static int removereplyto; static int removereturnto; +char currentgroup[81]; @@ -80,7 +81,6 @@ int needputrfc(rfcmsg *, int); -int charwrite(char *, FILE *); int charwrite(char *s, FILE *fp) { char *o; @@ -126,6 +126,65 @@ int kludgewrite(char *s, FILE *fp) +/* + * Build a faked RFC msgid, use the CRC32 of the FTN msgid, + * the current group and the configured system's fqdn. This + * gives a unique string specific for the message. + */ +char *make_msgid(char *msgid) +{ + static char buf[100]; + + sprintf(buf, "<%8lx$%s@%s>", StringCRC32(msgid), currentgroup, CFG.sysdomain); + return buf; +} + + + +int findorigmsg(char *msgid, char *o) +{ + unsigned long i, start, end; + char *gen2; + + if (msgid == NULL) { + return 0; + } + + if (!Msg_Open(msgs.Base)) { + return 0; + } + Msg_Number(); + Msg_Highest(); + Msg_Lowest(); + + if (MsgBase.Open == FALSE) { + Syslog('-', "Base closed"); + return 0; + } + + strcpy(currentgroup,msgs.Newsgroup); + start = MsgBase.Lowest; + end = MsgBase.Highest; + + gen2 = calloc(strlen(msgid)+1,sizeof(char)); + strcpy(gen2, strchr(msgid,'<')); + for (i = start; i <= end; i++) { + if (Msg_ReadHeader(i)) { + if (strncmp(gen2,make_msgid(Msg.Msgid),strlen(gen2)-1) == 0) { + Syslog('m',"Found msgid: %s",make_msgid(Msg.Msgid)); +// realloc(o,(strlen(Msg.Msgid)+1)* sizeof(char)); + strcpy(o,Msg.Msgid); + free(gen2); + return 1; + } + } + } + free(gen2); + return 0; +} + + + /* * Input a RFC message. */ @@ -159,9 +218,10 @@ int rfc2ftn(FILE *fp, faddr *recipient) newsmode = hdr((char *)"Newsgroups", msg) ?TRUE:FALSE; Syslog('m', "RFC message is %s", newsmode ? "news article":"e-mail message"); - if (newsmode) + if (newsmode) { news_in++; - else + sprintf(currentgroup, "%s", msgs.Newsgroup); + } else email_in++; if (!CFG.allowcontrol) { @@ -190,6 +250,16 @@ int rfc2ftn(FILE *fp, faddr *recipient) if ((p = hdr((char *)"References",msg))) { p = strrchr(p,' '); ftnmsgid(p,&fmsg->reply_a, &fmsg->reply_n,fmsg->area); + +//Griffin + fmsg->reply_s=calloc(256,sizeof(char)); + findorigmsg(p,fmsg->reply_s); + + fmsg->to->name=calloc(strlen(Msg.From)+1,sizeof(char)); + strcpy(fmsg->to->name,Msg.From); + Syslog('m', "fmsg to-name %s",fmsg->to->name); + Syslog('m', "reply_s %s",fmsg->reply_s); + if (!chkftnmsgid(p)) { hash_update_s(&fmsg->reply_n, fmsg->area); } @@ -238,7 +308,16 @@ int rfc2ftn(FILE *fp, faddr *recipient) q++; Syslog('m', "charset part: %s", printable(q, 0)); if (q && (strncasecmp(q, "charset=", 8) == 0)) { - charset = xstrcpy(q + 8); + /* + * google.com quotes the charset name + */ + if (strchr(q, '"')) { + charset = xstrcpy(q + 9); + charset[strlen(charset)-1] = '\0'; + Syslog('m', "Unquoted charset name"); + } else { + charset = xstrcpy(q + 8); + } Syslog('m', "Charset \"%s\"", printable(charset, 0)); } } diff --git a/mbfido/rfc2ftn.h b/mbfido/rfc2ftn.h index 77e0f74e..8643ba55 100644 --- a/mbfido/rfc2ftn.h +++ b/mbfido/rfc2ftn.h @@ -1,6 +1,9 @@ #ifndef _RFC2FTN_H #define _RFC2FTN_H +/* $Id$ */ + +int charwrite(char *, FILE *); int rfc2ftn(FILE *fp, faddr *); #endif diff --git a/mbfido/scan.c b/mbfido/scan.c index bbcff9fa..79ed0647 100644 --- a/mbfido/scan.c +++ b/mbfido/scan.c @@ -695,7 +695,7 @@ void ExportNews(unsigned long MsgNum, fa_list **sbl) Syslog('m', "Msg.From %s", Msg.From); Syslog('m', "Msg.FromAddress %s", Msg.FromAddress); Syslog('m', "Msg.To %s", Msg.To); - Syslog('m', "Msg.ToAddress", Msg.ToAddress); + Syslog('m', "Msg.ToAddress %s", Msg.ToAddress); flags |= (Msg.Private) ? M_PVT : 0; from = fido2faddr(msgs.Aka); @@ -717,7 +717,6 @@ void ExportNews(unsigned long MsgNum, fa_list **sbl) dest = NULL; fprintf(qp, "AREA:%s\n", msgs.Tag); - Syslog('m', "AREA:%s", msgs.Tag); if (Msg_Read(MsgNum, 79)) { if ((p = (char *)MsgText_First()) != NULL) { @@ -730,18 +729,13 @@ void ExportNews(unsigned long MsgNum, fa_list **sbl) kludges = FALSE; fprintf(qp, "\001TID: MBSE-FIDO %s (%s-%s)\n", VERSION, OsName(), OsCPU()); fprintf(qp, "Subject: %s\n", Msg.Subject); - Syslog('m', "Subject: %s", Msg.Subject); fprintf(qp, "\n"); - Syslog('m', "\n"); fprintf(qp, "%s\n", p); - Syslog('m', "%s", p); } else { fprintf(qp, "%s\n", p+1); - Syslog('m', "%s", p+1); } } else { fprintf(qp, "%s", p); - Syslog('m', "%s", printable(p, 0)); if (strncmp(p, " * Origin:", 10) == 0) break; @@ -775,7 +769,6 @@ void ExportNews(unsigned long MsgNum, fa_list **sbl) fprintf(qp, "%s", sbe); } fprintf(qp, "\n\001PATH: %u/%u\n", msgs.Aka.net, msgs.Aka.node); - Syslog('m', "\\001PATH: %u/%u", msgs.Aka.net, msgs.Aka.node); rewind(qp); most_debug = TRUE;