Reinstalled charset translation on the gateway

This commit is contained in:
Michiel Broek 2005-08-14 12:50:19 +00:00
parent 7313533a2c
commit 3d0f28ecd0
8 changed files with 154 additions and 114 deletions

View File

@ -12,9 +12,15 @@ v0.71.4 12-Aug-2005
Fixed a segfault when echomail.jam or netmail.jam is corrupted
when scanning for outgoing mail.
Fixed fido style from address for posting news.
Reinstalled charset translation on the gateway. Changed the
way how FTN kludges are translated to RFC headers.
mbsebbs:
Fixed headerlines for posting news.
Fixed headerlines for posting email.
mbnntp:
Improved charset detection.
v0.71.3 13-Jun-2005 - 12-Aug-2005

2
TODO
View File

@ -88,8 +88,6 @@ mbfido:
N: After readalias we need to free the alias list when mbfido ends.
N: Reimplement characterset conversion again on the gateway.
N: Hatch statistics are not updated.
N: Copy personal mail to a personal message area.

View File

@ -156,7 +156,7 @@ mbfrearc.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h mbfutil.
utic.o: ../config.h ../lib/mbselib.h tic.h mover.h tic.h utic.h
announce.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h ../lib/msg.h ../lib/msgtext.h ../lib/diesel.h grlist.h msgutil.h toberep.h announce.h
fflist.o: ../config.h ../lib/mbselib.h ../lib/msg.h fflist.h
ftn2rfc.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h rollover.h aliasdb.h postemail.h backalias.h msgflags.h ftn2rfc.h
ftn2rfc.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h rollover.h aliasdb.h postemail.h backalias.h msgflags.h rfc2ftn.h ftn2rfc.h
makestat.o: ../config.h ../lib/mbselib.h ../lib/diesel.h ../lib/msg.h mgrutil.h makestat.h
mbindex.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h mbindex.h
msgutil.o: ../config.h ../lib/mbselib.h ../lib/users.h ../lib/mbsedb.h ../lib/msg.h ../lib/msgtext.h ../lib/diesel.h msgutil.h

View File

@ -37,6 +37,7 @@
#include "postemail.h"
#include "backalias.h"
#include "msgflags.h"
#include "rfc2ftn.h"
#include "ftn2rfc.h"
@ -268,8 +269,8 @@ void Send(int newsmode, char *outstr)
char *p;
unsigned long crc;
fwrite(outstr, 1, strlen(outstr), nfp);
// Syslog('m', "+ %s\n", printable(outstr, 0));
charwrite(outstr, nfp);
Syslog('m', "+ %s", printable(outstr, 0));
if (newsmode) {
Striplf(outstr);
@ -302,13 +303,10 @@ void Send(int newsmode, char *outstr)
*/
int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int flags, FILE *pkt)
{
int rrq, result = 1, modtype = 0;
int waskludge = FALSE, badkludge;
int bNeedToGetAddressFromMsgid = (int)NULL;
int newsmode = 0, lines, pass, count, first;
char *newsgroup = NULL, *distribution = NULL, *moderator = NULL;
char *temp, *p, *q, *r, *l, *b;
char *To = NULL, buf[4096], c;
int rrq, result = 1, waskludge = FALSE, badkludge;
int bNeedToGetAddressFromMsgid = (int)NULL, newsmode = 0, lines, pass, count, first;
char *newsgroup = NULL, *charset = NULL;
char *temp, *p, *q, *r, *l, *b, *To = NULL, buf[4096], c, MailFrom[128], MailTo[128];
time_t now;
rfcmsg *kmsg = NULL, **tmsg, *qmsg, *msg = NULL;
off_t endmsg_off, tear_off, orig_off, via_off;
@ -316,7 +314,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
FILE *fp;
fa_list *rlist, *tfa, *ftnpath = NULL;
struct utsname utsbuf;
char MailFrom[128], MailTo[128];
temp = calloc(32768, sizeof(char));
tmsg = &kmsg;
@ -421,7 +418,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
(*tmsg)->val=xstrcpy(buf+4);
tmsg=&((*tmsg)->next);
}
Syslog('M', "tearline \"%s\" at offset %ld", buf,(long)tear_off);
} else if (!strncmp(buf," * Origin:",10)) {
orig_off = ftell(fp);
*tmsg = (rfcmsg *)malloc(sizeof(rfcmsg));
@ -506,42 +502,45 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
endmsg_off = orig_off;
if ((via_off) && (via_off < endmsg_off))
endmsg_off = via_off;
Syslog('M', "end message offset %ld",(long)endmsg_off);
rewind(fp);
msg = parsrfc(fp);
bestaka = bestaka_s(f);
rewind(fp);
/*
* Get characterset encoding from kludge or rfc header
*/
p = hdr((char *)"CHRS", kmsg);
if (p == NULL)
p = hdr((char *)"CHARSET", kmsg);
if (p == NULL)
p = hdr((char *)"CODEPAGE", kmsg);
if (p)
// outcode = readchrs(p);
Syslog('m', "outcode from ftn kludge: %s", printable(p, 0));
else {
if (p) {
while (*p && isspace(*p))
p++;
Striplf(p);
charset = xstrcpy(p);
Syslog('m', "Charset from ftn \"%s\"", printable(charset, 0));
} else {
p=hdr((char *)"Content-Type",msg);
if (p == NULL)
p=hdr((char *)"RFC-Content-Type",kmsg);
if (p == NULL)
p=hdr((char *)"Content-Type",kmsg);
if (p)
// outcode=readcharset(p);
Syslog('m', "outcode from rfc header: %s", printable(p, 0));
else {
// q = rfcmsgid(hdr((char *)"MSGID",kmsg),bestaka);
// if ((hdr((char *)"Message-ID",msg)) || (hdr((char *)"RFC-Message-ID",kmsg)) ||
// (hdr((char *)"Message-ID",kmsg)) || (hdr((char *)"RFCID",kmsg)) ||
// (hdr((char *)"ORIGID",kmsg)) || ((hdr((char *)"MSGID",kmsg)) && (!chkftnmsgid(q))))
// outcode = defaultrfcchar;
// else
// outcode = defaultftnchar;
// if (q)
// free(q);
// q = NULL;
Syslog('m', "outcode will use default");
if (p) {
q = strtok(p, " \n\0");
q = strtok(NULL, " \n\0");
while (*q && isspace(*q))
q++;
Syslog('m', "charset part: %s", printable(q, 0));
if (q && (strncasecmp(q, "charset=", 8) == 0)) {
charset = xstrcpy(q + 8);
Syslog('m', "Charset from rfc \"%s\"", printable(charset, 0));
}
} else {
charset = xstrcpy((char *)"iso-8859-1");
Syslog('m', "No charset, setting default to iso-8859-1");
}
}
@ -550,33 +549,12 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
* The msgs record is already loaded.
*/
newsgroup = xstrcpy(msgs.Newsgroup);
if (strlen(msgs.Distribution))
distribution = xstrcpy(msgs.Distribution);
Syslog('M', "newsgroup %s, distribution %s", printable(newsgroup, 0), printable(distribution, 0));
newsmode = TRUE;
if ((modtype == 1) && (!hdr((char *)"Approved",msg)) &&
(!hdr((char *)"RFC-Approved",kmsg)) && (!hdr((char *)"Approved",kmsg)))
Syslog('M', "newsgroup %s", printable(newsgroup, 0));
newsmode = TRUE;
} else
newsmode = FALSE;
Syslog('m', "Got %s message", newsmode?"echo":"netmail");
// if ((outcode == CHRS_NOTSET) && (hdr((char *)"MSGID", kmsg))) {
// p = rfcmsgid(hdr((char *)"MSGID",kmsg),bestaka);
// if ((hdr((char *)"Message-ID",msg)) || (hdr((char *)"RFC-Message-ID",kmsg)) ||
// (hdr((char *)"Message-ID",kmsg)) || (hdr((char *)"RFCID",kmsg)) ||
// (hdr((char *)"ORIGID",kmsg)) || ((hdr((char *)"MSGID",kmsg)) && (!chkftnmsgid(p))))
// outcode = defaultrfcchar;
// else
// outcode = defaultftnchar;
// free(p);
// }
// if (pgpsigned)
// incode = outcode;
// else if (incode == CHRS_NOTSET)
// incode = getincode(outcode);
/*
* fsc-0038 defines "^aDOMAIN: othernet 99:12/34 fidonet 2:293/2219"
*/
@ -669,9 +647,7 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
strncpy(buf, p, sizeof(buf) -1);
if (*(p = buf + strlen(buf) -1) == '\n')
*p='\0';
} else if (modtype == 1)
sprintf(buf,"%s",moderator);
else
} else
sprintf(buf,"%s",ascinode(t,0x7f));
substitute(buf);
Syslog('+', "mail from %s to %s",ascfnode(f,0x7f),buf);
@ -712,11 +688,13 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
}
Syslog('m', "Prepare is ready");
if (modtype == 1)
newsmode = TRUE;
}
/*
* Setup charset conversion
*/
charset_set_in_out(charset, getrfcchrs(msgs.Charset));
if (newsmode) {
/*
* Open temporary newsfile, append messages if it already exists.
@ -811,20 +789,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
Send(newsmode, temp);
}
if ((p=hdr((char *)"Distribution",msg))) {
sprintf(temp,"Distribution:%s",p);
Send(newsmode, temp);
} else if ((p=hdr((char *)"RFC-Distribution",kmsg))) {
sprintf(temp,"Distribution: %s",p);
Send(newsmode, temp);
} else if ((p=hdr((char *)"Distribution",kmsg))) {
sprintf(temp,"Distribution: %s",p);
Send(newsmode, temp);
} else if (distribution) {
sprintf(temp,"Distribution: %s\n",distribution);
Send(newsmode, temp);
}
p = hdr((char *)"Comment-To",msg);
if (p == NULL)
p=hdr((char *)"X-Comment-To",msg);
@ -861,9 +825,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
} else if ((p=hdr((char *)"Approved",kmsg))) {
sprintf(temp,"Approved: %s",p);
Send(newsmode, temp);
} else if (modtype==2) {
sprintf(temp,"Approved: %s\n",moderator);
Send(newsmode, temp);
}
} else { /* if newsmode */
@ -931,9 +892,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
Syslog('m', "2");
sprintf(temp,"To: %s\n",p);
Send(FALSE, temp);
} else if (modtype == 1) {
sprintf(temp,"To: %s\n",moderator);
Send(FALSE, temp);
} else if (is_local(t)) {
Syslog('m', "3");
sprintf(temp, "To: %s <%s>\n", t->name, buf);
@ -1225,14 +1183,20 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
Send(newsmode, temp);
}
/*
* We have setup translation, send the right charset name
*/
if ((p=hdr((char *)"Content-Type",msg))) {
sprintf(temp,"Content-Type:%s",p);
sprintf(temp, "Content-Type: text/plain; charset=%s\n", getrfcchrs(msgs.Charset));
// sprintf(temp,"Content-Type:%s",p);
Send(newsmode, temp);
} else if ((p=hdr((char *)"RFC-Content-Type",kmsg))) {
sprintf(temp,"Content-Type: %s",p);
sprintf(temp, "Content-Type: text/plain; charset=%s\n", getrfcchrs(msgs.Charset));
// sprintf(temp,"Content-Type: %s",p);
Send(newsmode, temp);
} else if ((p=hdr((char *)"Content-Type",kmsg))) {
sprintf(temp,"Content-Type: %s",p);
sprintf(temp, "Content-Type: text/plain; charset=%s\n", getrfcchrs(msgs.Charset));
// sprintf(temp,"Content-Type: %s",p);
Send(newsmode, temp);
}
@ -1247,15 +1211,16 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
Send(newsmode, temp);
}
temp[0] = '\0';
if ((p=hdr((char *)"Content-Transfer-Encoding",msg)))
if ((p=hdr((char *)"Content-Transfer-Encoding",msg))) {
sprintf(temp,"Content-Transfer-Encoding:%s",p);
else if ((p=hdr((char *)"RFC-Content-Transfer-Encoding",kmsg)))
sprintf(temp,"Content-Transfer-Encoding: %s",p);
else if ((p=hdr((char *)"Content-Transfer-Encoding",kmsg)))
sprintf(temp,"Content-Transfer-Encoding: %s",p);
if (temp[0])
Send(newsmode, temp);
} else if ((p=hdr((char *)"RFC-Content-Transfer-Encoding",kmsg))) {
sprintf(temp,"Content-Transfer-Encoding: %s",p);
Send(newsmode, temp);
} else if ((p=hdr((char *)"Content-Transfer-Encoding",kmsg))) {
sprintf(temp,"Content-Transfer-Encoding: %s",p);
Send(newsmode, temp);
}
if (newsmode) {
if ((p=hdr((char *)"X-Newsreader",msg))) {
@ -1322,7 +1287,7 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
strcasecmp(qmsg->key,"Supersedes") &&
strcasecmp(qmsg->key,"Mime-Version") &&
strcasecmp(qmsg->key,"Content-Type") &&
strcasecmp(qmsg->key,"Content-Lenght") &&
strcasecmp(qmsg->key,"Content-Length") &&
strcasecmp(qmsg->key,"Content-Transfer-Encoding") &&
strcasecmp(qmsg->key,"Lines") &&
strcasecmp(qmsg->key,"Path") &&
@ -1558,9 +1523,8 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
}
Syslog('m', "End sending message body");
if ((modtype==1) && (!hdr((char *)"Approved",msg)) &&
(!hdr((char *)"RFC-Approved",kmsg)) && (!hdr((char *)"Approved",kmsg)))
newsmode = FALSE;
if (charset)
free(charset);
tidyrfc(msg);
fclose(fp);
@ -1580,10 +1544,6 @@ int ftn2rfc(faddr *f, faddr *t, char *subj, char *origline, time_t mdate, int fl
if (newsgroup)
free(newsgroup);
if (distribution)
free(distribution);
if (moderator)
free(moderator);
rbuf = NULL;
free(temp);
return result;

View File

@ -79,7 +79,7 @@ int newspost(void)
while (fgets(buf, 10240, nfp)) {
if (start) {
if (nntp_cmd((char *)"POST\r\n", 340) != 0) {
WriteError("NNTP POST refused");
WriteError("NNTP: POST refused");
free(buf);
return TRUE;
}
@ -88,6 +88,7 @@ int newspost(void)
if (!strcmp(buf, ".\n")) {
if (nntp_cmd((char *)".\r\n", 240) == 0) {
news_out++;
Syslog('+', "NTTP: article %d accepted", news_out);
} else {
WriteError("NNTP: refused article %d", news_out+1);
news_bad++;

View File

@ -59,6 +59,7 @@ static int removeref;
static int removeinreply;
static int removereplyto;
static int removereturnto;
char currentgroup[81];
@ -80,7 +81,6 @@ int needputrfc(rfcmsg *, int);
int charwrite(char *, FILE *);
int charwrite(char *s, FILE *fp)
{
char *o;
@ -126,6 +126,65 @@ int kludgewrite(char *s, FILE *fp)
/*
* Build a faked RFC msgid, use the CRC32 of the FTN msgid,
* the current group and the configured system's fqdn. This
* gives a unique string specific for the message.
*/
char *make_msgid(char *msgid)
{
static char buf[100];
sprintf(buf, "<%8lx$%s@%s>", StringCRC32(msgid), currentgroup, CFG.sysdomain);
return buf;
}
int findorigmsg(char *msgid, char *o)
{
unsigned long i, start, end;
char *gen2;
if (msgid == NULL) {
return 0;
}
if (!Msg_Open(msgs.Base)) {
return 0;
}
Msg_Number();
Msg_Highest();
Msg_Lowest();
if (MsgBase.Open == FALSE) {
Syslog('-', "Base closed");
return 0;
}
strcpy(currentgroup,msgs.Newsgroup);
start = MsgBase.Lowest;
end = MsgBase.Highest;
gen2 = calloc(strlen(msgid)+1,sizeof(char));
strcpy(gen2, strchr(msgid,'<'));
for (i = start; i <= end; i++) {
if (Msg_ReadHeader(i)) {
if (strncmp(gen2,make_msgid(Msg.Msgid),strlen(gen2)-1) == 0) {
Syslog('m',"Found msgid: %s",make_msgid(Msg.Msgid));
// realloc(o,(strlen(Msg.Msgid)+1)* sizeof(char));
strcpy(o,Msg.Msgid);
free(gen2);
return 1;
}
}
}
free(gen2);
return 0;
}
/*
* Input a RFC message.
*/
@ -159,9 +218,10 @@ int rfc2ftn(FILE *fp, faddr *recipient)
newsmode = hdr((char *)"Newsgroups", msg) ?TRUE:FALSE;
Syslog('m', "RFC message is %s", newsmode ? "news article":"e-mail message");
if (newsmode)
if (newsmode) {
news_in++;
else
sprintf(currentgroup, "%s", msgs.Newsgroup);
} else
email_in++;
if (!CFG.allowcontrol) {
@ -190,6 +250,16 @@ int rfc2ftn(FILE *fp, faddr *recipient)
if ((p = hdr((char *)"References",msg))) {
p = strrchr(p,' ');
ftnmsgid(p,&fmsg->reply_a, &fmsg->reply_n,fmsg->area);
//Griffin
fmsg->reply_s=calloc(256,sizeof(char));
findorigmsg(p,fmsg->reply_s);
fmsg->to->name=calloc(strlen(Msg.From)+1,sizeof(char));
strcpy(fmsg->to->name,Msg.From);
Syslog('m', "fmsg to-name %s",fmsg->to->name);
Syslog('m', "reply_s %s",fmsg->reply_s);
if (!chkftnmsgid(p)) {
hash_update_s(&fmsg->reply_n, fmsg->area);
}
@ -238,7 +308,16 @@ int rfc2ftn(FILE *fp, faddr *recipient)
q++;
Syslog('m', "charset part: %s", printable(q, 0));
if (q && (strncasecmp(q, "charset=", 8) == 0)) {
/*
* google.com quotes the charset name
*/
if (strchr(q, '"')) {
charset = xstrcpy(q + 9);
charset[strlen(charset)-1] = '\0';
Syslog('m', "Unquoted charset name");
} else {
charset = xstrcpy(q + 8);
}
Syslog('m', "Charset \"%s\"", printable(charset, 0));
}
}

View File

@ -1,6 +1,9 @@
#ifndef _RFC2FTN_H
#define _RFC2FTN_H
/* $Id$ */
int charwrite(char *, FILE *);
int rfc2ftn(FILE *fp, faddr *);
#endif

View File

@ -695,7 +695,7 @@ void ExportNews(unsigned long MsgNum, fa_list **sbl)
Syslog('m', "Msg.From %s", Msg.From);
Syslog('m', "Msg.FromAddress %s", Msg.FromAddress);
Syslog('m', "Msg.To %s", Msg.To);
Syslog('m', "Msg.ToAddress", Msg.ToAddress);
Syslog('m', "Msg.ToAddress %s", Msg.ToAddress);
flags |= (Msg.Private) ? M_PVT : 0;
from = fido2faddr(msgs.Aka);
@ -717,7 +717,6 @@ void ExportNews(unsigned long MsgNum, fa_list **sbl)
dest = NULL;
fprintf(qp, "AREA:%s\n", msgs.Tag);
Syslog('m', "AREA:%s", msgs.Tag);
if (Msg_Read(MsgNum, 79)) {
if ((p = (char *)MsgText_First()) != NULL) {
@ -730,18 +729,13 @@ void ExportNews(unsigned long MsgNum, fa_list **sbl)
kludges = FALSE;
fprintf(qp, "\001TID: MBSE-FIDO %s (%s-%s)\n", VERSION, OsName(), OsCPU());
fprintf(qp, "Subject: %s\n", Msg.Subject);
Syslog('m', "Subject: %s", Msg.Subject);
fprintf(qp, "\n");
Syslog('m', "\n");
fprintf(qp, "%s\n", p);
Syslog('m', "%s", p);
} else {
fprintf(qp, "%s\n", p+1);
Syslog('m', "%s", p+1);
}
} else {
fprintf(qp, "%s", p);
Syslog('m', "%s", printable(p, 0));
if (strncmp(p, " * Origin:", 10) == 0)
break;
@ -775,7 +769,6 @@ void ExportNews(unsigned long MsgNum, fa_list **sbl)
fprintf(qp, "%s", sbe);
}
fprintf(qp, "\n\001PATH: %u/%u\n", msgs.Aka.net, msgs.Aka.node);
Syslog('m', "\\001PATH: %u/%u", msgs.Aka.net, msgs.Aka.node);
rewind(qp);
most_debug = TRUE;