changeset 741:b72bcb7e08a2

Arbitrary headers in listtexts, fix default Content-Transfer-Encoding: header, and document \uNNNN substitution Also, the interface to prepstdreply() has changed; there is no longer a customheaders argument, which was never used anyway, and is now essentially redundant due to this patch.
author Ben Schmidt
date Mon, 20 Sep 2010 01:44:58 +1000
parents 5db75af2d0db
children b00eb39643c1
files ChangeLog README.listtexts include/prepstdreply.h src/mlmmj-bounce.c src/mlmmj-process.c src/mlmmj-sub.c src/mlmmj-unsub.c src/prepstdreply.c src/send_help.c src/send_list.c
diffstat 10 files changed, 207 insertions(+), 107 deletions(-) [+]
line wrap: on
line diff
--- a/ChangeLog	Mon Sep 20 01:41:32 2010 +1000
+++ b/ChangeLog	Mon Sep 20 01:44:58 2010 +1000
@@ -1,5 +1,6 @@
+ o Allow arbitrary headers in list texts
  o Ensure digest listtext is always closed
- o Fix Content-Transfer-Encoding: header for digests
+ o Fix Content-Transfer-Encoding: header for digests and list texts
  o Fixed a bug that could cause a crash if $posteraddr$ appeared in the
    maxmaiilsize listtext
  o Documented listtexts
--- a/README.listtexts	Mon Sep 20 01:41:32 2010 +1000
+++ b/README.listtexts	Mon Sep 20 01:44:58 2010 +1000
@@ -106,22 +106,38 @@
 
 They have the following format:
 
-- Subject line
+- Headers
 - Blank line
 - Body
 
 They are expected to be in UTF-8 encoding and have Unix line endings.
 
-The subject line is the text 'Subject: ' (Mlmmj expects the space, even though
-this is not ordinarily required in mail messages) followed by the subject line
-to be used for the mail. The subject line may include UTF-8 characters, which
-will automatically be escaped using the =?utf-8?q?...?= quoting mechanism.
+The headers should be formatted as they should appear in the mail message. They
+will begin the mail message. Header continuation via lines beginning with
+linear whitespace is supported.
+
+Following the headers found in the list text, Mlmmj will output the following
+default headers, unless the same header is already provided in the list text.
+
+- From:
+- To:
+- Message-ID:
+- Date:
+- Subject: mlmmj administrivia
+- MIME-Version: 1.0
+- Content-Type: text/plain; charset=utf-8
+- Content-Transfer-Encoding: 8bit
+
+The Subject: header is treated specially: it may include UTF-8 characters,
+which will automatically be escaped using the =?utf-8?q?...?= quoting
+mechanism.
 
 List text substitutions
 -----------------------
 
-Both subject and body may include the following, which are substituted prior to
-sending the message:
+Both headers and body may include the following, which are substituted prior to
+sending the message (though note that some of these substitutions are
+multi-line substitutions and would not work in a header):
 
 - $bouncenumbers$
   (available only in bounceprobe)
@@ -222,5 +238,12 @@
   (available only in submod-moderator and [un]sub-confirm[-digest|-nomail])
   the address requested to be (un-)subscribed
 
+- \uNNNN
+  (NNNN are hex digits)
+  a Unicode character
+  (this is not really appropriate for use in a header, except perhaps the
+  Subject: header as Mlmmj does automatic quoting for that header as described
+  above)
 
 
+
--- a/include/prepstdreply.h	Mon Sep 20 01:41:32 2010 +1000
+++ b/include/prepstdreply.h	Mon Sep 20 01:44:58 2010 +1000
@@ -32,6 +32,6 @@
 int open_listtext(const char *listdir, const char *filename);
 char *prepstdreply(const char *listdir, const char *filename, const char *from,
 		const char *to, const char *replyto, size_t tokencount,
-		char **data, char *customheaders, const char *mailname);
+		char **data, const char *mailname);
 
 #endif /* PREPSTDREPLY_H */
--- a/src/mlmmj-bounce.c	Mon Sep 20 01:41:32 2010 +1000
+++ b/src/mlmmj-bounce.c	Mon Sep 20 01:44:58 2010 +1000
@@ -146,7 +146,7 @@
 
 	maildata[1] = indexstr;
 	queuefilename = prepstdreply(listdir, "bounce-probe", "$listowner$",
-					myaddr, NULL, 1, maildata, NULL, NULL);
+					myaddr, NULL, 1, maildata, NULL);
 	MY_ASSERT(queuefilename);
 	myfree(indexstr);
 
--- a/src/mlmmj-process.c	Mon Sep 20 01:41:32 2010 +1000
+++ b/src/mlmmj-process.c	Mon Sep 20 01:44:58 2010 +1000
@@ -133,7 +133,7 @@
 	myfree(listfqdn);
 
 	queuefilename = prepstdreply(listdir, "moderation", "$listowner$",
-				     to, replyto, 2, maildata, NULL,
+				     to, replyto, 2, maildata,
 				     mailfilename);
 
 	/* we might need to exec more than one mlmmj-send */
@@ -176,7 +176,7 @@
 
 	queuefilename = prepstdreply(listdir, "moderation-poster",
 				     "$listowner$", efromsender,
-				     NULL, 1, maildata+2, NULL, mailfilename);
+				     NULL, 1, maildata+2, mailfilename);
 
 	execlp(mlmmjsend, mlmmjsend,
 			"-l", "1",
@@ -696,7 +696,7 @@
 			queuefilename = prepstdreply(listdir,
 					"maxmailsize", "$listowner$",
 					fromemails.emaillist[0],
-					NULL, 1, maildata+2, NULL, donemailname);
+					NULL, 1, maildata+2, donemailname);
 			MY_ASSERT(queuefilename)
 			myfree(listdelim);
 			myfree(listname);
@@ -811,7 +811,7 @@
 				     listfqdn);
 		queuefilename = prepstdreply(listdir, "notintocc",
 					"$listowner$", fromemails.emaillist[0],
-					     NULL, 0, NULL, NULL, donemailname);
+					     NULL, 0, NULL, donemailname);
 		MY_ASSERT(queuefilename)
 		myfree(listdelim);
 		myfree(listname);
@@ -872,7 +872,7 @@
 					"bounces-help@", listfqdn);
 			queuefilename = prepstdreply(listdir, "subonlypost",
 					"$listowner$", fromemails.emaillist[0],
-						     NULL, 1, maildata, NULL, donemailname);
+						     NULL, 1, maildata, donemailname);
 			MY_ASSERT(queuefilename)
 			myfree(listaddr);
 			myfree(listdelim);
@@ -927,7 +927,7 @@
 			queuefilename = prepstdreply(listdir, "access",
 							"$listowner$",
 							fromemails.emaillist[0],
-						     NULL, 0, NULL, NULL, donemailname);
+						     NULL, 0, NULL, donemailname);
 			MY_ASSERT(queuefilename)
 			myfree(listaddr);
 			myfree(listdelim);
--- a/src/mlmmj-sub.c	Mon Sep 20 01:41:32 2010 +1000
+++ b/src/mlmmj-sub.c	Mon Sep 20 01:44:58 2010 +1000
@@ -147,7 +147,7 @@
 	maildata[5] = moderators;
 
 	queuefilename = prepstdreply(listdir, "submod-moderator",
-				"$listowner$", to, replyto, 3, maildata, NULL, NULL);
+				"$listowner$", to, replyto, 3, maildata, NULL);
 	
 	myfree(maildata[1]);
 	
@@ -189,7 +189,7 @@
 
 	from = concatstr(4, listname, listdelim, "bounces-help@", listfqdn);
 	queuefilename = prepstdreply(listdir, "submod-requester", "$listowner$",
-					subaddr, NULL, 0, NULL, NULL, NULL);
+					subaddr, NULL, 0, NULL, NULL);
 	
 	myfree(listname);
 	myfree(listfqdn);
@@ -282,7 +282,7 @@
 	}
 
 	queuefilename = prepstdreply(listdir, listtext, "$helpaddr$",
-				     subaddr, NULL, 0, NULL, NULL, NULL);
+				     subaddr, NULL, 0, NULL, NULL);
 	MY_ASSERT(queuefilename);
 	myfree(listtext);
 
@@ -329,7 +329,7 @@
 	}
 
 	queuefilename = prepstdreply(listdir, listtext, "$listowner$",
-				"$listowner$", NULL, 1, maildata, NULL, NULL);
+				"$listowner$", NULL, 1, maildata, NULL);
 	MY_ASSERT(queuefilename)
 	myfree(listtext);
 	myfree(maildata[1]);
@@ -416,7 +416,7 @@
 	maildata[3] = mystrdup(confirmaddr);
 
 	queuefilename = prepstdreply(listdir, listtext, "$helpaddr$", subaddr,
-				     confirmaddr, 2, maildata, NULL, NULL);
+				     confirmaddr, 2, maildata, NULL);
 
 	myfree(maildata[1]);
 	myfree(maildata[3]);
@@ -469,7 +469,7 @@
 	myfree(listdelim);
 
 	queuefilename = prepstdreply(listdir, "sub-subscribed", "$helpaddr$",
-				     subaddr, NULL, 0, NULL, NULL, NULL);
+				     subaddr, NULL, 0, NULL, NULL);
 	MY_ASSERT(queuefilename);
 
 	myfree(listaddr);
--- a/src/mlmmj-unsub.c	Mon Sep 20 01:41:32 2010 +1000
+++ b/src/mlmmj-unsub.c	Mon Sep 20 01:44:58 2010 +1000
@@ -76,7 +76,7 @@
 	}
 
 	queuefilename = prepstdreply(listdir, listtext, "$helpaddr$",
-				     subaddr, NULL, 0, NULL, NULL, NULL);
+				     subaddr, NULL, 0, NULL, NULL);
 	MY_ASSERT(queuefilename);
 	myfree(listtext);
 
@@ -124,7 +124,7 @@
 	}
 	
 	queuefilename = prepstdreply(listdir, listtext, "$listowner$",
-				     "$listowner$", NULL, 1, maildata, NULL, NULL);
+				     "$listowner$", NULL, 1, maildata, NULL);
 	MY_ASSERT(queuefilename);
 	myfree(listtext);
 	myfree(maildata[1]);
@@ -213,7 +213,7 @@
 	maildata[3] = mystrdup(confirmaddr);
 
 	queuefilename = prepstdreply(listdir, listtext, "$helpaddr$", subaddr,
-				     confirmaddr, 2, maildata, NULL, NULL);
+				     confirmaddr, 2, maildata, NULL);
 
 	myfree(maildata[1]);
 	myfree(maildata[3]);
@@ -305,7 +305,7 @@
 	myfree(listdelim);
 
 	queuefilename = prepstdreply(listdir, "unsub-notsubscribed",
-				     "$helpaddr$", subaddr, NULL, 0, NULL, NULL, NULL);
+				     "$helpaddr$", subaddr, NULL, 0, NULL, NULL);
 	MY_ASSERT(queuefilename);
 
 	myfree(listaddr);
--- a/src/prepstdreply.c	Mon Sep 20 01:41:32 2010 +1000
+++ b/src/prepstdreply.c	Mon Sep 20 01:44:58 2010 +1000
@@ -208,12 +208,14 @@
 
 char *prepstdreply(const char *listdir, const char *filename, const char *from,
 		   const char *to, const char *replyto, size_t tokencount,
-		   char **data, char *customheaders, const char *mailname)
+		   char **data, const char *mailname)
 {
+	size_t i, len;
 	int infd, outfd;
-	char *listaddr, *listdelim, *myfrom, *tmp, *subject, *retstr = NULL;
+	char *listaddr, *listdelim, *tmp, *retstr = NULL;
 	char *listfqdn, *line, *utfline, *utfsub, *utfsub2;
-	char *myreplyto, *myto, *str = NULL, *mydate, *mymsgid;
+	char *str = NULL;
+	char *headers[10] = { NULL }; /* relies on NULL to flag end */
 
 	if ((infd = open_listtext(listdir, filename)) < 0) {
 		return NULL;
@@ -223,49 +225,6 @@
 	listdelim = getlistdelim(listdir);
 	listfqdn = genlistfqdn(listaddr);
 
-	line = mygetline(infd);
-	if(!line || (strncasecmp(line, "Subject: ", 9) != 0)) {
-		log_error(LOG_ARGS, "No Subject in '%s' listtext. Using "
-				"standard subject", filename);
-		subject = mystrdup("mlmmj administrativa");
-	} else {
-		chomp(line);
-		utfsub = unistr_escaped_to_utf8(line + 9);
-		utfsub2 = substitute(utfsub, listaddr, listdelim, tokencount,
-				     data, NULL);
-		subject = unistr_utf8_to_header(utfsub2);
-		myfree(utfsub);
-		myfree(utfsub2);
-		myfree(line);
-
-		/* skip empty line after subject */
-		line = mygetline(infd);
-		if (line && (line[0] == '\n')) {
-			myfree(line);
-			line = NULL;
-		}
-	}
-	if (line) {
-		utfline = unistr_escaped_to_utf8(line);
-		myfree(line);
-	} else {
-		utfline = NULL;
-	}
-	
-	myfrom = substitute(from, listaddr, listdelim, tokencount, data, NULL);
-	myto = substitute(to, listaddr, listdelim, tokencount, data, NULL);
-	mydate = gendatestr();
-	mymsgid = genmsgid(listfqdn);
-
-	if(replyto) {
-		myreplyto = substitute(replyto, listaddr, listdelim,
-				       tokencount, data, NULL);
-		tmp = concatstr(3, "Reply-To: ", myreplyto, "\n");
-		myfree(myreplyto);
-		myreplyto = tmp;
-	} else
-		myreplyto = NULL;
-
 	do {
 		tmp = random_str();
 		myfree(retstr);
@@ -278,48 +237,164 @@
 	
 	if(outfd < 0) {
 		log_error(LOG_ARGS, "Could not open std mail %s", retstr);
-		myfree(str);
 		myfree(listaddr);
 		myfree(listdelim);
 		myfree(listfqdn);
-		myfree(utfline);
 		return NULL;
 	}
 
-	str = concatstr(14,
-			"From: ", myfrom,
-			"\nTo: ", myto,
-			"\n", myreplyto,
-			mymsgid,
-			mydate,
-			"Subject: ", subject,
-			"\nMIME-Version: 1.0"
-			"\nContent-Type: text/plain; charset=utf-8"
-			"\nContent-Encoding: 8bit"
-			"\n", customheaders,
-			"\n", utfline);
+	tmp = substitute(from, listaddr, listdelim,
+	                 tokencount, data, NULL);
+	headers[0] = concatstr(2, "From: ", tmp);
+	myfree(tmp);
+	tmp = substitute(to, listaddr, listdelim,
+	                 tokencount, data, NULL);
+	headers[1] = concatstr(2, "To: ", tmp);
+	myfree(tmp);
+	headers[2] = genmsgid(listfqdn);
+	chomp(headers[2]);
+	headers[3] = gendatestr();
+	chomp(headers[3]);
+	headers[4] = mystrdup("Subject: mlmmj administrivia");
+	headers[5] = mystrdup("MIME-Version: 1.0");
+	headers[6] = mystrdup("Content-Type: text/plain; charset=utf-8");
+	headers[7] = mystrdup("Content-Transfer-Encoding: 8bit");
+
+	if(replyto) {
+		tmp = substitute(replyto, listaddr, listdelim,
+		                 tokencount, data, NULL);
+		headers[8] = concatstr(2, "Reply-To: ", tmp);
+		myfree(tmp);
+	}
 
-	myfree(utfline);
+	for(;;) {
+		line = mygetline(infd);
+		if (!line) {
+			log_error(LOG_ARGS, "No body in '%s' listtext",
+					filename);
+			break;
+		}
+		if (*line == '\n') {
+			/* end of headers */
+			myfree(line);
+			line = NULL;
+			break;
+		}
+		chomp(line);
+		if (*line == ' ' || *line == '\t') {
+			/* line beginning with linear whitespace is a
+			   continuation of previous header line */
+			utfsub = unistr_escaped_to_utf8(line);
+			str = substitute(utfsub, listaddr, listdelim,
+			                 tokencount, data, NULL);
+			myfree(utfsub);
+			len = strlen(str);
+			str[len] = '\n';
+			if(writen(outfd, str, len+1) < 0) {
+				log_error(LOG_ARGS, "Could not write std mail");
+				myfree(str);
+				myfree(line);
+				myfree(listaddr);
+				myfree(listdelim);
+				myfree(listfqdn);
+				return NULL;
+			}
+			myfree(str);
+		} else {
+			tmp = line;
+			len = 0;
+			while (*tmp && *tmp != ':') {
+				tmp++;
+				len++;
+			}
+			if (!*tmp) {
+				log_error(LOG_ARGS, "No headers or invalid "
+						"header in '%s' listtext",
+						filename);
+				break;
+			}
+			tmp++;
+			len++;
+			/* remove the standard header if one matches */
+			for (i=0; headers[i] != NULL; i++) {
+				if (strncasecmp(line, headers[i], len) == 0) {
+					myfree(headers[i]);
+					while (headers[i] != NULL) {
+						headers[i] = headers[i+1];
+						i++;
+					}
+					break;
+				}
+			}
+			utfsub = unistr_escaped_to_utf8(tmp);
+			*tmp = '\0';
+			utfsub2 = substitute(utfsub, listaddr, listdelim,
+			                     tokencount, data, NULL);
+			myfree(utfsub);
+			if (strncasecmp(line, "Subject:", len) == 0) {
+				tmp = unistr_utf8_to_header(utfsub2);
+				myfree(utfsub2);
+				str = concatstr(2, line, tmp);
+				myfree(tmp);
+			} else {
+				str = concatstr(2, line, utfsub2);
+				myfree(utfsub2);
+			}
+			len = strlen(str);
+			str[len] = '\n';
+			if(writen(outfd, str, len+1) < 0) {
+				log_error(LOG_ARGS, "Could not write std mail");
+				myfree(str);
+				myfree(line);
+				myfree(listaddr);
+				myfree(listdelim);
+				myfree(listfqdn);
+				return NULL;
+			}
+			myfree(str);
+		}
+		myfree(line);
+	}
 
-	if(writen(outfd, str, strlen(str)) < 0) {
+	for (i=0; headers[i] != NULL; i++) {
+		len = strlen(headers[i]);
+		headers[i][len] = '\n';
+		if(writen(outfd, headers[i], len+1) < 0) {
 		log_error(LOG_ARGS, "Could not write std mail");
+			if (line)
+				myfree(line);
 		myfree(str);
 		myfree(listaddr);
 		myfree(listdelim);
 		myfree(listfqdn);
 		return NULL;
 	}
+	}
 
+	/* end the headers */
+	if(writen(outfd, "\n", 1) < 0) {
+		log_error(LOG_ARGS, "Could not write std mail");
+		myfree(str);
+		if (line)
+			myfree(line);
+		myfree(listaddr);
+		myfree(listdelim);
+		myfree(listfqdn);
+		return NULL;
+	}
+
+	if (line) {
+		str = concatstr(2, line, "\n");
+		myfree(line);
+	} else {
+		str = mygetline(infd);
+	}
+	while(str) {
+		utfline = unistr_escaped_to_utf8(str);
 	myfree(str);
 
-	while((str = mygetline(infd))) {
-		tmp = str;
-		utfline = unistr_escaped_to_utf8(str);
-		myfree(tmp);
-
-		tmp = utfline;
 		str = substitute(utfline, listaddr, listdelim, tokencount, data, mailname);
-		myfree(tmp);
+		myfree(utfline);
 
 		if(writen(outfd, str, strlen(str)) < 0) {
 			myfree(str);
@@ -330,6 +405,7 @@
 			return NULL;
 		}
 		myfree(str);
+		str = mygetline(infd);
 	}
 	
 	fsync(outfd);
--- a/src/send_help.c	Mon Sep 20 01:41:32 2010 +1000
+++ b/src/send_help.c	Mon Sep 20 01:44:58 2010 +1000
@@ -57,7 +57,7 @@
 	myfree(listdelim);
 
 	queuefilename = prepstdreply(listdir, textfile, "$listowner$",
-					emailaddr, NULL, 0, NULL, NULL, NULL);
+					emailaddr, NULL, 0, NULL, NULL);
 	if(queuefilename == NULL) {
 		log_error(LOG_ARGS, "Could not prepare %s mail", name);
 		exit(EXIT_FAILURE);
--- a/src/send_list.c	Mon Sep 20 01:41:32 2010 +1000
+++ b/src/send_list.c	Mon Sep 20 01:44:58 2010 +1000
@@ -99,7 +99,7 @@
 	myfree(listdelim);
 
 	queuefilename = prepstdreply(listdir, "listsubs", "$listowner$",
-					emailaddr, NULL, 0, NULL, NULL, NULL);
+					emailaddr, NULL, 0, NULL, NULL);
 	if(queuefilename == NULL) {
 		log_error(LOG_ARGS, "Could not prepare sub list mail");
 		exit(EXIT_FAILURE);