Mercurial > hg > mlmmj

changeset 814:757225257ef3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add formatted substitutions that are lists. These are %digestthreads%, %gatekeepers%, %listsubs%, %digestsubs%, %nomailsubs%, %moderators% and %bouncenumbers%.
author Ben Schmidt
date Tue, 17 Jan 2012 11:20:38 +1100
parents 74d5ebb67b34
children fdd43713fb52
files ChangeLog README.listtexts include/prepstdreply.h src/mlmmj-bounce.c src/mlmmj-process.c src/mlmmj-sub.c src/prepstdreply.c src/send_digest.c src/send_list.c
diffstat 9 files changed, 598 insertions(+), 176 deletions(-) [+]
line wrap: on
line diff
--- a/ChangeLog	Tue Jan 17 01:26:45 2012 +1100
+++ b/ChangeLog	Tue Jan 17 11:20:38 2012 +1100
@@ -1,3 +1,5 @@
+ o Add %digestthreads%, %gatekeepers%, %listsubs%, %digestsubs%, %nomailsubs%,
+   %moderators% and %bouncenumbers%
  o Deprecate various list text substitutions such as $newsub$, $oldsub$,
    $moderateaddr$
  o Add $permitaddr$ and $releaseaddr$ substitutions
--- a/README.listtexts	Tue Jan 17 01:26:45 2012 +1100
+++ b/README.listtexts	Tue Jan 17 11:20:38 2012 +1100
@@ -15,7 +15,7 @@
 - Supported list texts
 - Format
 - Conditionals
-- Formatted substitutions
+- Formatting and formatted substitutions
 - Unformatted substitutions
 
 Naming scheme
@@ -148,6 +148,10 @@
 - list---digest *
 - list---nomail *
   sent in response to an email to listname+list@domain.tld from the list owner
+  (DEPRECATED: if none of %listsubs%, %digestsubs% and %nomailsubs% is
+  encountered in the text, then they will be automatically added with some
+  default formatting; this functionality is expected to be removed in the
+  future)
 
 * Not yet used.
 
@@ -275,9 +279,17 @@
   (available only in gatekeep-sub and wait-sub)
   the list of moderators to whom the moderation request has been sent
 
-- %subs%
+- %listsubs%
+  (available only in list---*)
+  the list of normal subscribers
+
+- %digestsubs%
   (available only in list---*)
-  the list of subscribers
+  the list of digest subscribers
+
+- %nomailsubs%
+  (available only in list---*)
+  the list of nomail subscribers
 
 - %moderators%
   (available only in moderate-post-* and wait-post-*)
--- a/include/prepstdreply.h	Tue Jan 17 01:26:45 2012 +1100
+++ b/include/prepstdreply.h	Tue Jan 17 11:20:38 2012 +1100
@@ -28,17 +28,42 @@
 struct text;
 typedef struct text text;
 
+typedef void (*rewind_function)(void *state);
+typedef const char *(*get_function)(void *state);
+
+struct memory_lines_state;
+typedef struct memory_lines_state memory_lines_state;
+
+struct file_lines_state;
+typedef struct file_lines_state file_lines_state;
+
+
+memory_lines_state *init_memory_lines(const char *lines);
+void rewind_memory_lines(void *state);
+const char *get_memory_line(void *state);
+void finish_memory_lines(memory_lines_state *s);
+
+file_lines_state *init_file_lines(const char *filename, int open_now);
+file_lines_state *init_truncated_file_lines(const char *filename, int open_now,
+		char truncate);
+void rewind_file_lines(void *state);
+const char *get_file_line(void *state);
+void finish_file_lines(file_lines_state *s);
+
 char *substitute(const char *line, const char *listaddr, const char *listdelim,
 		const char *listdir, text *txt);
+
 text *open_text_file(const char *listdir, const char *filename);
 text *open_text(const char *listdir, const char *purpose, const char *action,
 		   const char *reason, const char *type, const char *compat);
 void register_unformatted(text *txt, const char *token, const char *subst);
 void register_originalmail(text *txt, const char *mailname);
+void register_formatted(text *txt, const char *token,
+		rewind_function rew, get_function get, void * state);
 char *get_processed_text_line(text *txt,
 		const char *listaddr, const char *listdelim, const char *listdir);
-void close_text(text *txt);
 char *prepstdreply(text *txt, const char *listdir,
 		const char *from, const char *to, const char *replyto);
+void close_text(text *txt);
 
 #endif /* PREPSTDREPLY_H */
--- a/src/mlmmj-bounce.c	Tue Jan 17 01:26:45 2012 +1100
+++ b/src/mlmmj-bounce.c	Tue Jan 17 11:20:38 2012 +1100
@@ -49,67 +49,12 @@
 #include "find_email_adr.h"
 #include "gethdrline.h"
 
-char *fetchindexes(const char *bouncefile)
-{
-	int fd;
-	char *indexstr = NULL, *s, *start, *line, *cur, *colon, *next;
-	size_t len;
-	struct stat st;
-
-	fd = open(bouncefile, O_RDONLY);
-	if(fd < 0) {
-		log_error(LOG_ARGS, "Could not open bounceindexfile %s",
-				bouncefile);
-		return NULL;
-	}
-
-	if(fstat(fd, &st) < 0) {
-		log_error(LOG_ARGS, "Could not open bounceindexfile %s",
-					bouncefile);
-	}
-
-	if(st.st_size == 0) {
-		log_error(LOG_ARGS, "Bounceindexfile %s is size 0",
-					bouncefile);
-		return NULL;
-	}
-
-	start = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
-	if(start == MAP_FAILED) {
-		log_error(LOG_ARGS, "Could not mmap bounceindexfile");
-		return NULL;
-	}
-
-
-	for(next = cur = start; next < start + st.st_size; next++) {
-		if(*next == '\n') {
-			len = next - cur;
-			line = mymalloc(len + 1);
-			strncpy(line, cur, len);
-			line[len] = '\0';
-			cur = next + 1;
-		} else
-	 		continue;
-
-		colon = strchr(line, ':');
-		MY_ASSERT(colon);
-		*colon = '\0';
-		s = indexstr;
-		indexstr = concatstr(4, s, "        ", line, "\n");
-		myfree(s);
-		myfree(line);
-	}
-
-	munmap(start, st.st_size);
-	close(fd);
-
-	return indexstr;
-}
 
 void do_probe(const char *listdir, const char *mlmmjsend, const char *addr)
 {
 	text *txt;
-	char *myaddr, *from, *a, *indexstr, *queuefilename, *listaddr;
+	file_lines_state *fls;
+	char *myaddr, *from, *a, *queuefilename, *listaddr;
 	char *listfqdn, *listname, *probefile, *listdelim=getlistdelim(listdir);
 	int fd;
 	time_t t;
@@ -138,20 +83,18 @@
 	}
 	*a = '@';
 
-	indexstr = fetchindexes(addr);
-	if(indexstr == NULL) {
-		log_error(LOG_ARGS, "Could not fetch bounceindexes");
-		exit(EXIT_FAILURE);
-	}
-
 	txt = open_text(listdir, "probe", NULL, NULL, NULL, "bounce-probe");
 	MY_ASSERT(txt);
-	register_unformatted(txt, "bouncenumbers", indexstr);
-	myfree(indexstr);
+	register_unformatted(txt, "bouncenumbers", "%bouncenumbers%"); /* DEPRECATED */
+	fls = init_truncated_file_lines(addr, 0, ':');
+	register_formatted(txt, "bouncenumbers",
+			rewind_file_lines, get_file_line, fls);
 	queuefilename = prepstdreply(txt, listdir, "$listowner$", myaddr, NULL);
 	MY_ASSERT(queuefilename);
 	close_text(txt);
 
+	finish_file_lines(fls);
+
 	probefile = concatstr(4, listdir, "/bounce/", addr, "-probe");
 	MY_ASSERT(probefile);
 	t = time(NULL);
--- a/src/mlmmj-process.c	Tue Jan 17 01:26:45 2012 +1100
+++ b/src/mlmmj-process.c	Tue Jan 17 11:20:38 2012 +1100
@@ -94,6 +94,7 @@
 	char *from, *listfqdn, *listname, *moderators = NULL;
 	char *buf, *replyto, *listaddr = getlistaddr(listdir), *listdelim;
 	text *txt;
+	memory_lines_state *mls;
 	char *queuefilename = NULL, *moderatorsfilename, *efromismod = NULL;
 	char *mailbasename = mybasename(mailfilename), *tmp, *to;
 	int moderatorsfd, foundaddr = 0, notifymod = 0, status;
@@ -130,7 +131,11 @@
 		efromismod = NULL;
 	}
 
+	if(efromismod) mls = init_memory_lines(efromismod);
+	else mls = init_memory_lines(moderators);
+
 	close(moderatorsfd);
+	myfree(moderators);
 
 	listdelim = getlistdelim(listdir);
 	replyto = concatstr(6, listname, listdelim, "moderate-", mailbasename,
@@ -150,8 +155,9 @@
 	register_unformatted(txt, "posteraddr", posteraddr);
 	register_unformatted(txt, "moderateaddr", replyto); /* DEPRECATED */
 	register_unformatted(txt, "releaseaddr", replyto);
-	if(efromismod) register_unformatted(txt, "moderators", efromismod);
-	else register_unformatted(txt, "moderators", moderators);
+	register_unformatted(txt, "moderators", "%moderators%"); /* DEPRECATED */
+	register_formatted(txt, "moderators",
+			rewind_memory_lines, get_memory_line, mls);
 	register_originalmail(txt, mailfilename);
 	queuefilename = prepstdreply(txt, listdir, "$listowner$", to, replyto);
 	MY_ASSERT(queuefilename);
@@ -174,6 +180,9 @@
 				pid = waitpid(childpid, &status, 0);
 			while(pid == -1 && errno == EINTR);
 		}
+
+		finish_memory_lines(mls);
+
 		if(efromismod)
 			execlp(mlmmjsend, mlmmjsend,
 					"-l", "1",
@@ -200,14 +209,17 @@
 	MY_ASSERT(txt);
 	register_unformatted(txt, "subject", subject);
 	register_unformatted(txt, "posteraddr", posteraddr);
-	if(efromismod) register_unformatted(txt, "moderators", efromismod);
-	else register_unformatted(txt, "moderators", moderators);
+	register_unformatted(txt, "moderators", "%moderators%"); /* DEPRECATED */
+	register_formatted(txt, "moderators",
+			rewind_memory_lines, get_memory_line, mls);
 	register_originalmail(txt, mailfilename);
 	queuefilename = prepstdreply(txt, listdir,
 			"$listowner$", efromsender, NULL);
 	MY_ASSERT(queuefilename);
 	close_text(txt);
 
+	finish_memory_lines(mls);
+
 	execlp(mlmmjsend, mlmmjsend,
 			"-l", "1",
 			"-L", listdir,
--- a/src/mlmmj-sub.c	Tue Jan 17 01:26:45 2012 +1100
+++ b/src/mlmmj-sub.c	Tue Jan 17 11:20:38 2012 +1100
@@ -72,6 +72,7 @@
 {
 	int i, fd, status, nosubmodmails = 0;
 	text *txt;
+	memory_lines_state *mls;
 	char *a = NULL, *queuefilename, *from, *listname, *listfqdn, *str;
 	char *modfilename, *randomstr, *mods, *to, *replyto, *moderators = NULL;
 	char *modfilebase;
@@ -156,6 +157,8 @@
 		moderators = concatstr(3, moderators, submods->strs[i], "\n");
 		myfree(str);
 	}
+	mls = init_memory_lines(moderators);
+	myfree(moderators);
 
 	txt = open_text(listdir,
 			"gatekeep", "sub", NULL, NULL, "submod-moderator");
@@ -163,7 +166,9 @@
 	register_unformatted(txt, "subaddr", subaddr);
 	register_unformatted(txt, "moderateaddr", replyto); /* DEPRECATED */
 	register_unformatted(txt, "permitaddr", replyto);
-	register_unformatted(txt, "moderators", moderators);
+	register_unformatted(txt, "moderators", "%gatekeepers"); /* DEPRECATED */
+	register_formatted(txt, "gatekeepers",
+			rewind_memory_lines, get_memory_line, mls);
 	queuefilename = prepstdreply(txt, listdir, "$listowner$", to, replyto);
 	MY_ASSERT(queuefilename);
 	close_text(txt);
@@ -186,6 +191,7 @@
 				pid = waitpid(childpid, &status, 0);
 			while(pid == -1 && errno == EINTR);
 		}
+		finish_memory_lines(mls);
 		execl(mlmmjsend, mlmmjsend,
 				"-a",
 				"-l", "4",
@@ -209,15 +215,17 @@
 			"wait", "sub", NULL, NULL, "submod-requester");
 	MY_ASSERT(txt);
 	register_unformatted(txt, "subaddr", subaddr);
-	register_unformatted(txt, "moderators", moderators);
+	register_unformatted(txt, "moderators", "%gatekeepers"); /* DEPRECATED */
+	register_formatted(txt, "gatekeepers",
+			rewind_memory_lines, get_memory_line, mls);
 	queuefilename = prepstdreply(txt, listdir,
 			"$listowner$", subaddr, NULL);
 	MY_ASSERT(queuefilename);
 	close_text(txt);
 
+	finish_memory_lines(mls);
 	myfree(listname);
 	myfree(listfqdn);
-	myfree(moderators);
 	execl(mlmmjsend, mlmmjsend,
 				"-l", "1",
 				"-L", listdir,
--- a/src/prepstdreply.c	Tue Jan 17 01:26:45 2012 +1100
+++ b/src/prepstdreply.c	Tue Jan 17 11:20:38 2012 +1100
@@ -46,19 +46,6 @@
 #include "unistr.h"
 
 
-struct source;
-typedef struct source source;
-struct source {
-	source *prev;
-	char *upcoming;
-	char *prefix;
-	char *suffix;
-	int fd;
-	int transparent;
-	int limit;
-};
-
-
 struct substitution;
 typedef struct substitution substitution;
 struct substitution {
@@ -68,13 +55,201 @@
 };
 
 
+struct formatted;
+typedef struct formatted formatted;
+struct formatted {
+	char *token;
+	rewind_function rew;
+	get_function get;
+	void *state;
+	formatted *next;
+};
+
+
+struct source;
+typedef struct source source;
+struct source {
+	source *prev;
+	char *upcoming;
+	char *prefix;
+	char *suffix;
+	int fd;
+	formatted *fmt;
+	int transparent;
+	int limit;
+};
+
+
 struct text {
 	source *src;
 	substitution *substs;
 	char *mailname;
+	formatted *fmts;
 };
 
 
+struct memory_lines_state {
+	char *lines;
+	char *pos;
+};
+
+
+struct file_lines_state {
+	char *filename;
+	int fd;
+	char truncate;
+	char *line;
+};
+
+
+memory_lines_state *init_memory_lines(const char *lines)
+{
+	/* We use a static variable rather than dynamic allocation as
+	 * there will never be two lists in use simultaneously */
+	static memory_lines_state s;
+	size_t len;
+
+	/* Ensure there *is* a trailing newline */
+	s.pos = NULL;
+	len = strlen(lines);
+	if (lines[len-1] == '\n') {
+		s.lines = mystrdup(lines);
+		return &s;
+	}
+	s.lines = mymalloc((len + 2) * sizeof(char));
+	strcpy(s.lines, lines);
+	s.lines[len] = '\n';
+	s.lines[len+1] = '\0';
+	return &s;
+}
+
+
+void rewind_memory_lines(void *state)
+{
+	memory_lines_state *s = (memory_lines_state *)state;
+	if (s == NULL) return;
+	s->pos = NULL;
+}
+
+
+const char *get_memory_line(void *state)
+{
+	memory_lines_state *s = (memory_lines_state *)state;
+	char *line, *pos;
+
+	if (s == NULL) return NULL;
+
+	if (s->pos != NULL) *s->pos++ = '\n';
+	else s->pos = s->lines;
+
+	line = s->pos;
+	pos = line;
+
+	if (*pos == '\0') {
+		s->pos = NULL;
+		return NULL;
+	}
+
+	while (*pos != '\n') pos++;
+	*pos = '\0';
+
+	s->pos = pos;
+	return line;
+}
+
+
+void finish_memory_lines(memory_lines_state *s)
+{
+	if (s == NULL) return;
+	myfree(s->lines);
+}
+
+
+file_lines_state *init_file_lines(const char *filename, int open_now)
+{
+	/* We use a static variable rather than dynamic allocation as
+	 * there will never be two lists in use simultaneously */
+	static file_lines_state s;
+
+	if (open_now) {
+		s.fd = open(filename, O_RDONLY);
+		s.filename = NULL;
+		if (s.fd < 0) return NULL;
+	} else {
+		s.filename = mystrdup(filename);
+		s.fd = -1;
+	}
+
+	s.truncate = '\0';
+	s.line = NULL;
+	return &s;
+}
+
+
+file_lines_state *init_truncated_file_lines(const char *filename, int open_now,
+		char truncate)
+{
+	file_lines_state *s;
+	s = init_file_lines(filename, open_now);
+	if (s == NULL) return NULL;
+	s->truncate = truncate;
+	return s;
+}
+
+
+void rewind_file_lines(void *state)
+{
+	file_lines_state *s = (file_lines_state *)state;
+	if (s == NULL) return;
+	if (s->filename != NULL) {
+		s->fd = open(s->filename, O_RDONLY);
+		myfree(s->filename);
+		s->filename = NULL;
+	}
+	if (s->fd >= 0) {
+		if(lseek(s->fd, 0, SEEK_SET) < 0) {
+			log_error(LOG_ARGS, "Could not seek to start of file");
+			close(s->fd);
+			s->fd = -1;
+		}
+	}
+}
+
+
+const char *get_file_line(void *state)
+{
+	file_lines_state *s = (file_lines_state *)state;
+	char *end;
+	if (s == NULL) return NULL;
+	if (s->line != NULL) {
+		myfree(s->line);
+		s->line = NULL;
+	}
+	if (s->fd >= 0) {
+		s->line = mygetline(s->fd);
+		if (s->line == NULL) return NULL;
+		if (s->truncate != '\0') {
+			end = strchr(s->line, s->truncate);
+			if (end == NULL) return NULL;
+			*end = '\0';
+		} else {
+			chomp(s->line);
+		}
+		return s->line;
+	}
+	return NULL;
+}
+
+
+void finish_file_lines(file_lines_state *s)
+{
+	if (s == NULL) return;
+	if (s->line != NULL) myfree(s->line);
+	if (s->fd >= 0) close(s->fd);
+	if (s->filename != NULL) myfree(s->filename);
+}
+
+
 static char *filename_token(char *token) {
 	char *pos;
 	if (*token == '\0') return NULL;
@@ -246,8 +421,10 @@
 	txt->src->suffix = NULL;
 	txt->src->transparent = 0;
 	txt->src->limit = -1;
+	txt->src->fmt = NULL;
 	txt->substs = NULL;
 	txt->mailname = NULL;
+	txt->fmts = NULL;
 
 	tmp = concatstr(3, listdir, "/text/", filename);
 	txt->src->fd = open(tmp, O_RDONLY);
@@ -308,6 +485,17 @@
 }
 
 
+void close_source(text *txt) {
+	source *tmp;
+	if (txt->src->fd != -1) close(txt->src->fd);
+	if (txt->src->prefix != NULL) myfree(txt->src->prefix);
+	if (txt->src->suffix != NULL) myfree(txt->src->suffix);
+	tmp = txt->src;
+	txt->src = txt->src->prev;
+	myfree(tmp);
+}
+
+
 void register_unformatted(text *txt, const char *token, const char *replacement)
 {
 	substitution * subst = mymalloc(sizeof(substitution));
@@ -324,6 +512,19 @@
 }
 
 
+void register_formatted(text *txt, const char *token,
+		rewind_function rew, get_function get, void *state)
+{
+	formatted * fmt = mymalloc(sizeof(formatted));
+	fmt->token = mystrdup(token);
+	fmt->rew = rew;
+	fmt->get = get;
+	fmt->state = state;
+	fmt->next = txt->fmts;
+	txt->fmts = fmt;
+}
+
+
 static void begin_new_source_file(text *txt, char **line_p, char **pos_p,
 		const char *filename) {
 	char *line = *line_p;
@@ -355,10 +556,16 @@
 	*tmp = '\0';
 	src->suffix = NULL;
 	src->fd = fd;
+	src->fmt = NULL;
 	src->transparent = 0;
 	src->limit = -1;
 	txt->src = src;
 	tmp = mygetline(fd);
+	if (tmp == NULL) {
+		close_source(txt);
+		**pos_p = '\0';
+		return;
+	}
 	line = concatstr(2, line, tmp);
 	*pos_p = line + (*pos_p - *line_p);
 	myfree(*line_p);
@@ -367,6 +574,54 @@
 }
 
 
+static void begin_new_formatted_source(text *txt, char **line_p, char **pos_p,
+		char *suffix, formatted *fmt) {
+	char *line = *line_p;
+	char *pos = *pos_p;
+	const char *str;
+	source *src;
+
+	/* Save any later lines for use after finishing the source */
+	while (*pos != '\0' && *pos != '\r' && *pos != '\n') pos++;
+	if (*pos == '\r') pos++;
+	if (*pos == '\n') pos++;
+	if (*pos != '\0') txt->src->upcoming = mystrdup(pos);
+
+	(*fmt->rew)(fmt->state);
+
+	src = mymalloc(sizeof(source));
+	src->prev = txt->src;
+	src->upcoming = NULL;
+	if (*line == '\0') {
+		src->prefix = NULL;
+	} else {
+		src->prefix = mystrdup(line);
+	}
+	if (*suffix == '\0' || *suffix == '\r' || *suffix == '\n') {
+		src->suffix = NULL;
+	} else {
+		src->suffix = mystrdup(suffix);
+	}
+	src->fd = -1;
+	src->fmt = fmt;
+	src->transparent = 0;
+	src->limit = -1;
+	txt->src = src;
+	str = (*fmt->get)(fmt->state);
+	if (str == NULL) {
+		close_source(txt);
+		**line_p = '\0';
+		*pos_p = *line_p;
+		return;
+	}
+	line = concatstr(2, line, str);
+	/* The suffix will be added back in get_processed_text_line() */
+	*pos_p = line + strlen(line);
+	myfree(*line_p);
+	*line_p = line;
+}
+
+
 static void handle_directive(text *txt, char **line_p, char **pos_p,
 		const char *listdir) {
 	char *line = *line_p;
@@ -375,6 +630,7 @@
 	char *endpos;
 	char *filename;
 	int limit;
+	formatted *fmt;
 
 	endpos = strchr(token, '%');
 	if (endpos == NULL) {
@@ -457,6 +713,16 @@
 		return;
 	}
 
+	fmt = txt->fmts;
+	while (fmt != NULL) {
+		if (strcmp(token, fmt->token) == 0) {
+			begin_new_formatted_source(txt, line_p, pos_p,
+					endpos + 1, fmt);
+			return;
+		}
+		fmt = fmt->next;
+	}
+
 	/* No recognised directive; just advance through the string. */
 	*pos = '%';
 	*endpos = '%';
@@ -472,7 +738,7 @@
 	char *line = NULL;
 	char *pos;
 	char *tmp;
-	source *src;
+	const char *item;
 
 	while (txt->src != NULL) {
 		if (txt->src->upcoming != NULL) {
@@ -486,16 +752,22 @@
 			break;
 		}
 		if (txt->src->limit != 0) {
+			if (txt->src->fd != -1) {
 			txt->src->upcoming = mygetline(txt->src->fd);
+			} else if (txt->src->fmt != NULL) {
+				item = (*txt->src->fmt->get)(
+						txt->src->fmt->state);
+				if (item == NULL) txt->src->upcoming = NULL;
+				else txt->src->upcoming = mystrdup(item);
+			} else {
+				txt->src->upcoming = NULL;
+			}
 			if (txt->src->limit > 0) txt->src->limit--;
 		} else {
 			txt->src->upcoming = NULL;
 		}
 		if (txt->src->upcoming != NULL) continue;
-		close(txt->src->fd);
-		src = txt->src;
-		txt->src = txt->src->prev;
-		myfree(src);
+		close_source(txt);
 	}
 	if (line == NULL) return NULL;
 
@@ -547,13 +819,10 @@
 
 
 void close_text(text *txt) {
-	source *tmp;
 	substitution *subst;
+	formatted *fmt;
 	while (txt->src != NULL) {
-		close(txt->src->fd);
-		tmp = txt->src;
-		txt->src = txt->src->prev;
-		myfree(tmp);
+		close_source(txt);
 	}
 	while (txt->substs != NULL) {
 		subst = txt->substs;
@@ -563,6 +832,12 @@
 		myfree(subst);
 	}
 	if (txt->mailname != NULL) myfree(txt->mailname);
+	while (txt->fmts != NULL) {
+		fmt = txt->fmts;
+		myfree(fmt->token);
+		txt->fmts = txt->fmts->next;
+		myfree(fmt);
+	}
 	myfree(txt);
 }
 
--- a/src/send_digest.c	Tue Jan 17 01:26:45 2012 +1100
+++ b/src/send_digest.c	Tue Jan 17 11:20:38 2012 +1100
@@ -59,20 +59,57 @@
 };
 
 
-static char *thread_list(const char *listdir, int firstindex, int lastindex)
+struct thread_list_state;
+typedef struct thread_list_state thread_list_state;
+struct thread_list_state {
+	const char *listdir;
+	int firstindex;
+	int lastindex;
+	int num_threads;
+	struct thread *threads;
+	int cur_thread;
+	int cur_mail;
+};
+
+
+static thread_list_state *init_thread_list(
+		const char *listdir, int firstindex, int lastindex)
 {
+	/* We use a static variable rather than dynamic allocation as
+	 * there will never be two lists in use simultaneously */
+	static thread_list_state s;
+	s.listdir = listdir;
+	s.firstindex = firstindex;
+	s.lastindex = lastindex;
+	s.num_threads = 0;
+	s.threads = NULL;
+	s.cur_thread = -1;
+	return &s;
+}
+
+
+static void rewind_thread_list(void * state)
+{
+	thread_list_state *s = (thread_list_state *)state;
 	int i, j, archivefd, thread_idx;
-	char *ret, *line, *tmp, *subj, *from;
+	char *line, *tmp, *subj, *from;
 	char *archivename;
 	int num_threads = 0;
 	struct thread *threads = NULL;
 	char buf[45];
 
-	for (i=firstindex; i<=lastindex; i++) {
+	if (s->cur_thread != -1) {
+		/* We have gathered the data already; just rewind */
+		s->cur_thread = 0;
+		s->cur_mail = -1;
+		return;
+	}
+
+	for (i=s->firstindex; i<=s->lastindex; i++) {
 
 		snprintf(buf, sizeof(buf), "%d", i);
 
-		archivename = concatstr(3, listdir, "/archive/", buf);
+		archivename = concatstr(3, s->listdir, "/archive/", buf);
 		archivefd = open(archivename, O_RDONLY);
 		myfree(archivename);
 
@@ -131,7 +168,7 @@
 			num_threads++;
 			threads = myrealloc(threads,
 					num_threads*sizeof(struct thread));
-			threads[num_threads-1].subject = mystrdup(tmp);
+			threads[num_threads-1].subject = concatstr(2,tmp,"\n");
 			threads[num_threads-1].num_mails = 0;
 			threads[num_threads-1].mails = NULL;
 			thread_idx = num_threads-1;
@@ -150,30 +187,47 @@
 		close(archivefd);
 	}
 
-	ret = mystrdup("");
+	s->num_threads = num_threads;
+	s->threads = threads;
+	s->cur_thread = 0;
+	s->cur_mail = -1;
+}
 
-	for (i=0; i<num_threads; i++) {
 
-		tmp = concatstr(3, ret, threads[i].subject, "\n");
-		myfree(ret);
-		ret = tmp;
-		myfree(threads[i].subject);
+static const char *get_thread_list_line(void * state)
+{
+	thread_list_state *s = (thread_list_state *)state;
+
+	if (s->cur_thread >= s->num_threads) return NULL;
+
+	if (s->cur_mail == -1) {
+		s->cur_mail = 0;
+		return s->threads[s->cur_thread].subject;
+	}
 
-		for (j=0; j<threads[i].num_mails; j++) {
-			tmp = concatstr(2, ret, threads[i].mails[j].from);
-			myfree(ret);
-			ret = tmp;
-			myfree(threads[i].mails[j].from);
+	if (s->cur_mail >= s->threads[s->cur_thread].num_mails) {
+		s->cur_thread++;
+		s->cur_mail = -1;
+		return "\n";
 		}
-		myfree(threads[i].mails);
+
+	return s->threads[s->cur_thread].mails[s->cur_mail++].from;
+}
+
 
-		tmp = concatstr(2, ret, "\n");
-		myfree(ret);
-		ret = tmp;
+static void finish_thread_list(thread_list_state * s)
+{
+	int i, j;
+	if (s->threads == NULL) return;
+	for (i=0; i<s->num_threads; i++) {
+		myfree(s->threads[i].subject);
+		for (j=0; j<s->threads[i].num_mails; j++) {
+			myfree(s->threads[i].mails[j].from);
 	}
-	myfree(threads);
-
-	return ret;
+		myfree(s->threads[i].mails);
+	}
+	myfree(s->threads);
+	s->threads = NULL;
 }
 
 
@@ -187,6 +241,7 @@
 	char *tmp, *queuename = NULL, *archivename, *subject = NULL, *line = NULL;
 	char *boundary, *listaddr, *listdelim, *listname, *listfqdn;
 	pid_t childpid, pid;
+	thread_list_state * tls;
 
 	if (addr) {
 		errno = 0;
@@ -246,8 +301,11 @@
 	snprintf(buf, sizeof(buf), "%d", issue);
 	register_unformatted(txt, "digestissue", buf);
 
-	tmp = thread_list(listdir, firstindex, lastindex);
-	register_unformatted(txt, "digestthreads", tmp);
+	register_unformatted(txt, "digestthreads", "%digestthreads%"); /* DEPRECATED */
+
+	tls = init_thread_list(listdir, firstindex, lastindex);
+	register_formatted(txt, "digestthreads", rewind_thread_list,
+			get_thread_list_line, tls);
 
 	line = get_processed_text_line(txt, listaddr, listdelim, listdir);
 
@@ -390,8 +448,10 @@
 			myfree(line);
 		}
 
+		finish_thread_list(tls);
 		close_text(txt);
 	} else if (txt != NULL) {
+		finish_thread_list(tls);
 		close_text(txt);
 	}
 
--- a/src/send_list.c	Tue Jan 17 01:26:45 2012 +1100
+++ b/src/send_list.c	Tue Jan 17 11:20:38 2012 +1100
@@ -43,50 +43,119 @@
 #include "memory.h"
 
 
+struct subs_list_state;
+typedef struct subs_list_state subs_list_state;
+struct subs_list_state {
+	char *dirname;
+	DIR *dirp;
+	int fd;
+	char *line;
+	int used;
+};
 
-static void print_subs(int cur_fd, char *dirname)
+
+static subs_list_state *init_subs_list(const char *dirname)
 {
-	char *fileiter;
-	DIR *dirp;
+	/* We use a static variable rather than dynamic allocation as
+	 * there will never be two lists in use simultaneously */
+	static subs_list_state s;
+	s.dirname = mystrdup(dirname);
+	s.dirp = NULL;
+	s.fd = -1;
+	s.used = 0;
+	return &s;
+}
+
+
+static void rewind_subs_list(void *state)
+{
+	subs_list_state *s = (subs_list_state *)state;
+	if (s == NULL) return;
+	if (s->dirp != NULL) closedir(s->dirp);
+	s->dirp = opendir(s->dirname);
+	if(s->dirp == NULL) {
+		log_error(LOG_ARGS, "Could not opendir(%s);\n", s->dirname);
+	}
+	s->used = 1;
+}
+
+
+static const char *get_sub(void *state)
+{
+	subs_list_state *s = (subs_list_state *)state;
+	char *filename;
 	struct dirent *dp;
-	int subfd;
+
+	if (s == NULL) return NULL;
+	if (s->dirp == NULL) return NULL;
+
+	if (s->line != NULL) {
+		myfree(s->line);
+		s->line = NULL;
+	}
 
-	dirp = opendir(dirname);
-	if(dirp == NULL) {
-		fprintf(stderr, "Could not opendir(%s);\n", dirname);
-		exit(EXIT_FAILURE);
+	for (;;) {
+		if (s->fd == -1) {
+			dp = readdir(s->dirp);
+			if (dp == NULL) {
+				closedir(s->dirp);
+				s->dirp = NULL;
+				return NULL;
 	}
-	while((dp = readdir(dirp)) != NULL) {
 		if((strcmp(dp->d_name, "..") == 0) ||
 		   (strcmp(dp->d_name, ".") == 0))
 			continue;
-
-		fileiter = concatstr(2, dirname, dp->d_name);
-		subfd = open(fileiter, O_RDONLY);
-		if(subfd < 0) {
-			log_error(LOG_ARGS, "Could not open %s for reading",
-					fileiter);
-			myfree(fileiter);
+			filename = concatstr(2, s->dirname, dp->d_name);
+			s->fd = open(filename, O_RDONLY);
+			if(s->fd < 0) {
+				log_error(LOG_ARGS,
+						"Could not open %s for reading",
+						filename);
+				myfree(filename);
+				continue;
+			}
+			myfree(filename);
+		}
+		s->line = mygetline(s->fd);
+		if (s->line == NULL) {
+			close(s->fd);
+			s->fd = -1;
 			continue;
 		}
-		if(dumpfd2fd(subfd, cur_fd) < 0) {
-			log_error(LOG_ARGS, "Error dumping subfile content"
-					" of %s to sub list mail",
-					fileiter);
+		chomp(s->line);
+		return s->line;
+	}
 		}
 
-		close(subfd);
-		myfree(fileiter);
-	}
-	closedir(dirp);
+
+static void finish_subs_list(subs_list_state *s)
+{
+	if (s == NULL) return;
+	if (s->line != NULL) myfree(s->line);
+	if (s->fd != -1) close(s->fd);
+	if (s->dirp != NULL) closedir(s->dirp);
+	myfree(s->dirname);
 }
 
 
+static void print_subs(int fd, subs_list_state *s)
+{
+	const char *sub;
+	rewind_subs_list(s);
+	while ((sub = get_sub(s)) != NULL) {
+		if (writen(fd, sub, strlen(sub)) < 0) {
+			log_error(LOG_ARGS, "error writing subs list");
+		}
+		writen(fd, "\n", 1);
+	}
+}
+
 
 void send_list(const char *listdir, const char *emailaddr,
 	       const char *mlmmjsend)
 {
 	text *txt;
+	subs_list_state *subsls, *digestsls, *nomailsls;
 	char *queuefilename, *listaddr, *listdelim, *listname, *listfqdn;
 	char *fromaddr, *subdir, *nomaildir, *digestdir;
 	int fd;
@@ -99,37 +168,53 @@
 	fromaddr = concatstr(4, listname, listdelim, "bounces-help@", listfqdn);
 	myfree(listdelim);
 
+	subdir = concatstr(2, listdir, "/subscribers.d/");
+	digestdir = concatstr(2, listdir, "/digesters.d/");
+	nomaildir = concatstr(2, listdir, "/nomailsubs.d/");
+	subsls = init_subs_list(subdir);
+	digestsls = init_subs_list(digestdir);
+	nomailsls = init_subs_list(nomaildir);
+	myfree(subdir);
+	myfree(digestdir);
+	myfree(nomaildir);
+
 	txt = open_text(listdir, "list", NULL, NULL, subtype_strs[SUB_ALL],
 			"listsubs");
 	MY_ASSERT(txt);
+	register_formatted(txt, "subs",
+			rewind_subs_list, get_sub, subsls);
+	register_formatted(txt, "digestsubs",
+			rewind_subs_list, get_sub, digestsls);
+	register_formatted(txt, "nomailsubs",
+			rewind_subs_list, get_sub, nomailsls);
 	queuefilename = prepstdreply(txt, listdir, "$listowner$", emailaddr, NULL);
 	MY_ASSERT(queuefilename);
 	close_text(txt);
 
+	/* DEPRECATED */
+	/* Add lists manually if they weren't encountered in the list text */
+	if (!subsls->used && !digestsls->used && !nomailsls->used) {
 	fd = open(queuefilename, O_WRONLY);
 	if(fd < 0) {
 		log_error(LOG_ARGS, "Could not open sub list mail");
 		exit(EXIT_FAILURE);
 	}
-
 	if(lseek(fd, 0, SEEK_END) < 0) {
 		log_error(LOG_ARGS, "Could not seek to end of file");
 		exit(EXIT_FAILURE);
 	}
-
-	subdir = concatstr(2, listdir, "/subscribers.d/");
-	nomaildir = concatstr(2, listdir, "/nomailsubs.d/");
-	digestdir = concatstr(2, listdir, "/digesters.d/");
-
-	print_subs(fd, subdir);
+		print_subs(fd, subsls);
+		writen(fd, "\n-- \n", 5);
+		print_subs(fd, nomailsls);
 	writen(fd, "\n-- \n", 5);
-	print_subs(fd, nomaildir);
-	writen(fd, "\n-- \n", 5);
-	print_subs(fd, digestdir);
+		print_subs(fd, digestsls);
 	writen(fd, "\n-- \nend of output\n", 19);
+		close(fd);
+	}
 
-	close(fd);
-
+	finish_subs_list(subsls);
+	finish_subs_list(digestsls);
+	finish_subs_list(nomailsls);
 	myfree(listaddr);
 	myfree(listname);
 	myfree(listfqdn);