From a16ada793a0b6750d01e5fd05172ba6f9f6c0cb0 Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Tue, 29 Sep 2015 18:36:20 +0300
Subject: [PATCH 1/7] Add whole thread[as mbox] link to message view


diff --git a/django/archives/mailarchives/templates/message.html b/django/archives/mailarchives/templates/message.html
index 1d09e24..d0eeebf 100644
--- a/django/archives/mailarchives/templates/message.html
+++ b/django/archives/mailarchives/templates/message.html
@@ -45,7 +45,7 @@ $(function(){
  </tr>
  <tr>
   <th>Message-ID:</th>
-  <td><a href="/message-id/{{msg.messageid}}">{{msg.messageid}}</a> (view <a href="/message-id/raw/{{msg.messageid}}">raw</a> or <a href="/message-id/flat/{{msg.messageid}}">whole thread</a>)</td>
+  <td><a href="/message-id/{{msg.messageid}}">{{msg.messageid}}</a> (view <a href="/message-id/raw/{{msg.messageid}}">raw</a> or <a href="/message-id/flat/{{msg.messageid}}">whole thread</a> <a href="/message-id/thread_raw/{{msg.messageid}}">[as mbox]</a>)</td>
  </tr>
  <tr>
   <th>Thread:</th>
diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index e011827..bfe0f26 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -255,6 +255,19 @@ SELECT id,_from,subject,date,messageid,has_attachment,parentid,datepath FROM t O
 	for id,_from,subject,date,messageid,has_attachment,parentid,parentpath in curs.fetchall():
 		yield {'id':id, 'mailfrom':_from, 'subject': subject, 'date': date, 'printdate': date.strftime("%Y-%m-%d %H:%M:%S"), 'messageid': messageid, 'hasattachment': has_attachment, 'parentid': parentid, 'indent': "&nbsp;" * len(parentpath)}
 
+def _retrieve_thread_raw_messages(threadid):
+	# Yeah, this is *way* too complicated for the django ORM Too
+	curs = connection.cursor()
+	curs.execute("""WITH RECURSIVE t(id, date, rawtxt) AS(
+  SELECT id, date, rawtxt FROM messages m WHERE m.threadid=%(threadid)s AND parentid IS NULL AND hiddenstatus IS NULL
+ UNION ALL
+  SELECT m.id, m.date, m.rawtxt FROM messages m INNER JOIN t ON t.id=m.parentid WHERE m.threadid=%(threadid)s AND hiddenstatus IS NULL
+)
+SELECT id, date, rawtxt FROM t;
+""", {'threadid': threadid})
+
+	for id, date,rawtxt in curs.fetchall():
+		yield {'id':id, 'date': date, 'rawtext': rawtxt}
 
 def _get_nextprevious(listmap, dt):
 	curs = connection.cursor()
@@ -390,6 +403,46 @@ def message_raw(request, msgid):
 	response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"'
 	return response
 
+@nocache
+def message_thread_raw(request, msgid):
+	if 'HTTP_AUTHORIZATION' in request.META:
+		auth = request.META['HTTP_AUTHORIZATION'].split()
+		if len(auth) != 2:
+			return HttpResponseForbidden("Invalid authentication")
+		if auth[0].lower() == "basic":
+			user, pwd = base64.b64decode(auth[1]).split(':')
+			if user == 'archives' and pwd == 'antispam':
+				try:
+					m = Message.objects.get(messageid=msgid)
+				except Message.DoesNotExist:
+					raise Http404('Message does not exist')
+
+				msgs = list(_retrieve_thread_raw_messages(m.threadid))
+				newest = calendar.timegm(max(msgs, key=lambda x: x['date'])['date'].utctimetuple())
+				if request.META.has_key('HTTP_IF_MODIFIED_SINCE') and not settings.DEBUG:
+					ims = parse_http_date_safe(request.META.get("HTTP_IF_MODIFIED_SINCE"))
+					if ims >= newest:
+						return HttpResponseNotModified()
+
+				payload, msgs = bytes(msgs[0]['rawtext']), msgs[1:]
+				while msgs:
+					msg,msgs = msgs[0], msgs[1:]
+					payload += b"\n"+bytes(msg['rawtext'])
+
+				r = HttpResponse(payload, content_type='text/plain')
+				r['X-pgthread'] = ":%s:" % m.threadid
+				return r
+
+			# Invalid password falls through
+		# Other authentication types fall through
+
+	# Require authentication
+	response = HttpResponse()
+	response.status_code = 401
+	response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"'
+	return response
+
+
 def testview(request, seqid):
 	m = Message.objects.get(pk=seqid)
 	try:
diff --git a/django/archives/urls.py b/django/archives/urls.py
index 16fc718..5ca545c 100644
--- a/django/archives/urls.py
+++ b/django/archives/urls.py
@@ -27,6 +27,7 @@ urlpatterns = patterns('',
     (r'^message-id/([^/]+)$', 'archives.mailarchives.views.message'),
     (r'^message-id/flat/([^/]+)$', 'archives.mailarchives.views.message_flat'),
     (r'^message-id/raw/([^/]+)$', 'archives.mailarchives.views.message_raw'),
+    (r'^message-id/thread_raw/([^/]+)$', 'archives.mailarchives.views.message_thread_raw'),
 
     # Search
     (r'^archives-search/', 'archives.mailarchives.views.search'),
-- 
2.4.3


From b998a628c8b43a851ea96762ee694cc4803be645 Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 02:49:41 +0300
Subject: [PATCH 2/7] Quote 'archives'/'antispam' in Basic Auth message


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index bfe0f26..45da93d 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -400,7 +400,7 @@ def message_raw(request, msgid):
 	# Require authentication
 	response = HttpResponse()
 	response.status_code = 401
-	response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"'
+	response['WWW-Authenticate'] = "Basic realm=\"Please authenticate with user 'archives' and password 'antispam'\""
 	return response
 
 @nocache
@@ -439,7 +439,7 @@ def message_thread_raw(request, msgid):
 	# Require authentication
 	response = HttpResponse()
 	response.status_code = 401
-	response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"'
+	response['WWW-Authenticate'] = "Basic realm=\"Please authenticate with user 'archives' and password 'antispam'\""
 	return response
 
 
-- 
2.4.3


From 3bd67deb2172e4c57b84c2471e8c3f39fb927a40 Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 03:28:08 +0300
Subject: [PATCH 3/7] Add settings.THREAD_MBOX_MAX_SIZE size limit on thread
 mbox response size


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index 45da93d..78c8d8f 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -417,18 +417,21 @@ def message_thread_raw(request, msgid):
 				except Message.DoesNotExist:
 					raise Http404('Message does not exist')
 
-				msgs = list(_retrieve_thread_raw_messages(m.threadid))
-				newest = calendar.timegm(max(msgs, key=lambda x: x['date'])['date'].utctimetuple())
+				payload, newest = b"", None
+				for msg in _retrieve_thread_raw_messages(m.threadid):
+					if newest is None:
+						newest = msg['date']
+					newest = max(newest , msg['date'] )
+					payload += b"\n"+bytes(msg['rawtext'])
+					if len(payload) > settings.THREAD_MBOX_MAX_SIZE:
+						return HttpResponseForbidden('Thread exceeded size limits.')
+
+				newest = calendar.timegm(newest.utctimetuple())
 				if request.META.has_key('HTTP_IF_MODIFIED_SINCE') and not settings.DEBUG:
 					ims = parse_http_date_safe(request.META.get("HTTP_IF_MODIFIED_SINCE"))
 					if ims >= newest:
 						return HttpResponseNotModified()
 
-				payload, msgs = bytes(msgs[0]['rawtext']), msgs[1:]
-				while msgs:
-					msg,msgs = msgs[0], msgs[1:]
-					payload += b"\n"+bytes(msg['rawtext'])
-
 				r = HttpResponse(payload, content_type='text/plain')
 				r['X-pgthread'] = ":%s:" % m.threadid
 				return r
diff --git a/django/archives/settings.py b/django/archives/settings.py
index 0a01c3b..57c06a8 100644
--- a/django/archives/settings.py
+++ b/django/archives/settings.py
@@ -155,6 +155,13 @@ MBOX_ARCHIVES_ROOT="/dev/null"
 SEARCH_CLIENTS = ('127.0.0.1',)
 API_CLIENTS = ('127.0.0.1',)
 
+# Hard limits applicable to on-the-fly mbox generation
+# see views.message_thread_raw.
+
+# Max size of response, in bytes
+# Enforced only *after* the databse fetches the data.
+THREAD_MBOX_MAX_SIZE = 1000000
+
 try:
 	from settings_local import *
 except ImportError:
-- 
2.4.3


From 49aceb5a8b8dc13514cc0f20f64547de37a6801c Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 04:17:06 +0300
Subject: [PATCH 4/7] Add settings.THREAD_MBOX_DISABLED emergency lever


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index 78c8d8f..906d279 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -405,6 +405,9 @@ def message_raw(request, msgid):
 
 @nocache
 def message_thread_raw(request, msgid):
+	if settings.THREAD_MBOX_DISABLED:
+		return HttpResponseForbidden("Downloading entire threads in mbox format is currently disabled.")
+
 	if 'HTTP_AUTHORIZATION' in request.META:
 		auth = request.META['HTTP_AUTHORIZATION'].split()
 		if len(auth) != 2:
diff --git a/django/archives/settings.py b/django/archives/settings.py
index 57c06a8..fc0fd6f 100644
--- a/django/archives/settings.py
+++ b/django/archives/settings.py
@@ -162,6 +162,10 @@ API_CLIENTS = ('127.0.0.1',)
 # Enforced only *after* the databse fetches the data.
 THREAD_MBOX_MAX_SIZE = 1000000
 
+# "The Ops Eject button": set this to true to turn all
+# requests into cheap 403's
+THREAD_MBOX_DISABLED = False
+
 try:
 	from settings_local import *
 except ImportError:
-- 
2.4.3


From bde21cf417b4b763dcf1666d0655b9ad35b0326f Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 05:25:17 +0300
Subject: [PATCH 5/7] Add settings.THREAD_MBOX_DB_BATCH_SIZE


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index 906d279..b9da0b0 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -255,7 +255,7 @@ SELECT id,_from,subject,date,messageid,has_attachment,parentid,datepath FROM t O
 	for id,_from,subject,date,messageid,has_attachment,parentid,parentpath in curs.fetchall():
 		yield {'id':id, 'mailfrom':_from, 'subject': subject, 'date': date, 'printdate': date.strftime("%Y-%m-%d %H:%M:%S"), 'messageid': messageid, 'hasattachment': has_attachment, 'parentid': parentid, 'indent': "&nbsp;" * len(parentpath)}
 
-def _retrieve_thread_raw_messages(threadid):
+def _retrieve_thread_raw_messages(threadid, batch_size):
 	# Yeah, this is *way* too complicated for the django ORM Too
 	curs = connection.cursor()
 	curs.execute("""WITH RECURSIVE t(id, date, rawtxt) AS(
@@ -266,8 +266,11 @@ def _retrieve_thread_raw_messages(threadid):
 SELECT id, date, rawtxt FROM t;
 """, {'threadid': threadid})
 
-	for id, date,rawtxt in curs.fetchall():
-		yield {'id':id, 'date': date, 'rawtext': rawtxt}
+	res = list(curs.fetchmany(batch_size))
+	while res:
+		for id, date,rawtxt in res:
+			yield {'id':id, 'date': date, 'rawtext': rawtxt}
+		res = list(curs.fetchmany(batch_size))
 
 def _get_nextprevious(listmap, dt):
 	curs = connection.cursor()
@@ -421,11 +424,17 @@ def message_thread_raw(request, msgid):
 					raise Http404('Message does not exist')
 
 				payload, newest = b"", None
-				for msg in _retrieve_thread_raw_messages(m.threadid):
+				# _retrieve_thread_raw_messages is a generator that
+				# used cursor.fetchmany(batch_size), so we use
+				# laziness to detect big threads before we take up
+				# too much memory, and bail out.
+				for msg in _retrieve_thread_raw_messages(m.threadid,batch_size=settings.THREAD_MBOX_DB_BATCH_SIZE):
 					if newest is None:
 						newest = msg['date']
+					else:
+						payload += b"\n"
 					newest = max(newest , msg['date'] )
-					payload += b"\n"+bytes(msg['rawtext'])
+					payload += bytes(msg['rawtext'])
 					if len(payload) > settings.THREAD_MBOX_MAX_SIZE:
 						return HttpResponseForbidden('Thread exceeded size limits.')
 
diff --git a/django/archives/settings.py b/django/archives/settings.py
index fc0fd6f..56eee35 100644
--- a/django/archives/settings.py
+++ b/django/archives/settings.py
@@ -158,8 +158,18 @@ API_CLIENTS = ('127.0.0.1',)
 # Hard limits applicable to on-the-fly mbox generation
 # see views.message_thread_raw.
 
-# Max size of response, in bytes
-# Enforced only *after* the databse fetches the data.
+# Fetch this many raw messages at a time from DB
+# when building a thread mbox.
+# The trade off is:
+#
+# network roundtrips + requests to server
+#   vs.
+# enforcing THREAD_MBOX_MAX_SIZE as early as possible
+#
+THREAD_MBOX_DB_BATCH_SIZE = 15
+
+# Max size of response, in bytes. If excedded a 403 will be sent.
+# Enforced after each batch is fetched from DB
 THREAD_MBOX_MAX_SIZE = 1000000
 
 # "The Ops Eject button": set this to true to turn all
-- 
2.4.3


From 2123a1d4e1f9d441e30d9e6a79ba62d226c42489 Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 08:13:38 +0300
Subject: [PATCH 6/7] Set Content-Disposition to hint downloaded filename to
 client


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index b9da0b0..8c59a48 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -258,18 +258,18 @@ SELECT id,_from,subject,date,messageid,has_attachment,parentid,datepath FROM t O
 def _retrieve_thread_raw_messages(threadid, batch_size):
 	# Yeah, this is *way* too complicated for the django ORM Too
 	curs = connection.cursor()
-	curs.execute("""WITH RECURSIVE t(id, date, rawtxt) AS(
-  SELECT id, date, rawtxt FROM messages m WHERE m.threadid=%(threadid)s AND parentid IS NULL AND hiddenstatus IS NULL
+	curs.execute("""WITH RECURSIVE t(id, date, subject, rawtxt) AS(
+  SELECT id, date, subject, rawtxt FROM messages m WHERE m.threadid=%(threadid)s AND parentid IS NULL AND hiddenstatus IS NULL
  UNION ALL
-  SELECT m.id, m.date, m.rawtxt FROM messages m INNER JOIN t ON t.id=m.parentid WHERE m.threadid=%(threadid)s AND hiddenstatus IS NULL
+  SELECT m.id, m.date, m.subject, m.rawtxt FROM messages m INNER JOIN t ON t.id=m.parentid WHERE m.threadid=%(threadid)s AND hiddenstatus IS NULL
 )
-SELECT id, date, rawtxt FROM t;
+SELECT id, date, subject, rawtxt FROM t ORDER by date;
 """, {'threadid': threadid})
 
 	res = list(curs.fetchmany(batch_size))
 	while res:
-		for id, date,rawtxt in res:
-			yield {'id':id, 'date': date, 'rawtext': rawtxt}
+		for id, date, subject, rawtxt in res:
+			yield {'id':id, 'date': date, 'subject': subject, 'rawtext': rawtxt}
 		res = list(curs.fetchmany(batch_size))
 
 def _get_nextprevious(listmap, dt):
@@ -408,6 +408,30 @@ def message_raw(request, msgid):
 
 @nocache
 def message_thread_raw(request, msgid):
+
+	def _get_mlname_for_threadid(tid):
+		"Returns ml name for tid if found, else None"
+
+		curs = connection.cursor()
+		curs.execute("""
+		SELECT ls.listname as mlname FROM list_threads lt
+		JOIN lists as ls
+		ON lt.threadid=%(threadid)s AND lt.listid = ls.listid""", {
+				'threadid': tid,
+				})
+		result = curs.fetchone()
+		mlname = None
+		if result:
+			mlname = result[0]
+		return mlname
+
+	def _get_filename_for_thread(tid, subject):
+
+		def escape_filename(s):
+			return re.sub('[^\-a-zA-Z0-9:_\[\]]','_',s)
+		mlname = _get_mlname_for_threadid(tid) or "pgml"
+		return '%s_%s.mbox' % (escape_filename(mlname),escape_filename(subject))
+
 	if settings.THREAD_MBOX_DISABLED:
 		return HttpResponseForbidden("Downloading entire threads in mbox format is currently disabled.")
 
@@ -423,14 +447,18 @@ def message_thread_raw(request, msgid):
 				except Message.DoesNotExist:
 					raise Http404('Message does not exist')
 
-				payload, newest = b"", None
-				# _retrieve_thread_raw_messages is a generator that
-				# used cursor.fetchmany(batch_size), so we use
-				# laziness to detect big threads before we take up
-				# too much memory, and bail out.
-				for msg in _retrieve_thread_raw_messages(m.threadid,batch_size=settings.THREAD_MBOX_DB_BATCH_SIZE):
+				payload = b""
+				newest = None
+				subject = "thread"
+
+				# _retrieve_thread_raw_messages is a generator that employs
+				# cursor.fetchmany() to batch DB fetches, so we can detect
+				# threads exceeding THREAD_MBOX_MAX_SIZE asap, and bail out.
+				for msg in _retrieve_thread_raw_messages(m.threadid,settings.THREAD_MBOX_DB_BATCH_SIZE):
 					if newest is None:
 						newest = msg['date']
+						subject = msg['subject'].decode(errors='replace',encoding='ascii').encode('ascii')
+						subject = subject.strip()
 					else:
 						payload += b"\n"
 					newest = max(newest , msg['date'] )
@@ -445,9 +473,10 @@ def message_thread_raw(request, msgid):
 						return HttpResponseNotModified()
 
 				r = HttpResponse(payload, content_type='text/plain')
+				filename = _get_filename_for_thread(m.threadid, subject)
+				r['Content-disposition'] = 'attachment; filename="%s"' % filename
 				r['X-pgthread'] = ":%s:" % m.threadid
 				return r
-
 			# Invalid password falls through
 		# Other authentication types fall through
 
-- 
2.4.3


From 16b50ad32373f7c876c2ae5a0480942354415388 Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 10:20:44 +0300
Subject: [PATCH 7/7] Disabling thread mbox dl should report 200 with a
 message, not 403


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index 8c59a48..4707a35 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -433,7 +433,8 @@ def message_thread_raw(request, msgid):
 		return '%s_%s.mbox' % (escape_filename(mlname),escape_filename(subject))
 
 	if settings.THREAD_MBOX_DISABLED:
-		return HttpResponseForbidden("Downloading entire threads in mbox format is currently disabled.")
+		return HttpResponse("Downloading threads in mbox format is currently disabled.",
+			                content_type='text/plain')
 
 	if 'HTTP_AUTHORIZATION' in request.META:
 		auth = request.META['HTTP_AUTHORIZATION'].split()
-- 
2.4.3

