From a16ada793a0b6750d01e5fd05172ba6f9f6c0cb0 Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Tue, 29 Sep 2015 18:36:20 +0300
Subject: [PATCH 1/4] Add whole thread[as mbox] link to message view


diff --git a/django/archives/mailarchives/templates/message.html b/django/archives/mailarchives/templates/message.html
index 1d09e24..d0eeebf 100644
--- a/django/archives/mailarchives/templates/message.html
+++ b/django/archives/mailarchives/templates/message.html
@@ -45,7 +45,7 @@ $(function(){
  </tr>
  <tr>
   <th>Message-ID:</th>
-  <td><a href="/message-id/{{msg.messageid}}">{{msg.messageid}}</a> (view <a href="/message-id/raw/{{msg.messageid}}">raw</a> or <a href="/message-id/flat/{{msg.messageid}}">whole thread</a>)</td>
+  <td><a href="/message-id/{{msg.messageid}}">{{msg.messageid}}</a> (view <a href="/message-id/raw/{{msg.messageid}}">raw</a> or <a href="/message-id/flat/{{msg.messageid}}">whole thread</a> <a href="/message-id/thread_raw/{{msg.messageid}}">[as mbox]</a>)</td>
  </tr>
  <tr>
   <th>Thread:</th>
diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index e011827..bfe0f26 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -255,6 +255,19 @@ SELECT id,_from,subject,date,messageid,has_attachment,parentid,datepath FROM t O
 	for id,_from,subject,date,messageid,has_attachment,parentid,parentpath in curs.fetchall():
 		yield {'id':id, 'mailfrom':_from, 'subject': subject, 'date': date, 'printdate': date.strftime("%Y-%m-%d %H:%M:%S"), 'messageid': messageid, 'hasattachment': has_attachment, 'parentid': parentid, 'indent': "&nbsp;" * len(parentpath)}
 
+def _retrieve_thread_raw_messages(threadid):
+	# Yeah, this is *way* too complicated for the django ORM Too
+	curs = connection.cursor()
+	curs.execute("""WITH RECURSIVE t(id, date, rawtxt) AS(
+  SELECT id, date, rawtxt FROM messages m WHERE m.threadid=%(threadid)s AND parentid IS NULL AND hiddenstatus IS NULL
+ UNION ALL
+  SELECT m.id, m.date, m.rawtxt FROM messages m INNER JOIN t ON t.id=m.parentid WHERE m.threadid=%(threadid)s AND hiddenstatus IS NULL
+)
+SELECT id, date, rawtxt FROM t;
+""", {'threadid': threadid})
+
+	for id, date,rawtxt in curs.fetchall():
+		yield {'id':id, 'date': date, 'rawtext': rawtxt}
 
 def _get_nextprevious(listmap, dt):
 	curs = connection.cursor()
@@ -390,6 +403,46 @@ def message_raw(request, msgid):
 	response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"'
 	return response
 
+@nocache
+def message_thread_raw(request, msgid):
+	if 'HTTP_AUTHORIZATION' in request.META:
+		auth = request.META['HTTP_AUTHORIZATION'].split()
+		if len(auth) != 2:
+			return HttpResponseForbidden("Invalid authentication")
+		if auth[0].lower() == "basic":
+			user, pwd = base64.b64decode(auth[1]).split(':')
+			if user == 'archives' and pwd == 'antispam':
+				try:
+					m = Message.objects.get(messageid=msgid)
+				except Message.DoesNotExist:
+					raise Http404('Message does not exist')
+
+				msgs = list(_retrieve_thread_raw_messages(m.threadid))
+				newest = calendar.timegm(max(msgs, key=lambda x: x['date'])['date'].utctimetuple())
+				if request.META.has_key('HTTP_IF_MODIFIED_SINCE') and not settings.DEBUG:
+					ims = parse_http_date_safe(request.META.get("HTTP_IF_MODIFIED_SINCE"))
+					if ims >= newest:
+						return HttpResponseNotModified()
+
+				payload, msgs = bytes(msgs[0]['rawtext']), msgs[1:]
+				while msgs:
+					msg,msgs = msgs[0], msgs[1:]
+					payload += b"\n"+bytes(msg['rawtext'])
+
+				r = HttpResponse(payload, content_type='text/plain')
+				r['X-pgthread'] = ":%s:" % m.threadid
+				return r
+
+			# Invalid password falls through
+		# Other authentication types fall through
+
+	# Require authentication
+	response = HttpResponse()
+	response.status_code = 401
+	response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"'
+	return response
+
+
 def testview(request, seqid):
 	m = Message.objects.get(pk=seqid)
 	try:
diff --git a/django/archives/urls.py b/django/archives/urls.py
index 16fc718..5ca545c 100644
--- a/django/archives/urls.py
+++ b/django/archives/urls.py
@@ -27,6 +27,7 @@ urlpatterns = patterns('',
     (r'^message-id/([^/]+)$', 'archives.mailarchives.views.message'),
     (r'^message-id/flat/([^/]+)$', 'archives.mailarchives.views.message_flat'),
     (r'^message-id/raw/([^/]+)$', 'archives.mailarchives.views.message_raw'),
+    (r'^message-id/thread_raw/([^/]+)$', 'archives.mailarchives.views.message_thread_raw'),
 
     # Search
     (r'^archives-search/', 'archives.mailarchives.views.search'),
-- 
2.4.3


From b998a628c8b43a851ea96762ee694cc4803be645 Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 02:49:41 +0300
Subject: [PATCH 2/4] Quote 'archives'/'antispam' in Basic Auth message


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index bfe0f26..45da93d 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -400,7 +400,7 @@ def message_raw(request, msgid):
 	# Require authentication
 	response = HttpResponse()
 	response.status_code = 401
-	response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"'
+	response['WWW-Authenticate'] = "Basic realm=\"Please authenticate with user 'archives' and password 'antispam'\""
 	return response
 
 @nocache
@@ -439,7 +439,7 @@ def message_thread_raw(request, msgid):
 	# Require authentication
 	response = HttpResponse()
 	response.status_code = 401
-	response['WWW-Authenticate'] = 'Basic realm="Please authenticate with user archives and password antispam"'
+	response['WWW-Authenticate'] = "Basic realm=\"Please authenticate with user 'archives' and password 'antispam'\""
 	return response
 
 
-- 
2.4.3


From 3bd67deb2172e4c57b84c2471e8c3f39fb927a40 Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 03:28:08 +0300
Subject: [PATCH 3/4] Add settings.THREAD_MBOX_MAX_SIZE size limit on thread
 mbox response size


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index 45da93d..78c8d8f 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -417,18 +417,21 @@ def message_thread_raw(request, msgid):
 				except Message.DoesNotExist:
 					raise Http404('Message does not exist')
 
-				msgs = list(_retrieve_thread_raw_messages(m.threadid))
-				newest = calendar.timegm(max(msgs, key=lambda x: x['date'])['date'].utctimetuple())
+				payload, newest = b"", None
+				for msg in _retrieve_thread_raw_messages(m.threadid):
+					if newest is None:
+						newest = msg['date']
+					newest = max(newest , msg['date'] )
+					payload += b"\n"+bytes(msg['rawtext'])
+					if len(payload) > settings.THREAD_MBOX_MAX_SIZE:
+						return HttpResponseForbidden('Thread exceeded size limits.')
+
+				newest = calendar.timegm(newest.utctimetuple())
 				if request.META.has_key('HTTP_IF_MODIFIED_SINCE') and not settings.DEBUG:
 					ims = parse_http_date_safe(request.META.get("HTTP_IF_MODIFIED_SINCE"))
 					if ims >= newest:
 						return HttpResponseNotModified()
 
-				payload, msgs = bytes(msgs[0]['rawtext']), msgs[1:]
-				while msgs:
-					msg,msgs = msgs[0], msgs[1:]
-					payload += b"\n"+bytes(msg['rawtext'])
-
 				r = HttpResponse(payload, content_type='text/plain')
 				r['X-pgthread'] = ":%s:" % m.threadid
 				return r
diff --git a/django/archives/settings.py b/django/archives/settings.py
index 0a01c3b..57c06a8 100644
--- a/django/archives/settings.py
+++ b/django/archives/settings.py
@@ -155,6 +155,13 @@ MBOX_ARCHIVES_ROOT="/dev/null"
 SEARCH_CLIENTS = ('127.0.0.1',)
 API_CLIENTS = ('127.0.0.1',)
 
+# Hard limits applicable to on-the-fly mbox generation
+# see views.message_thread_raw.
+
+# Max size of response, in bytes
+# Enforced only *after* the databse fetches the data.
+THREAD_MBOX_MAX_SIZE = 1000000
+
 try:
 	from settings_local import *
 except ImportError:
-- 
2.4.3


From 49aceb5a8b8dc13514cc0f20f64547de37a6801c Mon Sep 17 00:00:00 2001
From: Amir <Am-Ro@users.noreply.github.com>
Date: Wed, 30 Sep 2015 04:17:06 +0300
Subject: [PATCH 4/4] Add settings.THREAD_MBOX_DISABLED emergency lever


diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index 78c8d8f..906d279 100644
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -405,6 +405,9 @@ def message_raw(request, msgid):
 
 @nocache
 def message_thread_raw(request, msgid):
+	if settings.THREAD_MBOX_DISABLED:
+		return HttpResponseForbidden("Downloading entire threads in mbox format is currently disabled.")
+
 	if 'HTTP_AUTHORIZATION' in request.META:
 		auth = request.META['HTTP_AUTHORIZATION'].split()
 		if len(auth) != 2:
diff --git a/django/archives/settings.py b/django/archives/settings.py
index 57c06a8..fc0fd6f 100644
--- a/django/archives/settings.py
+++ b/django/archives/settings.py
@@ -162,6 +162,10 @@ API_CLIENTS = ('127.0.0.1',)
 # Enforced only *after* the databse fetches the data.
 THREAD_MBOX_MAX_SIZE = 1000000
 
+# "The Ops Eject button": set this to true to turn all
+# requests into cheap 403's
+THREAD_MBOX_DISABLED = False
+
 try:
 	from settings_local import *
 except ImportError:
-- 
2.4.3

