Author: Benjamin Richter Date: Sun Jan 25 23:22:46 2015 +0100 Subject: [1.4.x] Fixed #24158 -- Allowed GZipMiddleware to work with streaming responses Backport of django.utils.text.compress_sequence and fix for django.middleware.gzip.GZipMiddleware when using iterators as response.content. Patch further backported to Django 1.2 by Raphaël Hertzog . Origin: backport, https://github.com/django/django/commit/1e39d0f6280abf34c7719db5e7ed1c333f5e5919 Bug: https://code.djangoproject.com/ticket/24158 --- a/django/middleware/gzip.py +++ b/django/middleware/gzip.py @@ -1,6 +1,6 @@ import re -from django.utils.text import compress_string +from django.utils.text import compress_string, compress_sequence from django.utils.cache import patch_vary_headers re_accepts_gzip = re.compile(r'\bgzip\b') @@ -12,8 +12,10 @@ class GZipMiddleware(object): on the Accept-Encoding header. """ def process_response(self, request, response): + # The response object can tell us whether content is a string or an iterable # It's not worth compressing non-OK or really short responses. - if response.status_code != 200 or len(response.content) < 200: + if response.status_code != 200 or (response._is_string and + len(response.content) < 200): return response patch_vary_headers(response, ('Accept-Encoding',)) @@ -32,7 +34,17 @@ class GZipMiddleware(object): if not re_accepts_gzip.search(ae): return response - response.content = compress_string(response.content) + if response._is_string: + compressed_content = compress_string(response.content) + response.content = compressed_content + response['Content-Length'] = str(len(response.content)) + else: + # If the response content is iterable we don't know the length, + # so delete the header. + del response['Content-Length'] + # Wrap the response content in a streaming gzip iterator (direct + # access to inner response._container) + response.content = compress_sequence(response._container) + response['Content-Encoding'] = 'gzip' - response['Content-Length'] = str(len(response.content)) return response --- a/django/utils/text.py +++ b/django/utils/text.py @@ -186,6 +186,40 @@ def compress_string(s): ustring_re = re.compile(u"([\u0080-\uffff])") +# Backported from django 1.5 +class StreamingBuffer(object): + def __init__(self): + self.vals = [] + + def write(self, val): + self.vals.append(val) + + def read(self): + ret = ''.join(self.vals) + self.vals = [] + return ret + + def flush(self): + return + + def close(self): + return + +# Backported from django 1.5 +# Like compress_string, but for iterators of strings. +def compress_sequence(sequence): + import gzip + buf = StreamingBuffer() + zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=buf) + # Output headers... + yield buf.read() + for item in sequence: + zfile.write(item) + zfile.flush() + yield buf.read() + zfile.close() + yield buf.read() + def javascript_quote(s, quote_double_quotes=False): def fix(match): --- a/django/http/__init__.py +++ b/django/http/__init__.py @@ -326,12 +326,7 @@ class HttpResponse(object): if not content_type: content_type = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE, self._charset) - if not isinstance(content, basestring) and hasattr(content, '__iter__'): - self._container = content - self._is_string = False - else: - self._container = [content] - self._is_string = True + self.content = content self.cookies = CompatCookie() if status: self.status_code = status @@ -408,8 +403,12 @@ class HttpResponse(object): return smart_str(''.join(self._container), self._charset) def _set_content(self, value): - self._container = [value] - self._is_string = True + if not isinstance(value, basestring) and hasattr(value, '__iter__'): + self._container = value + self._is_string = False + else: + self._container = [value] + self._is_string = True content = property(_get_content, _set_content) --- a/tests/regressiontests/middleware/tests.py +++ b/tests/regressiontests/middleware/tests.py @@ -1,8 +1,13 @@ # -*- coding: utf-8 -*- +import gzip +import random +import StringIO + from django.test import TestCase -from django.http import HttpRequest +from django.http import HttpRequest, HttpResponse from django.middleware.common import CommonMiddleware +from django.middleware.gzip import GZipMiddleware from django.conf import settings class CommonMiddlewareTest(TestCase): @@ -125,7 +130,7 @@ class CommonMiddlewareTest(TestCase): self.assertEquals(r.status_code, 301) self.assertEquals(r['Location'], 'http://www.testserver/middleware/slash/') - + # The following tests examine expected behavior given a custom urlconf that # overrides the default one through the request object. @@ -165,7 +170,7 @@ class CommonMiddlewareTest(TestCase): request = self._get_request('customurlconf/slash') request.urlconf = 'regressiontests.middleware.extra_urls' r = CommonMiddleware().process_request(request) - self.failIf(r is None, + self.failIf(r is None, "CommonMiddlware failed to return APPEND_SLASH redirect using request.urlconf") self.assertEquals(r.status_code, 301) self.assertEquals(r['Location'], 'http://testserver/middleware/customurlconf/slash/') @@ -209,7 +214,7 @@ class CommonMiddlewareTest(TestCase): request = self._get_request('customurlconf/needsquoting#') request.urlconf = 'regressiontests.middleware.extra_urls' r = CommonMiddleware().process_request(request) - self.failIf(r is None, + self.failIf(r is None, "CommonMiddlware failed to return APPEND_SLASH redirect using request.urlconf") self.assertEquals(r.status_code, 301) self.assertEquals( @@ -246,3 +251,80 @@ class CommonMiddlewareTest(TestCase): self.assertEquals(r.status_code, 301) self.assertEquals(r['Location'], 'http://www.testserver/middleware/customurlconf/slash/') + + +class GZipMiddlewareTest(TestCase): + """ + Tests the GZip middleware. + """ + short_string = "This string is too short to be worth compressing." + compressible_string = 'a' * 500 + uncompressible_string = ''.join(chr(random.randint(0, 255)) for _ in xrange(500)) + iterator_as_content = iter(compressible_string) + + def setUp(self): + self.req = HttpRequest() + self.req.META = { + 'SERVER_NAME': 'testserver', + 'SERVER_PORT': 80, + } + self.req.path = self.req.path_info = "/" + self.req.META['HTTP_ACCEPT_ENCODING'] = 'gzip, deflate' + self.req.META['HTTP_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1' + self.resp = HttpResponse() + self.resp.status_code = 200 + self.resp.content = self.compressible_string + self.resp['Content-Type'] = 'text/html; charset=UTF-8' + + @staticmethod + def decompress(gzipped_string): + return gzip.GzipFile(mode='rb', fileobj=StringIO.StringIO(gzipped_string)).read() + + def test_compress_response(self): + """ + Tests that compression is performed on responses with compressible content. + """ + r = GZipMiddleware().process_response(self.req, self.resp) + self.assertEqual(self.decompress(r.content), self.compressible_string) + self.assertEqual(r['Content-Encoding'], 'gzip') + self.assertEqual(r['Content-Length'], str(len(r.content))) + + def test_no_compress_short_response(self): + """ + Tests that compression isn't performed on responses with short content. + """ + self.resp.content = self.short_string + r = GZipMiddleware().process_response(self.req, self.resp) + self.assertEqual(r.content, self.short_string) + self.assertFalse(r.has_header('Content-Encoding')) + + def test_no_compress_compressed_response(self): + """ + Tests that compression isn't performed on responses that are already compressed. + """ + self.resp['Content-Encoding'] = 'deflate' + r = GZipMiddleware().process_response(self.req, self.resp) + self.assertEqual(r.content, self.compressible_string) + self.assertEqual(r['Content-Encoding'], 'deflate') + + def test_no_compress_ie_js_requests(self): + """ + Tests that compression isn't performed on JavaScript requests from Internet Explorer. + """ + self.req.META['HTTP_USER_AGENT'] = 'Mozilla/4.0 (compatible; MSIE 5.00; Windows 98)' + self.resp['Content-Type'] = 'application/javascript; charset=UTF-8' + r = GZipMiddleware().process_response(self.req, self.resp) + self.assertEqual(r.content, self.compressible_string) + self.assertFalse(r.has_header('Content-Encoding')) + + def test_streaming_compression(self): + """ + Tests that iterators as response content return a compressed stream without consuming + the whole response.content while doing so. + See #24158. + """ + self.resp.content = self.iterator_as_content + r = GZipMiddleware().process_response(self.req, self.resp) + self.assertEqual(self.decompress(''.join(r.content)), self.compressible_string) + self.assertEqual(r['Content-Encoding'], 'gzip') + self.assertFalse(r.has_header('Content-Length'))