[Inteproxy-commits] r352 - in branches/compression: . inteproxy
scm-commit at wald.intevation.org
scm-commit at wald.intevation.org
Thu Feb 23 18:53:21 CET 2012
Author: aheinecke
Date: 2012-02-23 18:53:21 +0100 (Thu, 23 Feb 2012)
New Revision: 352
Modified:
branches/compression/ChangeLog
branches/compression/inteproxy/httpmessage.py
branches/compression/inteproxy/proxycore.py
Log:
Move compression logic out of the httpmessage classes.
The decompression will still be transparent for the transfer_data
functions but no longer be handled by the Httpmessage class.
Also the accept-encoding header is no longer overwritten and is
only added if the client did not request gzip or deflate.
If the request already contained accept-encoding and no rewrite
is neccessary the response will stay encoded.
Modified: branches/compression/ChangeLog
===================================================================
--- branches/compression/ChangeLog 2012-02-23 15:26:28 UTC (rev 351)
+++ branches/compression/ChangeLog 2012-02-23 17:53:21 UTC (rev 352)
@@ -1,3 +1,21 @@
+2012-02-22 Andre Heinecke <aheinecke at intevation.de>
+
+ * M inteproxy/httpmessage.py:
+ Remove compression handling but still use the read function
+ of HTTPMessage for reading the entire body.
+ * M inteproxy/proxycore.py:
+ Move compression handling into the proxycore. Add method
+ get_decompress_object to select the correct decompression
+ algorithm and Only do decompression if the client has not
+ requested a compressed response or if we need to rewrite urls.
+
+2012-02-22 Andre Heinecke <aheinecke at intevation.de>
+
+ * M inteproxy/httpmessage.py:
+ Only decompress responses, remove the Content
+ Encoding header, decide compression on inititalization
+ of a httpresponse.
+
2012-02-23 Andre Heinecke <aheinecke at intevation.de>
* M test/test_inteproxy.py:
Modified: branches/compression/inteproxy/httpmessage.py
===================================================================
--- branches/compression/inteproxy/httpmessage.py 2012-02-23 15:26:28 UTC (rev 351)
+++ branches/compression/inteproxy/httpmessage.py 2012-02-23 17:53:21 UTC (rev 352)
@@ -1,7 +1,6 @@
-# Copyright (C) 2008, 2012 by Intevation GmbH
+# Copyright (C) 2008 by Intevation GmbH
# Authors:
# Bernhard Herzog <bh at intevation.de>
-# Andre Heinecke <aheinecke at intevation.de>
#
# This program is free software under the GPL (>=v2)
# Read the file COPYING coming with the software for details.
@@ -9,7 +8,6 @@
"""Abstractions for http request and response messages"""
from StringIO import StringIO
-import zlib
class HTTPMessage(object):
@@ -38,7 +36,6 @@
self.headers = headers
self.infile = infile
self._body = None
- self.decompressobj = None
self._body_stream = None
def debug_log_message(self, log_function):
@@ -80,30 +77,15 @@
raise NotImplementedError
def read(self, amount):
- """ Read data from the messagebody
- Parameters:
- amount -- the amount of (possibly compressed) bytes to read
- Returns:
- A string representing the data from the amount of bytes in
- the message.
- This can be significantly larger then the "amount" parameter.
- Decompression is done if the property decompressobj is set.
- """
if self._body_stream is None and self.body_has_been_read():
self._body_stream = StringIO(self.body)
-
if self._body_stream is not None:
- # Body stream is already decompressed
- return self._body_stream.read(amount)
-
- data = self.infile.read(amount)
-
- if self.decompressobj is None:
- # Nothing to decompress
- return data
+ data = self._body_stream.read(amount)
else:
- return self.decompressobj.decompress(data)
+ data = self.infile.read(amount)
+ return data
+
class HTTPRequestMessage(HTTPMessage):
"""Represents a HTTP Message for a request.
@@ -156,7 +138,6 @@
All of these parameters are available as attributes of the same
name.
- If the response is compressed it will be decompressed.
"""
def __init__(self, version, status, reason, headers, infile):
@@ -165,19 +146,6 @@
self.status = status
self.reason = reason
- # Decompress the response
- if self.headers.get("Content-Encoding") == "deflate":
- self.decompressobj = zlib.decompressobj(-zlib.MAX_WBITS)
- elif self.headers.get("Content-Encoding") == "gzip":
- self.decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS)
-
- if self.decompressobj:
- del self.headers["Content-Encoding"]
- # Need to extract the message fist to calculate
- # an uncompressed content length
- if int(self.headers.get("Content-Length", "0")):
- self.read_entire_message()
-
def debug_log_message(self, log_function):
log_function("HTTPResponseMessage: %s %s %s",
self.version, self.status, self.reason)
Modified: branches/compression/inteproxy/proxycore.py
===================================================================
--- branches/compression/inteproxy/proxycore.py 2012-02-23 15:26:28 UTC (rev 351)
+++ branches/compression/inteproxy/proxycore.py 2012-02-23 17:53:21 UTC (rev 352)
@@ -15,6 +15,7 @@
import BaseHTTPServer
import socket
import base64
+import zlib
# the ssl modules was introduced in Python 2.6. If available, we use
# that, otherwise the older ssl support from the socket module
@@ -222,8 +223,19 @@
extra_headers = [("Host", "%s:%d" % remote_address)]
- # Accept supported compression algorithms
- extra_headers.append(("Accept-Encoding", "gzip, deflate"))
+ # Request compression even if the client did not.
+ # In that case the response will be decompressed
+ if not client_request.headers.get("Accept-Encoding"):
+ extra_headers.append(("Accept-Encoding", "gzip, deflate"))
+ self.do_decompress_response = True
+ elif ( not "gzip" in client_request.headers["Accept-Encoding"] and
+ not "deflate" in client_request.headers["Accept-Encoding"] ):
+ client_request.headers["Accept-Encoding"] = \
+ ", ".join([client_request.headers["Accept-Encoding"],
+ "gzip", "deflate"])
+ self.do_decompress_response = True
+ else:
+ self.do_decompress_response = False
sock = None
@@ -309,6 +321,13 @@
self.send_header(header, value)
self.end_headers()
+ def get_decompress_object(self, response):
+ # Set up the response for decompression
+ if response.headers.get("Content-Encoding") == "deflate":
+ return zlib.decompressobj(-zlib.MAX_WBITS)
+ elif response.headers.get("Content-Encoding") == "gzip":
+ return zlib.decompressobj(16 + zlib.MAX_WBITS)
+
def handle_response(self, response):
# The HTTP version in the reply generated by send_response is
# taken from self.protocol_version. We simply set it to
@@ -321,6 +340,21 @@
do_rewrite = self.have_to_rewrite()
do_chunked = response.headers.get("Transfer-encoding") == "chunked"
+ if do_rewrite or self.do_decompress_response:
+ decompressor = self.get_decompress_object(response)
+ if decompressor:
+ def decompressed_read(amount):
+ if not response.body_has_been_read():
+ return decompressor.decompress(
+ HTTPResponseMessage.read(response, amount))
+ else:
+ return HTTPResponseMessage.read(response, amount)
+
+ response.read = decompressed_read
+ del response.headers["Content-Encoding"]
+ if int(response.headers.get("Content-Length", "0")):
+ response.read_entire_message()
+
if do_chunked and do_rewrite:
self.send_headers(response)
self.transfer_data_rewrite_chunked(response)
More information about the Inteproxy-commits
mailing list