[Inteproxy-commits] r357 - in branches/compression: . inteproxy test
scm-commit at wald.intevation.org
scm-commit at wald.intevation.org
Fri Mar 2 11:23:30 CET 2012
Author: aheinecke
Date: 2012-03-02 11:23:30 +0100 (Fri, 02 Mar 2012)
New Revision: 357
Added:
branches/compression/test/test_decompressstream.py
Modified:
branches/compression/ChangeLog
branches/compression/inteproxy/decompressstream.py
branches/compression/inteproxy/proxycore.py
Log:
Fix Bug in decompressstream reading that caused the
read not to read everything correctly when a negative
parameter was passed after some parts had already been
read.
Added unit test for decompressstream reading.
Some cleanup in proxycore
Modified: branches/compression/ChangeLog
===================================================================
--- branches/compression/ChangeLog 2012-03-02 10:09:39 UTC (rev 356)
+++ branches/compression/ChangeLog 2012-03-02 10:23:30 UTC (rev 357)
@@ -1,3 +1,13 @@
+2012-03-02 Andre Heinecke <aheinecke at intevation.de>
+ * A test/decompressstream.py:
+ Added test for decompressed reading
+ * M inteproxy/decompressstream.py:
+ Fix reading of the complete stream after starting to read
+ small chunks.
+ * M inteproxy/proxycore.py:
+ Remove the response parameter again for the transfer_data
+ functions.
+
2012-02-29 Andre Heinecke <aheinecke at intevation.de>
* M inteproxy/proxycore.py:
Remove decompressed_read method.
Modified: branches/compression/inteproxy/decompressstream.py
===================================================================
--- branches/compression/inteproxy/decompressstream.py 2012-03-02 10:09:39 UTC (rev 356)
+++ branches/compression/inteproxy/decompressstream.py 2012-03-02 10:23:30 UTC (rev 357)
@@ -9,24 +9,34 @@
""" On the fly decompression of a data stream """
class DecompressStream(object):
- """A class to wrap around a data stream that contains
- compressed data.
+ """A class to wrap around a data stream that contains compressed data.
- The decompression object can be given at construction time.
+ The decompression object can be given on initalization.
"""
def __init__(self, infile, decompressobj):
+ """Initialize the DecompressStream Object
+
+ The parameter infile is used as input stream
+ and the parameter decompressobj to provide the decompression.
"""
- Initialize the DecompressStream with a input stream and a decompressor
- """
self.infile = infile
self.decompressor = decompressobj
- def read(self, amount):
- if amount == -1:
- return self.decompressor.decompress(self.infile.read())
+ def read(self, amount = -1):
+ """Decompressed the stream and returns the uncompressed data.
+
+ A negative parameter for amount indicates that the complete
+ stream should be decompressed.
+ """
decompressed_chunks = []
count = 0
+
+ if amount < 0:
+ compressed = self.decompressor.unconsumed_tail
+ compressed += self.infile.read()
+ return self.decompressor.decompress(compressed)
+
while count < amount:
max_read = amount - count
compressed = self.decompressor.unconsumed_tail
Modified: branches/compression/inteproxy/proxycore.py
===================================================================
--- branches/compression/inteproxy/proxycore.py 2012-03-02 10:09:39 UTC (rev 356)
+++ branches/compression/inteproxy/proxycore.py 2012-03-02 10:23:30 UTC (rev 357)
@@ -15,7 +15,6 @@
import BaseHTTPServer
import socket
import base64
-import zlib
# the ssl modules was introduced in Python 2.6. If available, we use
# that, otherwise the older ssl support from the socket module
@@ -353,7 +352,7 @@
self.rewrite_urls(response, do_rewrite)
self.send_headers(response)
self.transfer_data(response.read, self.wfile.write,
- response, chunked = do_chunked)
+ chunked = do_chunked)
def transfer_data_rewrite_chunked(self, response):
"""Transfers the incoming data of the origin server in chunks
@@ -363,8 +362,7 @@
transcoder_map = self.server.transcoder_map
prefix = self.server.get_inteproxy_url()
rewrite = transcoder_map.url_rewriter(prefix, self.log_debug)
- self.transfer_chunked_rewrite(rewrite, response.read, self.wfile.write,
- response)
+ self.transfer_chunked_rewrite(rewrite, response.read, self.wfile.write)
def wrap_read_write_debug(self, read, write):
# wrap the read/write functions if debug logging is active so
@@ -392,7 +390,7 @@
return read, write
- def transfer_chunked_rewrite(self, rewrite, read, write, response,
+ def transfer_chunked_rewrite(self, rewrite, read, write,
separator='>', length=4096):
"""Transfers data from read() to write() in chunks. The
data is splitted by a given separator.
@@ -406,7 +404,7 @@
append = data.append
while True:
- chunk = response.read(length)
+ chunk = read(length)
if not chunk:
break
@@ -435,7 +433,7 @@
rewritten = None
writer.finish()
- def transfer_data(self, read, write, response, length=None, chunked=False):
+ def transfer_data(self, read, write, length=None, chunked=False):
"""Transfer data from one 'file' to another in chunks
The files are given by their read and write methods so it
@@ -455,7 +453,7 @@
chunk_size = min(length, max_chunk_size)
else:
chunk_size = max_chunk_size
- chunk = response.read(chunk_size)
+ chunk = read(chunk_size)
if not chunk:
break
if length is not None:
Added: branches/compression/test/test_decompressstream.py
===================================================================
--- branches/compression/test/test_decompressstream.py (rev 0)
+++ branches/compression/test/test_decompressstream.py 2012-03-02 10:23:30 UTC (rev 357)
@@ -0,0 +1,44 @@
+# Copyright (C) 2012 by Intevation GmbH
+# Authors:
+# Andre Heinecke <aheinecke at intevation.de>
+#
+# This program is free software under the GPL (>=v2)
+# Read the file COPYING coming with the software for details
+
+"""Tests for the inteproxy.decompressstream module"""
+
+import unittest
+import StringIO
+import zlib
+
+from random import Random
+
+from inteproxy.decompressstream import DecompressStream
+
+class DecompressStreamTest(unittest.TestCase):
+
+ def test_read_amounts(self):
+ """Test for the ChunkedTransferEncodingWriter"""
+
+ DATA = "It's still magic even if you know how it's done."
+
+ compressed_data = zlib.compress(DATA)
+ compressed_stream = StringIO.StringIO(compressed_data)
+ decompressobj = zlib.decompressobj()
+
+ dstream = DecompressStream(compressed_stream, decompressobj)
+
+ # Test reading small "bites"
+ result = dstream.read(5)
+ self.assertEqual(result, DATA[:5])
+ result2 = dstream.read(1)
+ self.assertEqual(result2, DATA[5])
+ result3 = dstream.read(-123)
+ self.assertEqual(result + result2 + result3, DATA)
+
+ # Test reading everything
+ compressed_stream = StringIO.StringIO(compressed_data)
+ decompressobj = zlib.decompressobj()
+
+ dstream = DecompressStream(compressed_stream, decompressobj)
+ self.assertEqual(dstream.read(), DATA)
More information about the Inteproxy-commits
mailing list