[Inteproxy-commits] r324 - in branches/streaming: . inteproxy

[scm-commit@wald.intevation.org]

Author: teichmann
Date: 2011-12-23 18:16:09 +0100 (Fri, 23 Dec 2011)
New Revision: 324

Modified:
   branches/streaming/ChangeLog
   branches/streaming/inteproxy/proxycore.py
Log:
Added method to transcode URLs while streaming data.

Modified: branches/streaming/ChangeLog
===================================================================
--- branches/streaming/ChangeLog	2011-12-23 16:43:28 UTC (rev 323)
+++ branches/streaming/ChangeLog	2011-12-23 17:16:09 UTC (rev 324)
@@ -1,5 +1,11 @@
 2011-12-23	Sascha L. Teichmann	<sascha.teichmann at intevation.de>
 
+	* inteproxy/proxycore.py(transfer_chunked_rewrite): Added method
+	  to stream the data from incoming response to the output in chunks
+	  transcoding the URLs on the run. TODO: Integrate it.
+
+2011-12-23	Sascha L. Teichmann	<sascha.teichmann at intevation.de>
+
 	* inteproxy/chunkedwriter.py(ChunkedTransferEncodingWriter): New.
 	  Added class to write HTTP chunked transfer encoding. Useful
 	  if the input is given as short byte arrays to be aggregated

Modified: branches/streaming/inteproxy/proxycore.py
===================================================================
--- branches/streaming/inteproxy/proxycore.py	2011-12-23 16:43:28 UTC (rev 323)
+++ branches/streaming/inteproxy/proxycore.py	2011-12-23 17:16:09 UTC (rev 324)
@@ -30,6 +30,7 @@
 from inteproxy.httpmessage import HTTPRequestMessage, HTTPResponseMessage
 from inteproxy.httpconnection import connect_tcp, connect_http_connect, \
      connect_ssl, SocketHTTPConnection, parse_netloc
+from inteproxy.chunkedwriter import ChunkedTransferEncodingWriter
 
 
 # same as the BaseHTTPRequestHandler method, but as a standalone function:
@@ -318,17 +319,7 @@
         self.transfer_data(response.read, self.wfile.write,
                            chunked = (transfer_encoding == "chunked"))
 
-    def transfer_data(self, read, write, length=None, chunked=False):
-        """Transfer data from one 'file' to another in chunks
-
-        The files are given by their read and write methods so it
-        doesn't have to be a file.  The read parameter must be callable
-        with an integer argument indicating the maximum number of bytes
-        to read and the write parameter must be callable with a string.
-        If the parameter chunked is true, the method uses the 'chunked'
-        transfer encoding when writing the data.
-        """
-
+    def wrap_read_write_debug(read, write):
         # wrap the read/write functions if debug logging is active so
         # that the data read from the server and written to the client
         # is logged.
@@ -352,6 +343,59 @@
                 self.log_debug("to client: %r", limit_length(data))
                 orig_write(data)
 
+        return read, write
+
+    def transfer_chunked_rewrite(self, rewrite, read, write,
+                                 separator='>', length=4096):
+        """ TODO: Document me! """
+
+        read, write = self.wrap_read_write_debug(read, write)
+        writer = ChunkedTransferEncodingWriter(write, length)
+        write = writer.append
+
+        data = []
+        append = data.append
+
+        while True:
+            chunk = read(length)
+            if not chunk:
+                break
+
+            pos = 0
+            while True:
+                idx = chunk.find(separator, pos)
+
+            if idx > 0:
+                rest = chunk[pos:idx]
+                append(rest)
+                rewritten = rewrite(''.join(data))
+                del data[:]
+                append(separator)
+                write(rewritten)
+                rewritten = None
+                pos = idx+1
+            else:
+                append(chunk[pos:] if pos else chunk)
+                break
+
+        rewritten = rewrite(''.join(data))
+        write(rewritten)
+        rewritten = None
+        writer.finish()
+
+    def transfer_data(self, read, write, length=None, chunked=False):
+        """Transfer data from one 'file' to another in chunks
+
+        The files are given by their read and write methods so it
+        doesn't have to be a file.  The read parameter must be callable
+        with an integer argument indicating the maximum number of bytes
+        to read and the write parameter must be callable with a string.
+        If the parameter chunked is true, the method uses the 'chunked'
+        transfer encoding when writing the data.
+        """
+
+        read, write = self.wrap_read_write_debug(read, write)
+
         # Now transfer the data in blocks of max_chunk_size
         max_chunk_size = 4096
         while 1: