Merge pull request #280 from tilezen/remove-gzip

Remove gzip rawr formatter

tilequeue/command.py

@@ -1537,7 +1537,6 @@ def tilequeue_rawr_process(cfg, peripherals):
     rawr_postgresql_yaml = rawr_yaml.get('postgresql')
     assert rawr_postgresql_yaml, 'Missing rawr postgresql config'
 
-    from raw_tiles.formatter.gzip import Gzip
     from raw_tiles.formatter.msgpack import Msgpack
     from raw_tiles.gen import RawrGenerator
     from raw_tiles.source.conn import ConnectionContextManager
@@ -1596,7 +1595,7 @@ def tilequeue_rawr_process(cfg, peripherals):
 
     logger = make_logger(cfg, 'rawr_process')
     rawr_osm_source = OsmSource(conn_ctx)
-    rawr_formatter = Gzip(Msgpack())
+    rawr_formatter = Msgpack()
     rawr_gen = RawrGenerator(rawr_osm_source, rawr_formatter, rawr_sink)
     stats_handler = RawrTilePipelineStatsHandler(peripherals.stats)
     rawr_proc_logger = JsonRawrProcessingLogger(logger)

tilequeue/rawr.py

@@ -1,6 +1,7 @@
 from botocore.exceptions import ClientError
 from collections import defaultdict
 from collections import namedtuple
+from contextlib import closing
 from cStringIO import StringIO
 from ModestMaps.Core import Coordinate
 from msgpack import Unpacker
@@ -390,25 +391,22 @@ def make_rawr_zip_payload(rawr_tile, date_time=None):
     return buf.getvalue()
 
 
-def unpack_rawr_zip_payload(table_sources, filelike):
+def unpack_rawr_zip_payload(table_sources, payload):
     """unpack a zipfile and turn it into a callable "tables" object."""
     # the io we get from S3 is streaming, so we can't seek on it, but zipfile
     # seems to require that. so we buffer it all in memory. RAWR tiles are
     # generally up to around 100MB in size, which should be safe to store in
     # RAM.
     from tilequeue.query.common import Table
     from io import BytesIO
-    from gzip import GzipFile
 
-    buf = BytesIO(filelike.read())
-    zfh = zipfile.ZipFile(buf, 'r')
+    zfh = zipfile.ZipFile(BytesIO(payload), 'r')
 
     def get_table(table_name):
         # need to extract the whole compressed file from zip reader, as it
         # doesn't support .tell() on the filelike, which gzip requires.
         data = zfh.open(table_name, 'r').read()
-        ungzip = GzipFile(table_name, 'rb', 0, BytesIO(data))
-        unpacker = Unpacker(file_like=ungzip)
+        unpacker = Unpacker(file_like=BytesIO(data))
         source = table_sources[table_name]
         return Table(source, unpacker)
 
@@ -514,7 +512,8 @@ def __call__(self, tile):
         # check that the response isn't a delete marker.
         assert 'DeleteMarker' not in response
 
-        body = response['Body']
+        with closing(response['Body']) as body_fp:
+            body = body_fp.read()
         return unpack_rawr_zip_payload(self.table_sources, body)
 
 
@@ -535,7 +534,7 @@ def _get_object(self, tile):
 
     def __call__(self, tile):
         payload = self._get_object(tile)
-        return unpack_rawr_zip_payload(self.table_sources, StringIO(payload))
+        return unpack_rawr_zip_payload(self.table_sources, payload)
 
 
 def make_rawr_queue(name, region, wait_time_secs):