implement gzip decompression in vtquery

bench/vtquery.bench.js

@@ -3,7 +3,7 @@
 const argv = require('minimist')(process.argv.slice(2));
 if (!argv.iterations || !argv.concurrency) {
   console.error('Please provide desired iterations, concurrency');
-  console.error('Example: \nnode bench/vtquery.bench.js --iterations 50 --concurrency 10');
+  console.error('Example: \nnode bench/vtquery.bench.js --gzipjs --iterations 50 --concurrency 10');
   process.exit(1);
 }
 
@@ -19,6 +19,7 @@ const assert = require('assert');
 const Queue = require('d3-queue').queue;
 const vtquery = require('../lib/index.js');
 const rules = require('./rules');
+const zlib = require('zlib');
 let ruleCount = 1;
 
 // run each rule synchronously
@@ -32,21 +33,51 @@ ruleQueue.awaitAll(function(err, res) {
   process.stdout.write('\n');
 });
 
+function decompress(tile,cb) {
+   zlib.gunzip(tile.buffer,function(err, buf) {
+     tile.buffer = buf;
+     return cb(err);
+   });
+}
+
+
 function runRule(rule, ruleCallback) {
 
   process.stdout.write(`\n${ruleCount}: ${rule.description} ... `);
 
   let runs = 0;
   let runsQueue = Queue();
+  // compress tiles such that the
+  // benchmark has to decompress on the fly
+  rule.tiles.forEach(function(t) {
+    t.buffer = zlib.gzipSync(t.buffer);
+  });
 
   function run(cb) {
-    vtquery(rule.tiles, rule.queryPoint, rule.options, function(err, result) {
-      if (err) {
-        return cb(err);
-      }
-      ++runs;
-      return cb();
-    });
+    if (argv.gzipjs) {
+      const gzipQueue = Queue();
+      rule.tiles.forEach(function(t) {
+        gzipQueue.defer(decompress,t);
+      });
+      gzipQueue.awaitAll(function(error) {
+        if (error) return cb(error);
+        vtquery(rule.tiles, rule.queryPoint, rule.options, function(err, result) {
+          if (err) {
+            return cb(err);
+          }
+          ++runs;
+          return cb();
+        });
+      });
+    } else {
+      vtquery(rule.tiles, rule.queryPoint, rule.options, function(err, result) {
+        if (err) {
+          return cb(err);
+        }
+        ++runs;
+        return cb();
+      });
+    }
   }
 
   // Start monitoring time before async work begins within the defer iterator below.

binding.gyp

@@ -63,6 +63,9 @@
         './src/module.cpp',
         './src/vtquery.cpp'
       ],
+      "link_settings": {
+          "libraries": [ "-lz" ]
+      },
       'ldflags': [
         '-Wl,-z,now',
       ],

scripts/install_deps.sh

@@ -20,3 +20,4 @@ install spatial-algorithms 0.1.0
 install boost 1.65.1
 install cheap-ruler 2.5.3
 install vector-tile f4728da
+install gzip bb80aac

scripts/setup.sh

@@ -4,7 +4,7 @@ set -eu
 set -o pipefail
 
 export MASON_RELEASE="${MASON_RELEASE:-7c4f95d}"
-export MASON_LLVM_RELEASE="${MASON_LLVM_RELEASE:-5.0.0}"
+export MASON_LLVM_RELEASE="${MASON_LLVM_RELEASE:-5.0.1}"
 
 PLATFORM=$(uname | tr A-Z a-z)
 if [[ ${PLATFORM} == 'darwin' ]]; then
@@ -89,8 +89,8 @@ function run() {
     echo "export MSAN_SYMBOLIZER_PATH=${llvm_toolchain_dir}/bin/llvm-symbolizer" >> ${config}
     echo "export UBSAN_OPTIONS=print_stacktrace=1" >> ${config}
     echo "export LSAN_OPTIONS=suppressions=${SUPPRESSION_FILE}" >> ${config}
-    echo "export ASAN_OPTIONS=symbolize=1:abort_on_error=1:detect_container_overflow=1:check_initialization_order=1:detect_stack_use_after_return=1" >> ${config}
-    echo 'export MASON_SANITIZE="-fsanitize=address,undefined,integer -fno-sanitize=vptr,function"' >> ${config}
+    echo "export ASAN_OPTIONS=detect_leaks=1:symbolize=1:abort_on_error=1:detect_container_overflow=1:check_initialization_order=1:detect_stack_use_after_return=1" >> ${config}
+    echo 'export MASON_SANITIZE="-fsanitize=address,undefined,integer,leak -fno-sanitize=vptr,function"' >> ${config}
     echo 'export MASON_SANITIZE_CXXFLAGS="${MASON_SANITIZE} -fno-sanitize=vptr,function -fsanitize-address-use-after-scope -fno-omit-frame-pointer -fno-common"' >> ${config}
     echo 'export MASON_SANITIZE_LDFLAGS="${MASON_SANITIZE}"' >> ${config}
 

src/vtquery.cpp

@@ -3,6 +3,8 @@
 
 #include <algorithm>
 #include <exception>
+#include <gzip/decompress.hpp>
+#include <gzip/utils.hpp>
 #include <iostream>
 #include <map>
 #include <mapbox/geometry/algorithms/closest_point.hpp>
@@ -28,9 +30,12 @@ const char* getGeomTypeString(int enumVal) {
     return GeomTypeStrings[enumVal];
 }
 
+using materialized_prop_type = std::pair<std::string, mapbox::feature::value>;
+
 /// main storage item for returning to the user
 struct ResultObject {
     std::vector<vtzero::property> properties_vector;
+    std::vector<materialized_prop_type> properties_vector_materialized;
     std::string layer_name;
     mapbox::geometry::point<double> coordinates;
     double distance;
@@ -39,6 +44,7 @@ struct ResultObject {
     uint64_t id;
 
     ResultObject() : properties_vector(),
+                     properties_vector_materialized(),
                      layer_name(),
                      coordinates(0.0, 0.0),
                      distance(std::numeric_limits<double>::max()),
@@ -150,11 +156,9 @@ struct property_value_visitor {
 };
 
 /// used to create the final v8 (JSON) object to return to the user
-void set_property(vtzero::property const& property,
+void set_property(materialized_prop_type const& property,
                   v8::Local<v8::Object>& properties_obj) {
-
-    auto val = vtzero::convert_property_value<mapbox::feature::value, mapbox::vector_tile::detail::property_value_mapping>(property.value());
-    mapbox::util::apply_visitor(property_value_visitor{properties_obj, std::string(property.key())}, val);
+    mapbox::util::apply_visitor(property_value_visitor{properties_obj, property.first}, property.second);
 }
 
 GeomType get_geometry_type(vtzero::feature const& f) {
@@ -268,11 +272,25 @@ struct Worker : Nan::AsyncWorker {
             // query point lng/lat geometry.hpp point (used for distance calculation later on)
             mapbox::geometry::point<double> query_lnglat{data.longitude, data.latitude};
 
-            // for each tile
+            gzip::Decompressor decompressor;
+            std::string uncompressed;
+            std::vector<std::string> buffers;
+            std::vector<std::tuple<vtzero::vector_tile, std::uint32_t, std::uint32_t, std::uint32_t>> tiles;
+            tiles.reserve(data.tiles.size());
             for (auto const& tile_ptr : data.tiles) {
                 TileObject const& tile_obj = *tile_ptr;
-                vtzero::vector_tile tile{tile_obj.data};
+                if (gzip::is_compressed(tile_obj.data.data(), tile_obj.data.size())) {
+                    decompressor.decompress(uncompressed, tile_obj.data.data(), tile_obj.data.size());
+                    buffers.emplace_back(std::move(uncompressed));
+                    tiles.emplace_back(vtzero::vector_tile(buffers.back()), tile_obj.z, tile_obj.x, tile_obj.y);
+                } else {
+                    tiles.emplace_back(vtzero::vector_tile(tile_obj.data), tile_obj.z, tile_obj.x, tile_obj.y);
+                }
+            }
 
+            // for each tile
+            for (auto& tile_obj : tiles) {
+                vtzero::vector_tile& tile = std::get<0>(tile_obj);
                 while (auto layer = tile.next_layer()) {
 
                     // check if this is a layer we should query
@@ -282,8 +300,11 @@ struct Worker : Nan::AsyncWorker {
                     }
 
                     std::uint32_t extent = layer.extent();
+                    std::uint32_t tile_obj_z = std::get<1>(tile_obj);
+                    std::uint32_t tile_obj_x = std::get<2>(tile_obj);
+                    std::uint32_t tile_obj_y = std::get<3>(tile_obj);
                     // query point in relation to the current tile the layer extent
-                    mapbox::geometry::point<std::int64_t> query_point = utils::create_query_point(data.longitude, data.latitude, extent, tile_obj.z, tile_obj.x, tile_obj.y);
+                    mapbox::geometry::point<std::int64_t> query_point = utils::create_query_point(data.longitude, data.latitude, extent, tile_obj_z, tile_obj_x, tile_obj_y);
 
                     while (auto feature = layer.next_feature()) {
                         auto original_geometry_type = get_geometry_type(feature);
@@ -306,7 +327,7 @@ struct Worker : Nan::AsyncWorker {
 
                         // if distance from the query point is greater than 0.0 (not a direct hit) so recalculate the latlng
                         if (cp_info.distance > 0.0) {
-                            ll = utils::convert_vt_to_ll(extent, tile_obj.z, tile_obj.x, tile_obj.y, cp_info);
+                            ll = utils::convert_vt_to_ll(extent, tile_obj_z, tile_obj_x, tile_obj_y, cp_info);
                             meters = utils::distance_in_meters(query_lnglat, ll);
                         }
 
@@ -352,15 +373,26 @@ struct Worker : Nan::AsyncWorker {
                     } // end tile.layer.feature loop
                 }     // end tile.layer loop
             }         // end tile loop
+            // Here we create "materialized" properties. We do this because, when reading from a compressed
+            // buffer, it is unsafe to touch `feature.properties_vector` once we've left this loop.
+            // That is because the buffer may represent uncompressed data that is not in scope outside of Execute()
+            for (auto& feature : results_queue_) {
+                feature.properties_vector_materialized.reserve(feature.properties_vector.size());
+                for (auto const& property : feature.properties_vector) {
+                    auto val = vtzero::convert_property_value<mapbox::feature::value, mapbox::vector_tile::detail::property_value_mapping>(property.value());
+                    feature.properties_vector_materialized.emplace_back(std::string(property.key()), std::move(val));
+                }
+                // we are now done with the feature.properties_vector
+                //feature.properties_vector.clear();
+            }
         } catch (const std::exception& e) {
             SetErrorMessage(e.what());
         }
     }
 
     void HandleOKCallback() override {
+        Nan::HandleScope scope;
         try {
-            Nan::HandleScope scope;
-
             v8::Local<v8::Object> results_object = Nan::New<v8::Object>();
             v8::Local<v8::Array> features_array = Nan::New<v8::Array>();
             results_object->Set(Nan::New("type").ToLocalChecked(), Nan::New<v8::String>("FeatureCollection").ToLocalChecked());
@@ -384,7 +416,7 @@ struct Worker : Nan::AsyncWorker {
 
                     // create properties object
                     v8::Local<v8::Object> properties_obj = Nan::New<v8::Object>();
-                    for (auto const& prop : feature.properties_vector) {
+                    for (auto const& prop : feature.properties_vector_materialized) {
                         set_property(prop, properties_obj);
                     }