set up for computing durations while unpacking them

copy dummy cache over

implement retrievePackedPathFromSearchSpace

calculate packed_path_from_source_to_middle

debugging the retrievePackedPathFromSearchSpace function implementation

adding in packed_path_from_source_to_middle

cache is partway working

unpack path and get duration that way

the computeDurationForEdge method

comment out cache

clean up the code

move vector creation and allocation to outside of loop

hack to not return vectors on facade.GetUncompressedForwardDurations and facade.GetUncompressedReverseDurations

clean up hack

add exclude_index to cache key

clearing cache with timestamp

rebase against vectors->range pr

swapped out unordered_map cache with a boost_lru implementation

calculation for cache size

cleaned up comment about cache size calculations

unit tests

cache uses unsigned char for exclude index

clean up cache and unit tests

pass in a hashed key to the threadlocal cache

500 mb threadlocal 2 t

fixes and a rebase

correct calculation

features/testbot/distance_matrix.feature

@@ -23,6 +23,79 @@ Feature: Basic Distance Matrix
             | a | 0  | 10 |
             | b | 10 | 0  |
 
+    Scenario: Testbot - Travel time matrix of minimal network with excludes
+        Given the query options
+            | exclude  | toll        |
+
+        Given the node map
+            """
+            a b
+            c d
+            """
+
+        And the ways
+            | nodes | highway  | toll | #                                        |
+            | ab    | motorway |      | not drivable for exclude=motorway        |
+            | cd    | primary  |      | always drivable                          |
+            | ac    | highway  | yes  | not drivable for exclude=motorway exclude=toll and exclude=motorway,toll |
+            | bd    | highway  | yes  | not drivable for exclude=motorway exclude=toll |
+
+        When I request a travel time matrix I should get
+            |   | a  | b  | c  | d  |
+            | a | 0  | 15 |    |    |
+            | b | 15 | 0  |    |    |
+            | c |    |    | 0  | 10 |
+            | d |    |    | 10 | 0  |
+
+    Scenario: Testbot - Travel time matrix of minimal network with different exclude
+        Given the query options
+            | exclude  | motorway  |
+
+        Given the node map
+            """
+            a b
+            c d
+            """
+
+        And the ways
+            | nodes | highway  | toll | #                                        |
+            | ab    | motorway |      | not drivable for exclude=motorway        |
+            | cd    | primary  |      | always drivable                          |
+            | ac    | highway  | yes  | not drivable for exclude=motorway exclude=toll and exclude=motorway,toll |
+            | bd    | highway  | yes  | not drivable for exclude=motorway exclude=toll |
+
+
+        When I request a travel time matrix I should get
+            |   | a  | b  | c  | d  |
+            | a | 0  | 40 | 15 | 25 |
+            | b | 40 | 0  | 25 | 15 |
+            | c | 15 | 25 | 0  | 10 |
+            | d | 25 | 15 | 10 | 0  |
+
+    Scenario: Testbot - Travel time matrix of minimal network with excludes combination
+        Given the query options
+            | exclude  | motorway,toll  |
+
+        Given the node map
+            """
+            a b
+            c d
+            """
+
+        And the ways
+            | nodes | highway  | toll | #                                        |
+            | ab    | motorway |      | not drivable for exclude=motorway        |
+            | cd    | primary  |      | always drivable                          |
+            | ac    | highway  | yes  | not drivable for exclude=motorway exclude=toll and exclude=motorway,toll |
+            | bd    | highway  | yes  | not drivable for exclude=motorway exclude=toll |
+
+        When I request a travel time matrix I should get
+            |   | a  | b  | c  | d  |
+            | a | 0  | 10 | 0  | 10 |
+            | b | 10 | 0  | 10 | 0  |
+            | c | 0  | 10 | 0  | 10 |
+            | d | 10 | 0  | 10 | 0  |
+
     Scenario: Testbot - Travel time matrix with different way speeds
         Given the node map
             """

include/engine/data_watchdog.hpp

@@ -60,7 +60,8 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
                 DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
                     std::make_shared<datafacade::SharedMemoryAllocator>(
                         std::vector<storage::SharedRegionRegister::ShmKey>{
-                            static_region.shm_key, updatable_region.shm_key}));
+                            static_region.shm_key, updatable_region.shm_key}),
+                    static_region.timestamp);
         }
 
         watcher = std::thread(&DataWatchdogImpl::Run, this);
@@ -115,7 +116,8 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
                 DataFacadeFactory<datafacade::ContiguousInternalMemoryDataFacade, AlgorithmT>(
                     std::make_shared<datafacade::SharedMemoryAllocator>(
                         std::vector<storage::SharedRegionRegister::ShmKey>{
-                            static_region.shm_key, updatable_region.shm_key}));
+                            static_region.shm_key, updatable_region.shm_key}),
+                    static_region.timestamp);
         }
 
         util::Log() << "DataWatchdog thread stopped";

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -133,7 +133,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     using RTreeNode = SharedRTree::TreeNode;
 
     extractor::ClassData exclude_mask;
-    std::string m_timestamp;
     extractor::ProfileProperties *m_profile_properties;
     extractor::Datasources *m_datasources;
 
@@ -169,6 +168,8 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     // allocator that keeps the allocation data
     std::shared_ptr<ContiguousBlockAllocator> allocator;
+    std::size_t m_exclude_index;
+    unsigned m_timestamp;
 
     void InitializeInternalPointers(const storage::SharedDataIndex &index,
                                     const std::string &metric_name,
@@ -184,6 +185,8 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
         m_check_sum = *index.GetBlockPtr<std::uint32_t>("/common/connectivity_checksum");
 
+        m_exclude_index = exclude_index;
+
         std::tie(m_coordinate_list, m_osmnodeid_list) =
             make_nbn_data_view(index, "/common/nbn_data");
 
@@ -218,12 +221,16 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     }
 
   public:
+    std::size_t GetTimestamp() const { return m_timestamp; }
+    std::size_t GetExcludeIndex() const { return m_exclude_index; }
+
     // allows switching between process_memory/shared_memory datafacade, based on the type of
     // allocator
     ContiguousInternalMemoryDataFacadeBase(std::shared_ptr<ContiguousBlockAllocator> allocator_,
                                            const std::string &metric_name,
-                                           const std::size_t exclude_index)
-        : allocator(std::move(allocator_))
+                                           const std::size_t exclude_index,
+                                           unsigned timestamp)
+        : allocator(std::move(allocator_)), m_timestamp(timestamp)
     {
         InitializeInternalPointers(allocator->GetIndex(), metric_name, exclude_index);
     }
@@ -619,10 +626,10 @@ class ContiguousInternalMemoryDataFacade<CH>
   public:
     ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
                                        const std::string &metric_name,
-                                       const std::size_t exclude_index)
-        : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
+                                       const std::size_t exclude_index,
+                                       unsigned timestamp)
+        : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index, timestamp),
           ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, metric_name, exclude_index)
-
     {
     }
 };
@@ -717,10 +724,10 @@ class ContiguousInternalMemoryDataFacade<MLD> final
   public:
     ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
                                        const std::string &metric_name,
-                                       const std::size_t exclude_index)
-        : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
+                                       const std::size_t exclude_index,
+                                       unsigned timestamp)
+        : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index, timestamp),
           ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, metric_name, exclude_index)
-
     {
     }
 };

include/engine/datafacade/datafacade_base.hpp

@@ -33,7 +33,6 @@
 
 #include <boost/range/adaptor/reversed.hpp>
 #include <boost/range/any_range.hpp>
-
 #include <cstddef>
 
 #include <string>

include/engine/datafacade_factory.hpp

@@ -30,8 +30,8 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
     DataFacadeFactory() = default;
 
     template <typename AllocatorT>
-    DataFacadeFactory(std::shared_ptr<AllocatorT> allocator)
-        : DataFacadeFactory(allocator, has_exclude_flags)
+    DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, unsigned timestamp)
+        : DataFacadeFactory(allocator, has_exclude_flags, timestamp)
     {
         BOOST_ASSERT_MSG(facades.size() >= 1, "At least one datafacade is needed");
     }
@@ -44,7 +44,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
   private:
     // Algorithm with exclude flags
     template <typename AllocatorT>
-    DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type)
+    DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type, unsigned timestamp)
     {
         const auto &index = allocator->GetIndex();
         properties = index.template GetBlockPtr<extractor::ProfileProperties>("/common/properties");
@@ -71,7 +71,8 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
             std::size_t index =
                 std::stoi(exclude_prefix.substr(index_begin + 1, exclude_prefix.size()));
             BOOST_ASSERT(index >= 0 && index < facades.size());
-            facades[index] = std::make_shared<const Facade>(allocator, metric_name, index);
+            facades[index] =
+                std::make_shared<const Facade>(allocator, metric_name, index, timestamp);
         }
 
         for (const auto index : util::irange<std::size_t>(0, properties->class_names.size()))
@@ -86,12 +87,12 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
 
     // Algorithm without exclude flags
     template <typename AllocatorT>
-    DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type)
+    DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type, unsigned timestamp)
     {
         const auto &index = allocator->GetIndex();
         properties = index.template GetBlockPtr<extractor::ProfileProperties>("/common/properties");
         const auto &metric_name = properties->GetWeightName();
-        facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0));
+        facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0, timestamp));
     }
 
     std::shared_ptr<const Facade> Get(const api::TileParameters &, std::false_type) const

include/engine/datafacade_provider.hpp

@@ -34,7 +34,8 @@ class ExternalProvider final : public DataFacadeProvider<AlgorithmT, FacadeT>
 
     ExternalProvider(const storage::StorageConfig &config,
                      const boost::filesystem::path &memory_file)
-        : facade_factory(std::make_shared<datafacade::MMapMemoryAllocator>(config, memory_file))
+        : facade_factory(std::make_shared<datafacade::MMapMemoryAllocator>(config, memory_file),
+                         0) // is it ok to add timestamp as zero here?
     {
     }
 
@@ -58,7 +59,7 @@ class ImmutableProvider final : public DataFacadeProvider<AlgorithmT, FacadeT>
     using Facade = typename DataFacadeProvider<AlgorithmT, FacadeT>::Facade;
 
     ImmutableProvider(const storage::StorageConfig &config)
-        : facade_factory(std::make_shared<datafacade::ProcessMemoryAllocator>(config))
+        : facade_factory(std::make_shared<datafacade::ProcessMemoryAllocator>(config), 0)
     {
     }
 

include/engine/geospatial_query.hpp

@@ -13,6 +13,7 @@
 
 #include <algorithm>
 #include <cmath>
+#include <iterator>
 #include <memory>
 #include <vector>
 

include/engine/routing_algorithms/many_to_many.hpp

@@ -21,17 +21,26 @@ namespace
 struct NodeBucket
 {
     NodeID middle_node;
+    NodeID parent_node;
     unsigned column_index; // a column in the weight/duration matrix
     EdgeWeight weight;
     EdgeDuration duration;
 
-    NodeBucket(NodeID middle_node, unsigned column_index, EdgeWeight weight, EdgeDuration duration)
-        : middle_node(middle_node), column_index(column_index), weight(weight), duration(duration)
+    NodeBucket(NodeID middle_node,
+               NodeID parent_node,
+               unsigned column_index,
+               EdgeWeight weight,
+               EdgeDuration duration)
+        : middle_node(middle_node), parent_node(parent_node), column_index(column_index),
+          weight(weight), duration(duration)
     {
     }
 
     // partial order comparison
-    bool operator<(const NodeBucket &rhs) const { return middle_node < rhs.middle_node; }
+    bool operator<(const NodeBucket &rhs) const
+    {
+        return std::tie(middle_node, column_index) < std::tie(rhs.middle_node, rhs.column_index);
+    }
 
     // functor for equal_range
     struct Compare
@@ -46,6 +55,24 @@ struct NodeBucket
             return lhs < rhs.middle_node;
         }
     };
+
+    // functor for equal_range
+    struct ColumnCompare
+    {
+        unsigned column_idx;
+
+        ColumnCompare(unsigned column_idx) : column_idx(column_idx){};
+
+        bool operator()(const NodeBucket &lhs, const NodeID &rhs) const // lowerbound
+        {
+            return std::tie(lhs.middle_node, lhs.column_index) < std::tie(rhs, column_idx);
+        }
+
+        bool operator()(const NodeID &lhs, const NodeBucket &rhs) const // upperbound
+        {
+            return std::tie(lhs, column_idx) < std::tie(rhs.middle_node, rhs.column_index);
+        }
+    };
 };
 }
 

include/engine/routing_algorithms/routing_base.hpp

@@ -177,6 +177,7 @@ void annotatePath(const FacadeT &facade,
         const auto geometry_index = facade.GetGeometryIndex(node_id);
         get_segment_geometry(geometry_index);
 
+
         BOOST_ASSERT(id_vector.size() > 0);
         BOOST_ASSERT(datasource_vector.size() > 0);
         BOOST_ASSERT(weight_vector.size() + 1 == id_vector.size());
@@ -405,6 +406,31 @@ InternalRouteResult extractRoute(const DataFacade<AlgorithmT> &facade,
     return raw_route_data;
 }
 
+template <typename FacadeT>
+EdgeDuration computeEdgeDuration(const FacadeT &facade, NodeID node_id, NodeID turn_id)
+{
+    const auto geometry_index = facade.GetGeometryIndex(node_id);
+
+    // datastructures to hold extracted data from geometry
+    EdgeDuration total_duration;
+
+    if (geometry_index.forward)
+    {
+        auto duration_range = facade.GetUncompressedForwardDurations(geometry_index.id);
+        total_duration = std::accumulate(duration_range.begin(), duration_range.end(), 0);
+    }
+    else
+    {
+        auto duration_range = facade.GetUncompressedReverseDurations(geometry_index.id);
+        total_duration = std::accumulate(duration_range.begin(), duration_range.end(), 0);
+    }
+
+    const auto turn_duration = facade.GetDurationPenaltyForEdgeID(turn_id);
+    total_duration += turn_duration;
+
+    return total_duration;
+}
+
 } // namespace routing_algorithms
 } // namespace engine
 } // namespace osrm