From db238b717098358d8f493e2d22d013cc58e649d9 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Thu, 22 Mar 2018 19:16:19 +0000
Subject: [PATCH 01/19] Remove fixed block identifiers

---
 .../datafacade/contiguous_block_allocator.hpp |   2 +-
 .../contiguous_internalmem_datafacade.hpp     | 291 +++++++-------
 .../datafacade/process_memory_allocator.hpp   |   4 +-
 .../datafacade/shared_memory_allocator.hpp    |   4 +-
 include/engine/datafacade_factory.hpp         |   2 +-
 include/engine/engine.hpp                     |  26 +-
 include/storage/io.hpp                        |   4 +
 include/storage/serialization.hpp             |  64 ++++
 include/storage/shared_datatype.hpp           | 227 +++--------
 .../datafacade/process_memory_allocator.cpp   |   9 +-
 .../datafacade/shared_memory_allocator.cpp    |  13 +-
 src/osrm/osrm.cpp                             |   2 +-
 src/storage/storage.cpp                       | 357 +++++++-----------
 unit_tests/storage/data_layout.cpp            |  54 +++
 14 files changed, 483 insertions(+), 576 deletions(-)
 create mode 100644 unit_tests/storage/data_layout.cpp

diff --git a/include/engine/datafacade/contiguous_block_allocator.hpp b/include/engine/datafacade/contiguous_block_allocator.hpp
index 0a185202524..cc2dbca5a8d 100644
--- a/include/engine/datafacade/contiguous_block_allocator.hpp
+++ b/include/engine/datafacade/contiguous_block_allocator.hpp
@@ -16,7 +16,7 @@ class ContiguousBlockAllocator
     virtual ~ContiguousBlockAllocator() = default;
 
     // interface to give access to the datafacades
-    virtual storage::DataLayout &GetLayout() = 0;
+    virtual const storage::DataLayout &GetLayout() = 0;
     virtual char *GetMemory() = 0;
 };
 
diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
index c8785661d25..96b424cd09d 100644
--- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
@@ -81,26 +81,25 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
     // allocator that keeps the allocation data
     std::shared_ptr<ContiguousBlockAllocator> allocator;
 
-    void InitializeGraphPointer(storage::DataLayout &data_layout,
+    void InitializeGraphPointer(const storage::DataLayout &data_layout,
                                 char *memory_block,
                                 const std::size_t exclude_index)
     {
-        auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
-            memory_block, storage::DataLayout::CH_GRAPH_NODE_LIST);
+        auto graph_nodes_ptr =
+            data_layout.GetBlockPtr<GraphNode>(memory_block, "/ch/contracted_graph/node_array");
 
-        auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
-            memory_block, storage::DataLayout::CH_GRAPH_EDGE_LIST);
+        auto graph_edges_ptr =
+            data_layout.GetBlockPtr<GraphEdge>(memory_block, "/ch/contracted_graph/edge_array");
 
-        auto filter_block_id = static_cast<storage::DataLayout::BlockID>(
-            storage::DataLayout::CH_EDGE_FILTER_0 + exclude_index);
+        auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index);
 
         auto edge_filter_ptr =
             data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
 
         util::vector_view<GraphNode> node_list(
-            graph_nodes_ptr, data_layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_NODE_LIST));
+            graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array"));
         util::vector_view<GraphEdge> edge_list(
-            graph_edges_ptr, data_layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_EDGE_LIST));
+            graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
 
         util::vector_view<bool> edge_filter(edge_filter_ptr,
                                             data_layout.GetBlockEntries(filter_block_id));
@@ -115,7 +114,7 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
         InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
     }
 
-    void InitializeInternalPointers(storage::DataLayout &data_layout,
+    void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
                                     const std::size_t exclude_index)
     {
@@ -224,29 +223,28 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     // allocator that keeps the allocation data
     std::shared_ptr<ContiguousBlockAllocator> allocator;
 
-    void InitializeProfilePropertiesPointer(storage::DataLayout &data_layout,
+    void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
                                             char *memory_block,
                                             const std::size_t exclude_index)
     {
         m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
-            memory_block, storage::DataLayout::PROPERTIES);
+            memory_block, "/common/properties");
 
         exclude_mask = m_profile_properties->excludable_classes[exclude_index];
     }
 
-    void InitializeChecksumPointer(storage::DataLayout &data_layout, char *memory_block)
+    void InitializeChecksumPointer(const storage::DataLayout &data_layout, char *memory_block)
     {
-        m_check_sum =
-            *data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::HSGR_CHECKSUM);
+        m_check_sum = *data_layout.GetBlockPtr<unsigned>(memory_block, "/ch/checksum");
         util::Log() << "set checksum: " << m_check_sum;
     }
 
-    void InitializeRTreePointers(storage::DataLayout &data_layout, char *memory_block)
+    void InitializeRTreePointers(const storage::DataLayout &data_layout, char *memory_block)
     {
         BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree");
 
         const auto file_index_ptr =
-            data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::FILE_INDEX_PATH);
+            data_layout.GetBlockPtr<char>(memory_block, "/common/rtree/file_index_path");
         file_index_path = boost::filesystem::path(file_index_ptr);
         if (!boost::filesystem::exists(file_index_path))
         {
@@ -256,15 +254,15 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         }
 
         const auto rtree_ptr =
-            data_layout.GetBlockPtr<RTreeNode>(memory_block, storage::DataLayout::R_SEARCH_TREE);
+            data_layout.GetBlockPtr<RTreeNode>(memory_block, "/common/rtree/search_tree");
         util::vector_view<RTreeNode> search_tree(
-            rtree_ptr, data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
+            rtree_ptr, data_layout.GetBlockEntries("/common/rtree/search_tree"));
 
         const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr<std::uint64_t>(
-            memory_block, storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
+            memory_block, "/common/rtree/search_tree_level_starts");
         util::vector_view<std::uint64_t> rtree_level_starts(
             rtree_levelstarts_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
+            data_layout.GetBlockEntries("/common/rtree/search_tree_level_starts"));
 
         m_static_rtree.reset(new SharedRTree{std::move(search_tree),
                                              std::move(rtree_level_starts),
@@ -274,70 +272,67 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
             new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
     }
 
-    void InitializeNodeInformationPointers(storage::DataLayout &layout, char *memory_ptr)
+    void InitializeNodeInformationPointers(const storage::DataLayout &layout, char *memory_ptr)
     {
         const auto coordinate_list_ptr =
-            layout.GetBlockPtr<util::Coordinate>(memory_ptr, storage::DataLayout::COORDINATE_LIST);
-        m_coordinate_list.reset(coordinate_list_ptr,
-                                layout.GetBlockEntries(storage::DataLayout::COORDINATE_LIST));
+            layout.GetBlockPtr<util::Coordinate>(memory_ptr, "/common/coordinates");
+        m_coordinate_list.reset(coordinate_list_ptr, layout.GetBlockEntries("/common/coordinates"));
 
         const auto osmnodeid_ptr = layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type>(
-            memory_ptr, storage::DataLayout::OSM_NODE_ID_LIST);
+            memory_ptr, "/common/osm_node_ids/packed");
         m_osmnodeid_list = extractor::PackedOSMIDsView(
             util::vector_view<extractor::PackedOSMIDsView::block_type>(
-                osmnodeid_ptr, layout.GetBlockEntries(storage::DataLayout::OSM_NODE_ID_LIST)),
+                osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")),
             // We (ab)use the number of coordinates here because we know we have the same amount of
             // ids
-            layout.GetBlockEntries(storage::DataLayout::COORDINATE_LIST));
+            layout.GetBlockEntries("/common/coordinates"));
     }
 
-    void InitializeEdgeBasedNodeDataInformationPointers(storage::DataLayout &layout,
+    void InitializeEdgeBasedNodeDataInformationPointers(const storage::DataLayout &layout,
                                                         char *memory_ptr)
     {
-        const auto edge_based_node_list_ptr = layout.GetBlockPtr<extractor::EdgeBasedNode>(
-            memory_ptr, storage::DataLayout::EDGE_BASED_NODE_DATA_LIST);
+        const auto edge_based_node_list_ptr =
+            layout.GetBlockPtr<extractor::EdgeBasedNode>(memory_ptr, "/common/ebg_node_data/nodes");
         util::vector_view<extractor::EdgeBasedNode> edge_based_node_data_list(
-            edge_based_node_list_ptr,
-            layout.GetBlockEntries(storage::DataLayout::EDGE_BASED_NODE_DATA_LIST));
+            edge_based_node_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes"));
 
         const auto annotation_data_list_ptr =
             layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation>(
-                memory_ptr, storage::DataLayout::ANNOTATION_DATA_LIST);
+                memory_ptr, "/common/ebg_node_data/annotations");
         util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data(
-            annotation_data_list_ptr,
-            layout.GetBlockEntries(storage::DataLayout::ANNOTATION_DATA_LIST));
+            annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations"));
 
         edge_based_node_data = extractor::EdgeBasedNodeDataView(
             std::move(edge_based_node_data_list), std::move(annotation_data));
     }
 
-    void InitializeEdgeInformationPointers(storage::DataLayout &layout, char *memory_ptr)
+    void InitializeEdgeInformationPointers(const storage::DataLayout &layout, char *memory_ptr)
     {
         const auto lane_data_id_ptr =
-            layout.GetBlockPtr<LaneDataID>(memory_ptr, storage::DataLayout::LANE_DATA_ID);
+            layout.GetBlockPtr<LaneDataID>(memory_ptr, "/common/turn_data/lane_data_ids");
         util::vector_view<LaneDataID> lane_data_ids(
-            lane_data_id_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DATA_ID));
+            lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids"));
 
         const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction>(
-            memory_ptr, storage::DataLayout::TURN_INSTRUCTION);
+            memory_ptr, "/common/turn_data/turn_instructions");
         util::vector_view<guidance::TurnInstruction> turn_instructions(
             turn_instruction_list_ptr,
-            layout.GetBlockEntries(storage::DataLayout::TURN_INSTRUCTION));
+            layout.GetBlockEntries("/common/turn_data/turn_instructions"));
 
         const auto entry_class_id_list_ptr =
-            layout.GetBlockPtr<EntryClassID>(memory_ptr, storage::DataLayout::ENTRY_CLASSID);
+            layout.GetBlockPtr<EntryClassID>(memory_ptr, "/common/turn_data/entry_class_ids");
         util::vector_view<EntryClassID> entry_class_ids(
-            entry_class_id_list_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASSID));
+            entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
 
         const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>(
-            memory_ptr, storage::DataLayout::PRE_TURN_BEARING);
+            memory_ptr, "/common/turn_data/pre_turn_bearings");
         util::vector_view<guidance::TurnBearing> pre_turn_bearings(
-            pre_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::PRE_TURN_BEARING));
+            pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings"));
 
         const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>(
-            memory_ptr, storage::DataLayout::POST_TURN_BEARING);
+            memory_ptr, "/common/turn_data/post_turn_bearings");
         util::vector_view<guidance::TurnBearing> post_turn_bearings(
-            post_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::POST_TURN_BEARING));
+            post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings"));
 
         turn_data = guidance::TurnDataView(std::move(turn_instructions),
                                            std::move(lane_data_ids),
@@ -346,124 +341,120 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
                                            std::move(post_turn_bearings));
     }
 
-    void InitializeNamePointers(storage::DataLayout &data_layout, char *memory_block)
+    void InitializeNamePointers(const storage::DataLayout &data_layout, char *memory_block)
     {
         const auto name_blocks_ptr =
             data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>(
-                memory_block, storage::DataLayout::NAME_BLOCKS);
+                memory_block, "/common/names/blocks");
         const auto name_values_ptr =
             data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>(
-                memory_block, storage::DataLayout::NAME_VALUES);
+                memory_block, "/common/names/values");
 
         util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
-            name_blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
+            name_blocks_ptr, data_layout.GetBlockEntries("/common/names/blocks"));
         util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
-            name_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
+            name_values_ptr, data_layout.GetBlockEntries("/common/names/values"));
 
         extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
         m_name_table = extractor::NameTableView{std::move(index_data_view)};
     }
 
-    void InitializeTurnLaneDescriptionsPointers(storage::DataLayout &data_layout,
+    void InitializeTurnLaneDescriptionsPointers(const storage::DataLayout &data_layout,
                                                 char *memory_block)
     {
-        auto offsets_ptr = data_layout.GetBlockPtr<std::uint32_t>(
-            memory_block, storage::DataLayout::LANE_DESCRIPTION_OFFSETS);
+        auto offsets_ptr =
+            data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/turn_lanes/offsets");
         util::vector_view<std::uint32_t> offsets(
-            offsets_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_OFFSETS));
+            offsets_ptr, data_layout.GetBlockEntries("/common/turn_lanes/offsets"));
         m_lane_description_offsets = std::move(offsets);
 
         auto masks_ptr = data_layout.GetBlockPtr<extractor::TurnLaneType::Mask>(
-            memory_block, storage::DataLayout::LANE_DESCRIPTION_MASKS);
+            memory_block, "/common/turn_lanes/masks");
 
         util::vector_view<extractor::TurnLaneType::Mask> masks(
-            masks_ptr, data_layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_MASKS));
+            masks_ptr, data_layout.GetBlockEntries("/common/turn_lanes/masks"));
         m_lane_description_masks = std::move(masks);
 
         const auto lane_tupel_id_pair_ptr =
-            data_layout.GetBlockPtr<util::guidance::LaneTupleIdPair>(
-                memory_block, storage::DataLayout::TURN_LANE_DATA);
+            data_layout.GetBlockPtr<util::guidance::LaneTupleIdPair>(memory_block,
+                                                                     "/common/turn_lanes/data");
         util::vector_view<util::guidance::LaneTupleIdPair> lane_tupel_id_pair(
-            lane_tupel_id_pair_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::TURN_LANE_DATA));
+            lane_tupel_id_pair_ptr, data_layout.GetBlockEntries("/common/turn_lanes/data"));
         m_lane_tupel_id_pairs = std::move(lane_tupel_id_pair);
     }
 
-    void InitializeTurnPenalties(storage::DataLayout &data_layout, char *memory_block)
+    void InitializeTurnPenalties(const storage::DataLayout &data_layout, char *memory_block)
     {
-        auto turn_weight_penalties_ptr = data_layout.GetBlockPtr<TurnPenalty>(
-            memory_block, storage::DataLayout::TURN_WEIGHT_PENALTIES);
+        auto turn_weight_penalties_ptr =
+            data_layout.GetBlockPtr<TurnPenalty>(memory_block, "/common/turn_penalty/weight");
         m_turn_weight_penalties = util::vector_view<TurnPenalty>(
-            turn_weight_penalties_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES));
-        auto turn_duration_penalties_ptr = data_layout.GetBlockPtr<TurnPenalty>(
-            memory_block, storage::DataLayout::TURN_DURATION_PENALTIES);
+            turn_weight_penalties_ptr, data_layout.GetBlockEntries("/common/turn_penalty/weight"));
+        auto turn_duration_penalties_ptr =
+            data_layout.GetBlockPtr<TurnPenalty>(memory_block, "/common/turn_penalty/duration");
         m_turn_duration_penalties = util::vector_view<TurnPenalty>(
             turn_duration_penalties_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::TURN_DURATION_PENALTIES));
+            data_layout.GetBlockEntries("/common/turn_penalty/duration"));
     }
 
-    void InitializeGeometryPointers(storage::DataLayout &data_layout, char *memory_block)
+    void InitializeGeometryPointers(const storage::DataLayout &data_layout, char *memory_block)
     {
         auto geometries_index_ptr =
-            data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::GEOMETRIES_INDEX);
+            data_layout.GetBlockPtr<unsigned>(memory_block, "/common/segment_data/index");
         util::vector_view<unsigned> geometry_begin_indices(
-            geometries_index_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_INDEX));
+            geometries_index_ptr, data_layout.GetBlockEntries("/common/segment_data/index"));
 
-        auto num_entries = data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_NODE_LIST);
-        auto geometries_node_list_ptr = data_layout.GetBlockPtr<NodeID>(
-            memory_block, storage::DataLayout::GEOMETRIES_NODE_LIST);
+        auto num_entries = data_layout.GetBlockEntries("/common/segment_data/nodes");
+        auto geometries_node_list_ptr =
+            data_layout.GetBlockPtr<NodeID>(memory_block, "/common/segment_data/nodes");
         util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
 
         auto geometries_fwd_weight_list_ptr =
             data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                memory_block, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
+                memory_block, "/common/segment_data/forward_weights/packed");
         extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
             util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
                 geometries_fwd_weight_list_ptr,
-                data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST)),
+                data_layout.GetBlockEntries("/common/segment_data/forward_weights/packed")),
             num_entries);
 
         auto geometries_rev_weight_list_ptr =
             data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                memory_block, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
+                memory_block, "/common/segment_data/reverse_weights/packed");
         extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
             util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
                 geometries_rev_weight_list_ptr,
-                data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST)),
+                data_layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")),
             num_entries);
 
         auto geometries_fwd_duration_list_ptr =
             data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                memory_block, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST);
+                memory_block, "/common/segment_data/forward_durations/packed");
         extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
             util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
                 geometries_fwd_duration_list_ptr,
-                data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST)),
+                data_layout.GetBlockEntries("/common/segment_data/forward_durations/packed")),
             num_entries);
 
         auto geometries_rev_duration_list_ptr =
             data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                memory_block, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST);
+                memory_block, "/common/segment_data/reverse_durations/packed");
         extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
             util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
                 geometries_rev_duration_list_ptr,
-                data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DURATION_LIST)),
+                data_layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")),
             num_entries);
 
         auto geometries_fwd_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
-            memory_block, storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST);
+            memory_block, "/common/segment_data/forward_data_sources");
         util::vector_view<DatasourceID> geometry_fwd_datasources_list(
             geometries_fwd_datasources_list_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST));
+            data_layout.GetBlockEntries("/common/segment_data/forward_data_sources"));
 
         auto geometries_rev_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
-            memory_block, storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST);
+            memory_block, "/common/segment_data/reverse_data_sources");
         util::vector_view<DatasourceID> geometry_rev_datasources_list(
             geometries_rev_datasources_list_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST));
+            data_layout.GetBlockEntries("/common/segment_data/reverse_data_sources"));
 
         segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices),
                                                   std::move(geometry_node_list),
@@ -475,30 +466,36 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
                                                   std::move(geometry_rev_datasources_list)};
 
         m_datasources = data_layout.GetBlockPtr<extractor::Datasources>(
-            memory_block, storage::DataLayout::DATASOURCES_NAMES);
+            memory_block, "/common/data_sources_names");
     }
 
-    void InitializeIntersectionClassPointers(storage::DataLayout &data_layout, char *memory_block)
+    void InitializeIntersectionClassPointers(const storage::DataLayout &data_layout,
+                                             char *memory_block)
     {
         auto bearing_class_id_ptr = data_layout.GetBlockPtr<BearingClassID>(
-            memory_block, storage::DataLayout::BEARING_CLASSID);
+            memory_block, "/common/intersection_bearings/node_to_class_id");
         util::vector_view<BearingClassID> bearing_class_id(
             bearing_class_id_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::BEARING_CLASSID));
+            data_layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id"));
 
         auto bearing_values_ptr = data_layout.GetBlockPtr<DiscreteBearing>(
-            memory_block, storage::DataLayout::BEARING_VALUES);
+            memory_block, "/common/intersection_bearings/bearing_values");
         util::vector_view<DiscreteBearing> bearing_values(
-            bearing_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_VALUES));
+            bearing_values_ptr,
+            data_layout.GetBlockEntries("/common/intersection_bearings/bearing_values"));
 
-        auto offsets_ptr =
-            data_layout.GetBlockPtr<unsigned>(memory_block, storage::DataLayout::BEARING_OFFSETS);
-        auto blocks_ptr =
-            data_layout.GetBlockPtr<IndexBlock>(memory_block, storage::DataLayout::BEARING_BLOCKS);
+        auto offsets_ptr = data_layout.GetBlockPtr<unsigned>(
+            memory_block, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
+        auto blocks_ptr = data_layout.GetBlockPtr<IndexBlock>(
+            memory_block, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
         util::vector_view<unsigned> bearing_offsets(
-            offsets_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_OFFSETS));
+            offsets_ptr,
+            data_layout.GetBlockEntries(
+                "/common/intersection_bearings/class_id_to_ranges/block_offsets"));
         util::vector_view<IndexBlock> bearing_blocks(
-            blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::BEARING_BLOCKS));
+            blocks_ptr,
+            data_layout.GetBlockEntries(
+                "/common/intersection_bearings/class_id_to_ranges/diff_blocks"));
 
         util::RangeTable<16, storage::Ownership::View> bearing_range_table(
             bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size()));
@@ -507,28 +504,29 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
             std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
 
         auto entry_class_ptr = data_layout.GetBlockPtr<util::guidance::EntryClass>(
-            memory_block, storage::DataLayout::ENTRY_CLASS);
+            memory_block, "/common/entry_classes");
         util::vector_view<util::guidance::EntryClass> entry_class_table(
-            entry_class_ptr, data_layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASS));
+            entry_class_ptr, data_layout.GetBlockEntries("/common/entry_classes"));
         m_entry_class_table = std::move(entry_class_table);
     }
 
-    void InitializeManeuverOverridePointers(storage::DataLayout &data_layout, char *memory_block)
+    void InitializeManeuverOverridePointers(const storage::DataLayout &data_layout,
+                                            char *memory_block)
     {
         auto maneuver_overrides_ptr = data_layout.GetBlockPtr<extractor::StorageManeuverOverride>(
-            memory_block, storage::DataLayout::MANEUVER_OVERRIDES);
+            memory_block, "/common/maneuver_overrides/overrides");
         m_maneuver_overrides = util::vector_view<extractor::StorageManeuverOverride>(
             maneuver_overrides_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::MANEUVER_OVERRIDES));
+            data_layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
 
         auto maneuver_override_node_sequences_ptr = data_layout.GetBlockPtr<NodeID>(
-            memory_block, storage::DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES);
+            memory_block, "/common/maneuver_overrides/node_sequences");
         m_maneuver_override_node_sequences = util::vector_view<NodeID>(
             maneuver_override_node_sequences_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES));
+            data_layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
     }
 
-    void InitializeInternalPointers(storage::DataLayout &data_layout,
+    void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
                                     const std::size_t exclude_index)
     {
@@ -939,7 +937,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         auto found_range = std::equal_range(
             m_maneuver_overrides.begin(), m_maneuver_overrides.end(), edge_based_node_id, Comp{});
 
-        std::for_each(found_range.first, found_range.second, [&](const auto & override) {
+        std::for_each(found_range.first, found_range.second, [&](const auto &override) {
             std::vector<NodeID> sequence(
                 m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_begin,
                 m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_end);
@@ -981,7 +979,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
 
     QueryGraph query_graph;
 
-    void InitializeInternalPointers(storage::DataLayout &data_layout,
+    void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
                                     const std::size_t exclude_index)
     {
@@ -989,39 +987,38 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
         InitializeGraphPointer(data_layout, memory_block);
     }
 
-    void InitializeMLDDataPointers(storage::DataLayout &data_layout,
+    void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
                                    char *memory_block,
                                    const std::size_t exclude_index)
     {
-        if (data_layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0)
+        if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
         {
-            BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
-            BOOST_ASSERT(data_layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0);
+            BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
+            BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
 
             auto level_data =
                 data_layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
-                    memory_block, storage::DataLayout::MLD_LEVEL_DATA);
+                    memory_block, "/mld/multilevelpartition/level_data");
 
             auto mld_partition_ptr = data_layout.GetBlockPtr<PartitionID>(
-                memory_block, storage::DataLayout::MLD_PARTITION);
+                memory_block, "/mld/multilevelpartition/partition");
             auto partition_entries_count =
-                data_layout.GetBlockEntries(storage::DataLayout::MLD_PARTITION);
+                data_layout.GetBlockEntries("/mld/multilevelpartition/partition");
             util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
 
             auto mld_chilren_ptr = data_layout.GetBlockPtr<CellID>(
-                memory_block, storage::DataLayout::MLD_CELL_TO_CHILDREN);
+                memory_block, "/mld/multilevelpartition/cell_to_children");
             auto children_entries_count =
-                data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_TO_CHILDREN);
+                data_layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
             util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
 
             mld_partition =
                 partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
         }
 
-        const auto weights_block_id = static_cast<storage::DataLayout::BlockID>(
-            storage::DataLayout::MLD_CELL_WEIGHTS_0 + exclude_index);
-        const auto durations_block_id = static_cast<storage::DataLayout::BlockID>(
-            storage::DataLayout::MLD_CELL_DURATIONS_0 + exclude_index);
+        const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights";
+        const auto durations_block_id =
+            "/mld/metrics/" + std::to_string(exclude_index) + "/durations";
 
         if (data_layout.GetBlockSize(weights_block_id) > 0)
         {
@@ -1029,10 +1026,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
                 data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
             auto mld_cell_durations_ptr =
                 data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
-            auto weight_entries_count =
-                data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_WEIGHTS_0);
-            auto duration_entries_count =
-                data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DURATIONS_0);
+            auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights");
+            auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations");
             BOOST_ASSERT(weight_entries_count == duration_entries_count);
             util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
             util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
@@ -1041,25 +1036,25 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
             mld_cell_metric = customizer::CellMetricView{std::move(weights), std::move(durations)};
         }
 
-        if (data_layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0)
+        if (data_layout.GetBlockSize("/mld/cellstorage/cells") > 0)
         {
 
-            auto mld_source_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
-                memory_block, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
+            auto mld_source_boundary_ptr =
+                data_layout.GetBlockPtr<NodeID>(memory_block, "/mld/cellstorage/source_boundary");
             auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
-                memory_block, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
+                memory_block, "/mld/cellstorage/destination_boundary");
             auto mld_cells_ptr = data_layout.GetBlockPtr<partitioner::CellStorageView::CellData>(
-                memory_block, storage::DataLayout::MLD_CELLS);
+                memory_block, "/mld/cellstorage/cells");
             auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>(
-                memory_block, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
+                memory_block, "/mld/cellstorage/level_to_cell_offset");
 
             auto source_boundary_entries_count =
-                data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
+                data_layout.GetBlockEntries("/mld/cellstorage/source_boundary");
             auto destination_boundary_entries_count =
-                data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
-            auto cells_entries_counts = data_layout.GetBlockEntries(storage::DataLayout::MLD_CELLS);
+                data_layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
+            auto cells_entries_counts = data_layout.GetBlockEntries("/mld/cellstorage/cells");
             auto cell_level_offsets_entries_count =
-                data_layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
+                data_layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
 
             util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
                                                       source_boundary_entries_count);
@@ -1076,24 +1071,24 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
                                                             std::move(level_offsets)};
         }
     }
-    void InitializeGraphPointer(storage::DataLayout &data_layout, char *memory_block)
+    void InitializeGraphPointer(const storage::DataLayout &data_layout, char *memory_block)
     {
-        auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
-            memory_block, storage::DataLayout::MLD_GRAPH_NODE_LIST);
+        auto graph_nodes_ptr =
+            data_layout.GetBlockPtr<GraphNode>(memory_block, "/mld/multilevelgraph/node_array");
 
-        auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
-            memory_block, storage::DataLayout::MLD_GRAPH_EDGE_LIST);
+        auto graph_edges_ptr =
+            data_layout.GetBlockPtr<GraphEdge>(memory_block, "/mld/multilevelgraph/edge_array");
 
         auto graph_node_to_offset_ptr = data_layout.GetBlockPtr<QueryGraph::EdgeOffset>(
-            memory_block, storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET);
+            memory_block, "/mld/multilevelgraph/node_to_edge_offset");
 
         util::vector_view<GraphNode> node_list(
-            graph_nodes_ptr, data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_LIST));
+            graph_nodes_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
         util::vector_view<GraphEdge> edge_list(
-            graph_edges_ptr, data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_EDGE_LIST));
+            graph_edges_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
         util::vector_view<QueryGraph::EdgeOffset> node_to_offset(
             graph_node_to_offset_ptr,
-            data_layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET));
+            data_layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
 
         query_graph =
             QueryGraph(std::move(node_list), std::move(edge_list), std::move(node_to_offset));
diff --git a/include/engine/datafacade/process_memory_allocator.hpp b/include/engine/datafacade/process_memory_allocator.hpp
index aa41dd6e4e8..705ab935e1b 100644
--- a/include/engine/datafacade/process_memory_allocator.hpp
+++ b/include/engine/datafacade/process_memory_allocator.hpp
@@ -27,12 +27,12 @@ class ProcessMemoryAllocator : public ContiguousBlockAllocator
     ~ProcessMemoryAllocator() override final;
 
     // interface to give access to the datafacades
-    storage::DataLayout &GetLayout() override final;
+    const storage::DataLayout &GetLayout() override final;
     char *GetMemory() override final;
 
   private:
+    storage::DataLayout internal_layout;
     std::unique_ptr<char[]> internal_memory;
-    std::unique_ptr<storage::DataLayout> internal_layout;
 };
 
 } // namespace datafacade
diff --git a/include/engine/datafacade/shared_memory_allocator.hpp b/include/engine/datafacade/shared_memory_allocator.hpp
index f69c6f09edf..e201e4ee483 100644
--- a/include/engine/datafacade/shared_memory_allocator.hpp
+++ b/include/engine/datafacade/shared_memory_allocator.hpp
@@ -27,10 +27,12 @@ class SharedMemoryAllocator : public ContiguousBlockAllocator
     ~SharedMemoryAllocator() override final;
 
     // interface to give access to the datafacades
-    storage::DataLayout &GetLayout() override final;
+    const storage::DataLayout &GetLayout() override final;
     char *GetMemory() override final;
 
   private:
+    std::size_t layout_size;
+    storage::DataLayout data_layout;
     std::unique_ptr<storage::SharedMemory> m_large_memory;
 };
 
diff --git a/include/engine/datafacade_factory.hpp b/include/engine/datafacade_factory.hpp
index 92efcf7ee9e..85de49bf5c0 100644
--- a/include/engine/datafacade_factory.hpp
+++ b/include/engine/datafacade_factory.hpp
@@ -51,7 +51,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
         }
 
         properties = allocator->GetLayout().template GetBlockPtr<extractor::ProfileProperties>(
-            allocator->GetMemory(), storage::DataLayout::PROPERTIES);
+            allocator->GetMemory(), "/common/properties");
 
         for (const auto index : util::irange<std::size_t>(0, properties->class_names.size()))
         {
diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp
index 6e49138e209..79df2f922f1 100644
--- a/include/engine/engine.hpp
+++ b/include/engine/engine.hpp
@@ -152,8 +152,8 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
         auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
-        return layout->GetBlockSize(storage::DataLayout::CH_GRAPH_NODE_LIST) > 4 &&
-               layout->GetBlockSize(storage::DataLayout::CH_GRAPH_EDGE_LIST) > 4;
+        return layout->GetBlockSize("/ch/contracted_graph/node_array") > 4 &&
+               layout->GetBlockSize("/ch/contracted_graph/edge_array") > 4;
     }
     else
     {
@@ -173,19 +173,19 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
         auto mem = storage::makeSharedMemory(barrier.data().region);
         auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
         // checks that all the needed memory blocks are populated
-        // DataLayout::MLD_CELL_SOURCE_BOUNDARY and DataLayout::MLD_CELL_DESTINATION_BOUNDARY
+        // "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
         // are not checked, because in situations where there are so few nodes in the graph that
         // they all fit into one cell, they can be empty.
-        bool empty_data = layout->GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_CELLS) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_NODE_LIST) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_EDGE_LIST) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_CELL_WEIGHTS_0) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_CELL_DURATIONS_0) > 0 &&
-                          layout->GetBlockSize(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET) > 0;
+        bool empty_data = layout->GetBlockSize("/mld/multilevelpartition/level_data") > 0 &&
+                          layout->GetBlockSize("/mld/multilevelpartition/partition") > 0 &&
+                          layout->GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0 &&
+                          layout->GetBlockSize("/mld/cellstorage/cells") > 0 &&
+                          layout->GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0 &&
+                          layout->GetBlockSize("/mld/multilevelgraph/node_array") > 0 &&
+                          layout->GetBlockSize("/mld/multilevelgraph/edge_array") > 0 &&
+                          layout->GetBlockSize("/mld/metrics/0/weights") > 0 &&
+                          layout->GetBlockSize("/mld/metrics/0/durations") > 0 &&
+                          layout->GetBlockSize("/mld/multilevelgraph/node_to_edge_offset") > 0;
         return empty_data;
     }
     else
diff --git a/include/storage/io.hpp b/include/storage/io.hpp
index eadf93e9bb4..159c149b708 100644
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -303,6 +303,10 @@ class BufferReader
         }
     }
 
+    std::size_t GetPosition() {
+        return input_stream.tellg();
+    }
+
     template <typename T> void ReadInto(T *dest, const std::size_t count)
     {
 #if !defined(__GNUC__) || (__GNUC__ > 4)
diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp
index 35a4d1a7759..97cd2d2e5c8 100644
--- a/include/storage/serialization.hpp
+++ b/include/storage/serialization.hpp
@@ -7,12 +7,14 @@
 
 #include "storage/io.hpp"
 #include "storage/tar.hpp"
+#include "storage/shared_datatype.hpp"
 
 #include <boost/function_output_iterator.hpp>
 #include <boost/iterator/function_input_iterator.hpp>
 
 #include <cmath>
 #include <cstdint>
+#include <tuple>
 
 #if USE_STXXL_LIBRARY
 #include <stxxl/vector>
@@ -81,6 +83,32 @@ template <typename T> void write(io::BufferWriter &writer, const std::vector<T>
     writer.WriteFrom(data.data(), count);
 }
 
+template<typename T>
+inline void write(io::BufferWriter &writer, const T &data)
+{
+    writer.WriteFrom(data);
+}
+
+template<typename T>
+inline void read(io::BufferReader &reader, T &data)
+{
+    reader.ReadInto(data);
+}
+
+inline void write(io::BufferWriter &writer, const std::string &data)
+{
+    const auto count = data.size();
+    writer.WriteElementCount64(count);
+    writer.WriteFrom(data.data(), count);
+}
+
+inline void read(io::BufferReader &reader, std::string &data)
+{
+    const auto count = reader.ReadElementCount64();
+    data.resize(count);
+    reader.ReadInto(const_cast<char *>(data.data()), count);
+}
+
 inline void write(tar::FileWriter &writer, const std::string &name, const std::string &data)
 {
     const auto count = data.size();
@@ -218,6 +246,42 @@ write<bool>(tar::FileWriter &writer, const std::string &name, const std::vector<
 {
     detail::writeBoolVector(writer, name, data);
 }
+
+template <typename K, typename V> void read(io::BufferReader &reader, std::map<K, V> &data)
+{
+    const auto count = reader.ReadElementCount64();
+    std::pair<K, V> pair;
+    for (auto index : util::irange<std::size_t>(0, count))
+    {
+        (void) index;
+        read(reader, pair.first);
+        read(reader, pair.second);
+        data.insert(pair);
+    }
+}
+
+template <typename K, typename V> void write(io::BufferWriter &writer, const std::map<K, V> &data)
+{
+    const auto count = data.size();
+    writer.WriteElementCount64(count);
+    for (const auto &pair : data)
+    {
+        write(writer, pair.first);
+        write(writer, pair.second);
+    }
+}
+
+inline void read(io::BufferReader &reader, DataLayout &layout)
+{
+    read(reader, layout.blocks);
+}
+
+inline void write(io::BufferWriter &writer, const DataLayout &layout)
+{
+    write(writer, layout.blocks);
+}
+
+
 }
 }
 }
diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index 56e6c99ba65..7d72ad912a0 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -2,6 +2,7 @@
 #define SHARED_DATA_TYPE_HPP
 
 #include "storage/block.hpp"
+#include "storage/io_fwd.hpp"
 
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
@@ -11,219 +12,91 @@
 
 #include <array>
 #include <cstdint>
+#include <map>
 
 namespace osrm
 {
 namespace storage
 {
 
+class DataLayout;
+namespace serialization
+{
+inline void read(io::BufferReader &reader, DataLayout &layout);
+
+inline void write(io::BufferWriter &writer, const DataLayout &layout);
+}
+
 // Added at the start and end of each block as sanity check
 const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
 
-const constexpr char *block_id_to_name[] = {"IGNORE_BLOCK",
-                                            "NAME_BLOCKS",
-                                            "NAME_VALUES",
-                                            "EDGE_BASED_NODE_DATA",
-                                            "ANNOTATION_DATA",
-                                            "CH_GRAPH_NODE_LIST",
-                                            "CH_GRAPH_EDGE_LIST",
-                                            "CH_EDGE_FILTER_0",
-                                            "CH_EDGE_FILTER_1",
-                                            "CH_EDGE_FILTER_2",
-                                            "CH_EDGE_FILTER_3",
-                                            "CH_EDGE_FILTER_4",
-                                            "CH_EDGE_FILTER_5",
-                                            "CH_EDGE_FILTER_6",
-                                            "CH_EDGE_FILTER_7",
-                                            "COORDINATE_LIST",
-                                            "OSM_NODE_ID_LIST",
-                                            "TURN_INSTRUCTION",
-                                            "ENTRY_CLASSID",
-                                            "R_SEARCH_TREE",
-                                            "R_SEARCH_TREE_LEVEL_STARTS",
-                                            "GEOMETRIES_INDEX",
-                                            "GEOMETRIES_NODE_LIST",
-                                            "GEOMETRIES_FWD_WEIGHT_LIST",
-                                            "GEOMETRIES_REV_WEIGHT_LIST",
-                                            "GEOMETRIES_FWD_DURATION_LIST",
-                                            "GEOMETRIES_REV_DURATION_LIST",
-                                            "GEOMETRIES_FWD_DATASOURCES_LIST",
-                                            "GEOMETRIES_REV_DATASOURCES_LIST",
-                                            "HSGR_CHECKSUM",
-                                            "FILE_INDEX_PATH",
-                                            "DATASOURCES_NAMES",
-                                            "PROPERTIES",
-                                            "BEARING_CLASSID",
-                                            "BEARING_OFFSETS",
-                                            "BEARING_BLOCKS",
-                                            "BEARING_VALUES",
-                                            "ENTRY_CLASS",
-                                            "LANE_DATA_ID",
-                                            "PRE_TURN_BEARING",
-                                            "POST_TURN_BEARING",
-                                            "TURN_LANE_DATA",
-                                            "LANE_DESCRIPTION_OFFSETS",
-                                            "LANE_DESCRIPTION_MASKS",
-                                            "TURN_WEIGHT_PENALTIES",
-                                            "TURN_DURATION_PENALTIES",
-                                            "MLD_LEVEL_DATA",
-                                            "MLD_PARTITION",
-                                            "MLD_CELL_TO_CHILDREN",
-                                            "MLD_CELL_WEIGHTS_0",
-                                            "MLD_CELL_WEIGHTS_1",
-                                            "MLD_CELL_WEIGHTS_2",
-                                            "MLD_CELL_WEIGHTS_3",
-                                            "MLD_CELL_WEIGHTS_4",
-                                            "MLD_CELL_WEIGHTS_5",
-                                            "MLD_CELL_WEIGHTS_6",
-                                            "MLD_CELL_WEIGHTS_7",
-                                            "MLD_CELL_DURATIONS_0",
-                                            "MLD_CELL_DURATIONS_1",
-                                            "MLD_CELL_DURATIONS_2",
-                                            "MLD_CELL_DURATIONS_3",
-                                            "MLD_CELL_DURATIONS_4",
-                                            "MLD_CELL_DURATIONS_5",
-                                            "MLD_CELL_DURATIONS_6",
-                                            "MLD_CELL_DURATIONS_7",
-                                            "MLD_CELL_SOURCE_BOUNDARY",
-                                            "MLD_CELL_DESTINATION_BOUNDARY",
-                                            "MLD_CELLS",
-                                            "MLD_CELL_LEVEL_OFFSETS",
-                                            "MLD_GRAPH_NODE_LIST",
-                                            "MLD_GRAPH_EDGE_LIST",
-                                            "MLD_GRAPH_NODE_TO_OFFSET",
-                                            "MANEUVER_OVERRIDES",
-                                            "MANEUVER_OVERRIDE_NODE_SEQUENCES"};
-
 class DataLayout
 {
   public:
-    enum BlockID
-    {
-        IGNORE_BLOCK = 0,
-        NAME_BLOCKS,
-        NAME_VALUES,
-        EDGE_BASED_NODE_DATA_LIST,
-        ANNOTATION_DATA_LIST,
-        CH_GRAPH_NODE_LIST,
-        CH_GRAPH_EDGE_LIST,
-        CH_EDGE_FILTER_0,
-        CH_EDGE_FILTER_1,
-        CH_EDGE_FILTER_2,
-        CH_EDGE_FILTER_3,
-        CH_EDGE_FILTER_4,
-        CH_EDGE_FILTER_5,
-        CH_EDGE_FILTER_6,
-        CH_EDGE_FILTER_7,
-        COORDINATE_LIST,
-        OSM_NODE_ID_LIST,
-        TURN_INSTRUCTION,
-        ENTRY_CLASSID,
-        R_SEARCH_TREE,
-        R_SEARCH_TREE_LEVEL_STARTS,
-        GEOMETRIES_INDEX,
-        GEOMETRIES_NODE_LIST,
-        GEOMETRIES_FWD_WEIGHT_LIST,
-        GEOMETRIES_REV_WEIGHT_LIST,
-        GEOMETRIES_FWD_DURATION_LIST,
-        GEOMETRIES_REV_DURATION_LIST,
-        GEOMETRIES_FWD_DATASOURCES_LIST,
-        GEOMETRIES_REV_DATASOURCES_LIST,
-        HSGR_CHECKSUM,
-        FILE_INDEX_PATH,
-        DATASOURCES_NAMES,
-        PROPERTIES,
-        BEARING_CLASSID,
-        BEARING_OFFSETS,
-        BEARING_BLOCKS,
-        BEARING_VALUES,
-        ENTRY_CLASS,
-        LANE_DATA_ID,
-        PRE_TURN_BEARING,
-        POST_TURN_BEARING,
-        TURN_LANE_DATA,
-        LANE_DESCRIPTION_OFFSETS,
-        LANE_DESCRIPTION_MASKS,
-        TURN_WEIGHT_PENALTIES,
-        TURN_DURATION_PENALTIES,
-        MLD_LEVEL_DATA,
-        MLD_PARTITION,
-        MLD_CELL_TO_CHILDREN,
-        MLD_CELL_WEIGHTS_0,
-        MLD_CELL_WEIGHTS_1,
-        MLD_CELL_WEIGHTS_2,
-        MLD_CELL_WEIGHTS_3,
-        MLD_CELL_WEIGHTS_4,
-        MLD_CELL_WEIGHTS_5,
-        MLD_CELL_WEIGHTS_6,
-        MLD_CELL_WEIGHTS_7,
-        MLD_CELL_DURATIONS_0,
-        MLD_CELL_DURATIONS_1,
-        MLD_CELL_DURATIONS_2,
-        MLD_CELL_DURATIONS_3,
-        MLD_CELL_DURATIONS_4,
-        MLD_CELL_DURATIONS_5,
-        MLD_CELL_DURATIONS_6,
-        MLD_CELL_DURATIONS_7,
-        MLD_CELL_SOURCE_BOUNDARY,
-        MLD_CELL_DESTINATION_BOUNDARY,
-        MLD_CELLS,
-        MLD_CELL_LEVEL_OFFSETS,
-        MLD_GRAPH_NODE_LIST,
-        MLD_GRAPH_EDGE_LIST,
-        MLD_GRAPH_NODE_TO_OFFSET,
-        MANEUVER_OVERRIDES,
-        MANEUVER_OVERRIDE_NODE_SEQUENCES,
-        NUM_BLOCKS
-    };
-
     DataLayout() : blocks{} {}
 
-    inline void SetBlock(BlockID bid, Block block) { blocks[bid] = std::move(block); }
+    inline void SetBlock(const std::string &name, Block block) { blocks[name] = std::move(block); }
 
-    inline uint64_t GetBlockEntries(BlockID bid) const { return blocks[bid].num_entries; }
+    inline uint64_t GetBlockEntries(const std::string &name) const
+    {
+        auto iter = blocks.find(name);
+        if (iter == blocks.end())
+        {
+            throw util::exception("Could not find block " + name);
+        }
 
-    inline uint64_t GetBlockSize(BlockID bid) const { return blocks[bid].byte_size; }
+        return iter->second.num_entries;
+    }
+
+    inline uint64_t GetBlockSize(const std::string &name) const
+    {
+        auto iter = blocks.find(name);
+        if (iter == blocks.end())
+        {
+            throw util::exception("Could not find block " + name);
+        }
+
+        return iter->second.byte_size;
+    }
 
     inline uint64_t GetSizeOfLayout() const
     {
         uint64_t result = 0;
-        for (auto i = 0; i < NUM_BLOCKS; i++)
+        for (const auto &name_and_block : blocks)
         {
-            result += 2 * sizeof(CANARY) + GetBlockSize(static_cast<BlockID>(i)) + BLOCK_ALIGNMENT;
+            result += 2 * sizeof(CANARY) + GetBlockSize(name_and_block.first) + BLOCK_ALIGNMENT;
         }
         return result;
     }
 
     template <typename T, bool WRITE_CANARY = false>
-    inline T *GetBlockPtr(char *shared_memory, BlockID bid) const
+    inline T *GetBlockPtr(char *shared_memory, const std::string &name) const
     {
         static_assert(BLOCK_ALIGNMENT % std::alignment_of<T>::value == 0,
                       "Datatype does not fit alignment constraints.");
 
-        char *ptr = (char *)GetAlignedBlockPtr(shared_memory, bid);
+        char *ptr = (char *)GetAlignedBlockPtr(shared_memory, name);
         if (WRITE_CANARY)
         {
             char *start_canary_ptr = ptr - sizeof(CANARY);
-            char *end_canary_ptr = ptr + GetBlockSize(bid);
+            char *end_canary_ptr = ptr + GetBlockSize(name);
             std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
             std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
         }
         else
         {
             char *start_canary_ptr = ptr - sizeof(CANARY);
-            char *end_canary_ptr = ptr + GetBlockSize(bid);
+            char *end_canary_ptr = ptr + GetBlockSize(name);
             bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
             bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
             if (!start_canary_alive)
             {
-                throw util::exception("Start canary of block corrupted. (" +
-                                      std::string(block_id_to_name[bid]) + ")" + SOURCE_REF);
+                throw util::exception("Start canary of block corrupted. (" + name + ")" +
+                                      SOURCE_REF);
             }
             if (!end_canary_alive)
             {
-                throw util::exception("End canary of block corrupted. (" +
-                                      std::string(block_id_to_name[bid]) + ")" + SOURCE_REF);
+                throw util::exception("End canary of block corrupted. (" + name + ")" + SOURCE_REF);
             }
         }
 
@@ -231,6 +104,9 @@ class DataLayout
     }
 
   private:
+    friend void serialization::read(io::BufferReader &reader, DataLayout &layout);
+    friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout);
+
     // Fit aligned storage in buffer to 64 bytes to conform with AVX 512 types
     inline void *align(void *&ptr) const noexcept
     {
@@ -239,13 +115,13 @@ class DataLayout
         return ptr = reinterpret_cast<void *>(aligned);
     }
 
-    inline void *GetAlignedBlockPtr(void *ptr, BlockID bid) const
+    inline void *GetAlignedBlockPtr(void *ptr, const std::string &name) const
     {
-        for (auto i = 0; i < bid; i++)
+        for (auto iter = blocks.begin(); iter != blocks.end() && iter->first != name; ++iter)
         {
             ptr = static_cast<char *>(ptr) + sizeof(CANARY);
             ptr = align(ptr);
-            ptr = static_cast<char *>(ptr) + GetBlockSize((BlockID)i);
+            ptr = static_cast<char *>(ptr) + GetBlockSize(name);
             ptr = static_cast<char *>(ptr) + sizeof(CANARY);
         }
 
@@ -254,14 +130,14 @@ class DataLayout
         return ptr;
     }
 
-    template <typename T> inline T *GetBlockEnd(char *shared_memory, BlockID bid) const
+    template <typename T> inline T *GetBlockEnd(char *shared_memory, const std::string &name) const
     {
-        auto begin = GetBlockPtr<T>(shared_memory, bid);
-        return begin + GetBlockEntries(bid);
+        auto begin = GetBlockPtr<T>(shared_memory, name);
+        return begin + GetBlockEntries(name);
     }
 
     static constexpr std::size_t BLOCK_ALIGNMENT = 64;
-    std::array<Block, NUM_BLOCKS> blocks;
+    std::map<std::string, Block> blocks;
 };
 
 enum SharedDataType
@@ -298,9 +174,6 @@ inline std::string regionToString(const SharedDataType region)
         return "INVALID_REGION";
     }
 }
-
-static_assert(sizeof(block_id_to_name) / sizeof(*block_id_to_name) == DataLayout::NUM_BLOCKS,
-              "Number of blocks needs to match the number of Block names.");
 }
 }
 
diff --git a/src/engine/datafacade/process_memory_allocator.cpp b/src/engine/datafacade/process_memory_allocator.cpp
index fcb88bad520..4c39c5877d1 100644
--- a/src/engine/datafacade/process_memory_allocator.cpp
+++ b/src/engine/datafacade/process_memory_allocator.cpp
@@ -15,17 +15,16 @@ ProcessMemoryAllocator::ProcessMemoryAllocator(const storage::StorageConfig &con
     storage::Storage storage(config);
 
     // Calculate the layout/size of the memory block
-    internal_layout = std::make_unique<storage::DataLayout>();
-    storage.PopulateLayout(*internal_layout);
+    storage.PopulateLayout(internal_layout);
 
     // Allocate the memory block, then load data from files into it
-    internal_memory = std::make_unique<char[]>(internal_layout->GetSizeOfLayout());
-    storage.PopulateData(*internal_layout, internal_memory.get());
+    internal_memory = std::make_unique<char[]>(internal_layout.GetSizeOfLayout());
+    storage.PopulateData(internal_layout, internal_memory.get());
 }
 
 ProcessMemoryAllocator::~ProcessMemoryAllocator() {}
 
-storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return *internal_layout.get(); }
+const storage::DataLayout &ProcessMemoryAllocator::GetLayout() { return internal_layout; }
 char *ProcessMemoryAllocator::GetMemory() { return internal_memory.get(); }
 
 } // namespace datafacade
diff --git a/src/engine/datafacade/shared_memory_allocator.cpp b/src/engine/datafacade/shared_memory_allocator.cpp
index 87814f98727..cb31d091e39 100644
--- a/src/engine/datafacade/shared_memory_allocator.cpp
+++ b/src/engine/datafacade/shared_memory_allocator.cpp
@@ -1,4 +1,7 @@
 #include "engine/datafacade/shared_memory_allocator.hpp"
+
+#include "storage/serialization.hpp"
+
 #include "util/log.hpp"
 
 #include "boost/assert.hpp"
@@ -16,17 +19,21 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
 
     BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region));
     m_large_memory = storage::makeSharedMemory(data_region);
+
+    storage::io::BufferReader reader(GetMemory());
+    storage::serialization::read(reader, data_layout);
+    layout_size = reader.GetPosition();
 }
 
 SharedMemoryAllocator::~SharedMemoryAllocator() {}
 
-storage::DataLayout &SharedMemoryAllocator::GetLayout()
+const storage::DataLayout &SharedMemoryAllocator::GetLayout()
 {
-    return *reinterpret_cast<storage::DataLayout *>(m_large_memory->Ptr());
+    return data_layout;
 }
 char *SharedMemoryAllocator::GetMemory()
 {
-    return reinterpret_cast<char *>(m_large_memory->Ptr()) + sizeof(storage::DataLayout);
+    return reinterpret_cast<char *>(m_large_memory->Ptr()) + layout_size;
 }
 
 } // namespace datafacade
diff --git a/src/osrm/osrm.cpp b/src/osrm/osrm.cpp
index 26f4187b32f..c672c77d50d 100644
--- a/src/osrm/osrm.cpp
+++ b/src/osrm/osrm.cpp
@@ -35,7 +35,7 @@ OSRM::OSRM(engine::EngineConfig &config)
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
         auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
-        if (layout->GetBlockSize(storage::DataLayout::NAME_VALUES) == 0)
+        if (layout->GetBlockSize("/common/names/values") == 0)
             throw util::exception(
                 "No name data loaded, cannot continue.  Have you run osrm-datastore to load data?");
     }
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 806f966e2fb..051dcc0c7b1 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -70,7 +70,7 @@ namespace storage
 {
 namespace
 {
-template <typename OutIter> void readBlocks(const boost::filesystem::path &path, OutIter out)
+inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
 {
     tar::FileReader reader(path, tar::FileReader::VerifyFingerprint);
 
@@ -83,7 +83,7 @@ template <typename OutIter> void readBlocks(const boost::filesystem::path &path,
         if (name_end == std::string::npos)
         {
             auto number_of_elements = reader.ReadElementCount64(entry.name);
-            *out++ = NamedBlock{entry.name, Block{number_of_elements, entry.size}};
+            layout.SetBlock(entry.name, Block{number_of_elements, entry.size});
         }
     }
 }
@@ -159,15 +159,19 @@ int Storage::Run(int max_wait)
     DataLayout layout;
     PopulateLayout(layout);
 
+    io::BufferWriter writer;
+    serialization::write(writer, layout);
+    auto encoded_layout = writer.GetBuffer();
+
     // Allocate shared memory block
-    auto regions_size = sizeof(layout) + layout.GetSizeOfLayout();
+    auto regions_size = encoded_layout.size() + layout.GetSizeOfLayout();
     util::Log() << "Allocating shared memory of " << regions_size << " bytes";
     auto data_memory = makeSharedMemory(next_region, regions_size);
 
     // Copy memory layout to shared memory and populate data
     char *shared_memory_ptr = static_cast<char *>(data_memory->Ptr());
-    memcpy(shared_memory_ptr, &layout, sizeof(layout));
-    PopulateData(layout, shared_memory_ptr + sizeof(layout));
+    memcpy(shared_memory_ptr, encoded_layout.data(), encoded_layout.size());
+    PopulateData(layout, shared_memory_ptr + encoded_layout.size());
 
     { // Lock for write access shared region mutex
         boost::interprocess::scoped_lock<Monitor::mutex_type> lock(monitor.get_mutex(),
@@ -236,91 +240,10 @@ void Storage::PopulateLayout(DataLayout &layout)
         auto absolute_file_index_path =
             boost::filesystem::absolute(config.GetPath(".osrm.fileIndex"));
 
-        layout.SetBlock(DataLayout::FILE_INDEX_PATH,
+        layout.SetBlock("/common/rtree/file_index_path",
                         make_block<char>(absolute_file_index_path.string().length() + 1));
     }
 
-    std::unordered_map<std::string, DataLayout::BlockID> name_to_block_id = {
-        {"/mld/multilevelgraph/node_array", DataLayout::MLD_GRAPH_NODE_LIST},
-        {"/mld/multilevelgraph/edge_array", DataLayout::MLD_GRAPH_EDGE_LIST},
-        {"/mld/multilevelgraph/node_to_edge_offset", DataLayout::MLD_GRAPH_NODE_TO_OFFSET},
-        {"/mld/multilevelgraph/connectivity_checksum", DataLayout::IGNORE_BLOCK},
-        {"/mld/multilevelpartition/level_data", DataLayout::MLD_LEVEL_DATA},
-        {"/mld/multilevelpartition/partition", DataLayout::MLD_PARTITION},
-        {"/mld/multilevelpartition/cell_to_children", DataLayout::MLD_CELL_TO_CHILDREN},
-        {"/mld/cellstorage/source_boundary", DataLayout::MLD_CELL_SOURCE_BOUNDARY},
-        {"/mld/cellstorage/destination_boundary", DataLayout::MLD_CELL_DESTINATION_BOUNDARY},
-        {"/mld/cellstorage/cells", DataLayout::MLD_CELLS},
-        {"/mld/cellstorage/level_to_cell_offset", DataLayout::MLD_CELL_LEVEL_OFFSETS},
-        {"/mld/metrics/0/weights", DataLayout::MLD_CELL_WEIGHTS_0},
-        {"/mld/metrics/1/weights", DataLayout::MLD_CELL_WEIGHTS_1},
-        {"/mld/metrics/2/weights", DataLayout::MLD_CELL_WEIGHTS_2},
-        {"/mld/metrics/3/weights", DataLayout::MLD_CELL_WEIGHTS_3},
-        {"/mld/metrics/4/weights", DataLayout::MLD_CELL_WEIGHTS_4},
-        {"/mld/metrics/5/weights", DataLayout::MLD_CELL_WEIGHTS_5},
-        {"/mld/metrics/6/weights", DataLayout::MLD_CELL_WEIGHTS_6},
-        {"/mld/metrics/7/weights", DataLayout::MLD_CELL_WEIGHTS_7},
-        {"/mld/metrics/0/durations", DataLayout::MLD_CELL_DURATIONS_0},
-        {"/mld/metrics/1/durations", DataLayout::MLD_CELL_DURATIONS_1},
-        {"/mld/metrics/2/durations", DataLayout::MLD_CELL_DURATIONS_2},
-        {"/mld/metrics/3/durations", DataLayout::MLD_CELL_DURATIONS_3},
-        {"/mld/metrics/4/durations", DataLayout::MLD_CELL_DURATIONS_4},
-        {"/mld/metrics/5/durations", DataLayout::MLD_CELL_DURATIONS_5},
-        {"/mld/metrics/6/durations", DataLayout::MLD_CELL_DURATIONS_6},
-        {"/mld/metrics/7/durations", DataLayout::MLD_CELL_DURATIONS_7},
-        {"/ch/checksum", DataLayout::HSGR_CHECKSUM},
-        {"/ch/contracted_graph/node_array", DataLayout::CH_GRAPH_NODE_LIST},
-        {"/ch/contracted_graph/edge_array", DataLayout::CH_GRAPH_EDGE_LIST},
-        {"/ch/connectivity_checksum", DataLayout::IGNORE_BLOCK},
-        {"/ch/edge_filter/0", DataLayout::CH_EDGE_FILTER_0},
-        {"/ch/edge_filter/1", DataLayout::CH_EDGE_FILTER_1},
-        {"/ch/edge_filter/2", DataLayout::CH_EDGE_FILTER_2},
-        {"/ch/edge_filter/3", DataLayout::CH_EDGE_FILTER_3},
-        {"/ch/edge_filter/4", DataLayout::CH_EDGE_FILTER_4},
-        {"/ch/edge_filter/5", DataLayout::CH_EDGE_FILTER_5},
-        {"/ch/edge_filter/6", DataLayout::CH_EDGE_FILTER_6},
-        {"/ch/edge_filter/7", DataLayout::CH_EDGE_FILTER_7},
-        {"/common/intersection_bearings/bearing_values", DataLayout::BEARING_VALUES},
-        {"/common/intersection_bearings/node_to_class_id", DataLayout::BEARING_CLASSID},
-        {"/common/intersection_bearings/class_id_to_ranges/block_offsets",
-         DataLayout::BEARING_OFFSETS},
-        {"/common/intersection_bearings/class_id_to_ranges/diff_blocks",
-         DataLayout::BEARING_BLOCKS},
-        {"/common/entry_classes", DataLayout::ENTRY_CLASS},
-        {"/common/properties", DataLayout::PROPERTIES},
-        {"/common/coordinates", DataLayout::COORDINATE_LIST},
-        {"/common/osm_node_ids/packed", DataLayout::OSM_NODE_ID_LIST},
-        {"/common/data_sources_names", DataLayout::DATASOURCES_NAMES},
-        {"/common/segment_data/index", DataLayout::GEOMETRIES_INDEX},
-        {"/common/segment_data/nodes", DataLayout::GEOMETRIES_NODE_LIST},
-        {"/common/segment_data/forward_weights/packed", DataLayout::GEOMETRIES_FWD_WEIGHT_LIST},
-        {"/common/segment_data/reverse_weights/packed", DataLayout::GEOMETRIES_REV_WEIGHT_LIST},
-        {"/common/segment_data/forward_durations/packed", DataLayout::GEOMETRIES_FWD_DURATION_LIST},
-        {"/common/segment_data/reverse_durations/packed", DataLayout::GEOMETRIES_REV_DURATION_LIST},
-        {"/common/segment_data/forward_data_sources", DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST},
-        {"/common/segment_data/reverse_data_sources", DataLayout::GEOMETRIES_REV_DATASOURCES_LIST},
-        {"/common/ebg_node_data/nodes", DataLayout::EDGE_BASED_NODE_DATA_LIST},
-        {"/common/ebg_node_data/annotations", DataLayout::ANNOTATION_DATA_LIST},
-        {"/common/turn_lanes/offsets", DataLayout::LANE_DESCRIPTION_OFFSETS},
-        {"/common/turn_lanes/masks", DataLayout::LANE_DESCRIPTION_MASKS},
-        {"/common/turn_lanes/data", DataLayout::TURN_LANE_DATA},
-        {"/common/maneuver_overrides/overrides", DataLayout::MANEUVER_OVERRIDES},
-        {"/common/maneuver_overrides/node_sequences", DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES},
-        {"/common/turn_penalty/weight", DataLayout::TURN_WEIGHT_PENALTIES},
-        {"/common/turn_penalty/duration", DataLayout::TURN_DURATION_PENALTIES},
-        {"/common/turn_data/pre_turn_bearings", DataLayout::PRE_TURN_BEARING},
-        {"/common/turn_data/post_turn_bearings", DataLayout::POST_TURN_BEARING},
-        {"/common/turn_data/turn_instructions", DataLayout::TURN_INSTRUCTION},
-        {"/common/turn_data/lane_data_ids", DataLayout::LANE_DATA_ID},
-        {"/common/turn_data/entry_class_ids", DataLayout::ENTRY_CLASSID},
-        {"/common/turn_data/connectivity_checksum", DataLayout::IGNORE_BLOCK},
-        {"/common/names/blocks", DataLayout::NAME_BLOCKS},
-        {"/common/names/values", DataLayout::NAME_VALUES},
-        {"/common/rtree/search_tree", DataLayout::R_SEARCH_TREE},
-        {"/common/rtree/search_tree_level_starts", DataLayout::R_SEARCH_TREE_LEVEL_STARTS},
-    };
-    std::vector<NamedBlock> blocks;
-
     constexpr bool REQUIRED = true;
     constexpr bool OPTIONAL = false;
     std::vector<std::pair<bool, boost::filesystem::path>> tar_files = {
@@ -349,7 +272,7 @@ void Storage::PopulateLayout(DataLayout &layout)
     {
         if (boost::filesystem::exists(file.second))
         {
-            readBlocks(file.second, std::back_inserter(blocks));
+            readBlocks(file.second, layout);
         }
         else
         {
@@ -360,17 +283,6 @@ void Storage::PopulateLayout(DataLayout &layout)
             }
         }
     }
-
-    for (const auto &block : blocks)
-    {
-        auto id_iter = name_to_block_id.find(std::get<0>(block));
-        if (id_iter == name_to_block_id.end())
-        {
-            throw util::exception("Could not map " + std::get<0>(block) +
-                                  " to a region in memory.");
-        }
-        layout.SetBlock(id_iter->second, std::get<1>(block));
-    }
 }
 
 void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
@@ -385,14 +297,14 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // store the filename of the on-disk portion of the RTree
     {
         const auto file_index_path_ptr =
-            layout.GetBlockPtr<char, true>(memory_ptr, DataLayout::FILE_INDEX_PATH);
+            layout.GetBlockPtr<char, true>(memory_ptr, "/common/rtree/file_index_path");
         // make sure we have 0 ending
         std::fill(file_index_path_ptr,
-                  file_index_path_ptr + layout.GetBlockSize(DataLayout::FILE_INDEX_PATH),
+                  file_index_path_ptr + layout.GetBlockSize("/common/rtree/file_index_path"),
                   0);
         const auto absolute_file_index_path =
             boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string();
-        BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize(DataLayout::FILE_INDEX_PATH)) >=
+        BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize("/common/rtree/file_index_path")) >=
                      absolute_file_index_path.size());
         std::copy(
             absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr);
@@ -402,15 +314,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     {
         const auto name_blocks_ptr =
             layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference, true>(
-                memory_ptr, DataLayout::NAME_BLOCKS);
+                memory_ptr, "/common/names/blocks");
         const auto name_values_ptr =
             layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType, true>(
-                memory_ptr, DataLayout::NAME_VALUES);
+                memory_ptr, "/common/names/values");
 
         util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
-            name_blocks_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
+            name_blocks_ptr, layout.GetBlockEntries("/common/names/blocks"));
         util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
-            name_values_ptr, layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
+            name_values_ptr, layout.GetBlockEntries("/common/names/values"));
 
         extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
         extractor::NameTableView name_table{index_data_view};
@@ -420,24 +332,24 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Turn lane data
     {
         const auto turn_lane_data_ptr = layout.GetBlockPtr<util::guidance::LaneTupleIdPair, true>(
-            memory_ptr, DataLayout::TURN_LANE_DATA);
+            memory_ptr, "/common/turn_lanes/data");
         util::vector_view<util::guidance::LaneTupleIdPair> turn_lane_data(
-            turn_lane_data_ptr, layout.GetBlockEntries(storage::DataLayout::TURN_LANE_DATA));
+            turn_lane_data_ptr, layout.GetBlockEntries("/common/turn_lanes/data"));
 
         extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
     }
 
     // Turn lane descriptions
     {
-        auto offsets_ptr = layout.GetBlockPtr<std::uint32_t, true>(
-            memory_ptr, storage::DataLayout::LANE_DESCRIPTION_OFFSETS);
+        auto offsets_ptr =
+            layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/turn_lanes/offsets");
         util::vector_view<std::uint32_t> offsets(
-            offsets_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_OFFSETS));
+            offsets_ptr, layout.GetBlockEntries("/common/turn_lanes/offsets"));
 
         auto masks_ptr = layout.GetBlockPtr<extractor::TurnLaneType::Mask, true>(
-            memory_ptr, storage::DataLayout::LANE_DESCRIPTION_MASKS);
+            memory_ptr, "/common/turn_lanes/masks");
         util::vector_view<extractor::TurnLaneType::Mask> masks(
-            masks_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DESCRIPTION_MASKS));
+            masks_ptr, layout.GetBlockEntries("/common/turn_lanes/masks"));
 
         extractor::files::readTurnLaneDescriptions(config.GetPath(".osrm.tls"), offsets, masks);
     }
@@ -445,17 +357,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Load edge-based nodes data
     {
         auto edge_based_node_data_list_ptr = layout.GetBlockPtr<extractor::EdgeBasedNode, true>(
-            memory_ptr, storage::DataLayout::EDGE_BASED_NODE_DATA_LIST);
+            memory_ptr, "/common/ebg_node_data/nodes");
         util::vector_view<extractor::EdgeBasedNode> edge_based_node_data(
-            edge_based_node_data_list_ptr,
-            layout.GetBlockEntries(storage::DataLayout::EDGE_BASED_NODE_DATA_LIST));
+            edge_based_node_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes"));
 
         auto annotation_data_list_ptr =
             layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation, true>(
-                memory_ptr, storage::DataLayout::ANNOTATION_DATA_LIST);
+                memory_ptr, "/common/ebg_node_data/annotations");
         util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data(
-            annotation_data_list_ptr,
-            layout.GetBlockEntries(storage::DataLayout::ANNOTATION_DATA_LIST));
+            annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations"));
 
         extractor::EdgeBasedNodeDataView node_data(std::move(edge_based_node_data),
                                                    std::move(annotation_data));
@@ -466,30 +376,30 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Load original edge data
     {
         const auto lane_data_id_ptr =
-            layout.GetBlockPtr<LaneDataID, true>(memory_ptr, storage::DataLayout::LANE_DATA_ID);
+            layout.GetBlockPtr<LaneDataID, true>(memory_ptr, "/common/turn_data/lane_data_ids");
         util::vector_view<LaneDataID> lane_data_ids(
-            lane_data_id_ptr, layout.GetBlockEntries(storage::DataLayout::LANE_DATA_ID));
+            lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids"));
 
         const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction, true>(
-            memory_ptr, storage::DataLayout::TURN_INSTRUCTION);
+            memory_ptr, "/common/turn_data/turn_instructions");
         util::vector_view<guidance::TurnInstruction> turn_instructions(
             turn_instruction_list_ptr,
-            layout.GetBlockEntries(storage::DataLayout::TURN_INSTRUCTION));
+            layout.GetBlockEntries("/common/turn_data/turn_instructions"));
 
-        const auto entry_class_id_list_ptr =
-            layout.GetBlockPtr<EntryClassID, true>(memory_ptr, storage::DataLayout::ENTRY_CLASSID);
+        const auto entry_class_id_list_ptr = layout.GetBlockPtr<EntryClassID, true>(
+            memory_ptr, "/common/turn_data/entry_class_ids");
         util::vector_view<EntryClassID> entry_class_ids(
-            entry_class_id_list_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASSID));
+            entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
 
         const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>(
-            memory_ptr, storage::DataLayout::PRE_TURN_BEARING);
+            memory_ptr, "/common/turn_data/pre_turn_bearings");
         util::vector_view<guidance::TurnBearing> pre_turn_bearings(
-            pre_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::PRE_TURN_BEARING));
+            pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings"));
 
         const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>(
-            memory_ptr, storage::DataLayout::POST_TURN_BEARING);
+            memory_ptr, "/common/turn_data/post_turn_bearings");
         util::vector_view<guidance::TurnBearing> post_turn_bearings(
-            post_turn_bearing_ptr, layout.GetBlockEntries(storage::DataLayout::POST_TURN_BEARING));
+            post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings"));
 
         guidance::TurnDataView turn_data(std::move(turn_instructions),
                                          std::move(lane_data_ids),
@@ -504,63 +414,63 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // load compressed geometry
     {
         auto geometries_index_ptr =
-            layout.GetBlockPtr<unsigned, true>(memory_ptr, storage::DataLayout::GEOMETRIES_INDEX);
+            layout.GetBlockPtr<unsigned, true>(memory_ptr, "/common/segment_data/index");
         util::vector_view<unsigned> geometry_begin_indices(
-            geometries_index_ptr, layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_INDEX));
+            geometries_index_ptr, layout.GetBlockEntries("/common/segment_data/index"));
 
-        auto num_entries = layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_NODE_LIST);
+        auto num_entries = layout.GetBlockEntries("/common/segment_data/nodes");
 
         auto geometries_node_list_ptr =
-            layout.GetBlockPtr<NodeID, true>(memory_ptr, storage::DataLayout::GEOMETRIES_NODE_LIST);
+            layout.GetBlockPtr<NodeID, true>(memory_ptr, "/common/segment_data/nodes");
         util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
 
         auto geometries_fwd_weight_list_ptr =
             layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
-                memory_ptr, storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST);
+                memory_ptr, "/common/segment_data/forward_weights/packed");
         extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
             util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
                 geometries_fwd_weight_list_ptr,
-                layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_WEIGHT_LIST)),
+                layout.GetBlockEntries("/common/segment_data/forward_weights/packed")),
             num_entries);
 
         auto geometries_rev_weight_list_ptr =
             layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
-                memory_ptr, storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST);
+                memory_ptr, "/common/segment_data/reverse_weights/packed");
         extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
             util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
                 geometries_rev_weight_list_ptr,
-                layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_WEIGHT_LIST)),
+                layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")),
             num_entries);
 
         auto geometries_fwd_duration_list_ptr =
             layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
-                memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST);
+                memory_ptr, "/common/segment_data/forward_durations/packed");
         extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
             util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
                 geometries_fwd_duration_list_ptr,
-                layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DURATION_LIST)),
+                layout.GetBlockEntries("/common/segment_data/forward_durations/packed")),
             num_entries);
 
         auto geometries_rev_duration_list_ptr =
             layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
-                memory_ptr, storage::DataLayout::GEOMETRIES_REV_DURATION_LIST);
+                memory_ptr, "/common/segment_data/reverse_durations/packed");
         extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
             util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
                 geometries_rev_duration_list_ptr,
-                layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DURATION_LIST)),
+                layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")),
             num_entries);
 
         auto geometries_fwd_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
-            memory_ptr, storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST);
+            memory_ptr, "/common/segment_data/forward_data_sources");
         util::vector_view<DatasourceID> geometry_fwd_datasources_list(
             geometries_fwd_datasources_list_ptr,
-            layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_FWD_DATASOURCES_LIST));
+            layout.GetBlockEntries("/common/segment_data/forward_data_sources"));
 
         auto geometries_rev_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
-            memory_ptr, storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST);
+            memory_ptr, "/common/segment_data/reverse_data_sources");
         util::vector_view<DatasourceID> geometry_rev_datasources_list(
             geometries_rev_datasources_list_ptr,
-            layout.GetBlockEntries(storage::DataLayout::GEOMETRIES_REV_DATASOURCES_LIST));
+            layout.GetBlockEntries("/common/segment_data/reverse_data_sources"));
 
         extractor::SegmentDataView segment_data{std::move(geometry_begin_indices),
                                                 std::move(geometry_node_list),
@@ -576,7 +486,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     {
         const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
-            memory_ptr, DataLayout::DATASOURCES_NAMES);
+            memory_ptr, "/common/data_sources_names");
         extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"),
                                           *datasources_names_ptr);
     }
@@ -584,38 +494,36 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Loading list of coordinates
     {
         const auto coordinates_ptr =
-            layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, DataLayout::COORDINATE_LIST);
+            layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, "/common/coordinates");
         const auto osmnodeid_ptr =
             layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type, true>(
-                memory_ptr, DataLayout::OSM_NODE_ID_LIST);
+                memory_ptr, "/common/osm_node_ids/packed");
         util::vector_view<util::Coordinate> coordinates(
-            coordinates_ptr, layout.GetBlockEntries(DataLayout::COORDINATE_LIST));
+            coordinates_ptr, layout.GetBlockEntries("/common/coordinates"));
         extractor::PackedOSMIDsView osm_node_ids(
             util::vector_view<extractor::PackedOSMIDsView::block_type>(
-                osmnodeid_ptr, layout.GetBlockEntries(DataLayout::OSM_NODE_ID_LIST)),
-            layout.GetBlockEntries(DataLayout::COORDINATE_LIST));
+                osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")),
+            layout.GetBlockEntries("/common/coordinates"));
 
         extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
     }
 
     // load turn weight penalties
     {
-        auto turn_duration_penalties_ptr = layout.GetBlockPtr<TurnPenalty, true>(
-            memory_ptr, storage::DataLayout::TURN_WEIGHT_PENALTIES);
+        auto turn_duration_penalties_ptr =
+            layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, "/common/turn_penalty/weight");
         util::vector_view<TurnPenalty> turn_duration_penalties(
-            turn_duration_penalties_ptr,
-            layout.GetBlockEntries(storage::DataLayout::TURN_WEIGHT_PENALTIES));
+            turn_duration_penalties_ptr, layout.GetBlockEntries("/common/turn_penalty/weight"));
         extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
                                                 turn_duration_penalties);
     }
 
     // load turn duration penalties
     {
-        auto turn_duration_penalties_ptr = layout.GetBlockPtr<TurnPenalty, true>(
-            memory_ptr, storage::DataLayout::TURN_DURATION_PENALTIES);
+        auto turn_duration_penalties_ptr =
+            layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, "/common/turn_penalty/duration");
         util::vector_view<TurnPenalty> turn_duration_penalties(
-            turn_duration_penalties_ptr,
-            layout.GetBlockEntries(storage::DataLayout::TURN_DURATION_PENALTIES));
+            turn_duration_penalties_ptr, layout.GetBlockEntries("/common/turn_penalty/duration"));
         extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
                                                   turn_duration_penalties);
     }
@@ -624,15 +532,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     {
 
         const auto rtree_ptr =
-            layout.GetBlockPtr<RTreeNode, true>(memory_ptr, DataLayout::R_SEARCH_TREE);
+            layout.GetBlockPtr<RTreeNode, true>(memory_ptr, "/common/rtree/search_tree");
         util::vector_view<RTreeNode> search_tree(
-            rtree_ptr, layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
+            rtree_ptr, layout.GetBlockEntries("/common/rtree/search_tree"));
 
         const auto rtree_levelstarts_ptr = layout.GetBlockPtr<std::uint64_t, true>(
-            memory_ptr, DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
+            memory_ptr, "/common/rtree/search_tree_level_starts");
         util::vector_view<std::uint64_t> rtree_level_starts(
             rtree_levelstarts_ptr,
-            layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
+            layout.GetBlockEntries("/common/rtree/search_tree_level_starts"));
 
         // we need this purely for the interface
         util::vector_view<util::Coordinate> empty_coords;
@@ -648,7 +556,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // load profile properties
     {
         const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>(
-            memory_ptr, DataLayout::PROPERTIES);
+            memory_ptr, "/common/properties");
         extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
                                                 *profile_properties_ptr);
     }
@@ -656,24 +564,29 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Load intersection data
     {
         auto bearing_class_id_ptr = layout.GetBlockPtr<BearingClassID, true>(
-            memory_ptr, storage::DataLayout::BEARING_CLASSID);
+            memory_ptr, "/common/intersection_bearings/node_to_class_id");
         util::vector_view<BearingClassID> bearing_class_id(
-            bearing_class_id_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_CLASSID));
+            bearing_class_id_ptr,
+            layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id"));
 
         auto bearing_values_ptr = layout.GetBlockPtr<DiscreteBearing, true>(
-            memory_ptr, storage::DataLayout::BEARING_VALUES);
+            memory_ptr, "/common/intersection_bearings/bearing_values");
         util::vector_view<DiscreteBearing> bearing_values(
-            bearing_values_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_VALUES));
+            bearing_values_ptr,
+            layout.GetBlockEntries("/common/intersection_bearings/bearing_values"));
 
-        auto offsets_ptr =
-            layout.GetBlockPtr<unsigned, true>(memory_ptr, storage::DataLayout::BEARING_OFFSETS);
+        auto offsets_ptr = layout.GetBlockPtr<unsigned, true>(
+            memory_ptr, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
         auto blocks_ptr =
             layout.GetBlockPtr<util::RangeTable<16, storage::Ownership::View>::BlockT, true>(
-                memory_ptr, storage::DataLayout::BEARING_BLOCKS);
+                memory_ptr, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
         util::vector_view<unsigned> bearing_offsets(
-            offsets_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_OFFSETS));
+            offsets_ptr,
+            layout.GetBlockEntries(
+                "/common/intersection_bearings/class_id_to_ranges/block_offsets"));
         util::vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT> bearing_blocks(
-            blocks_ptr, layout.GetBlockEntries(storage::DataLayout::BEARING_BLOCKS));
+            blocks_ptr,
+            layout.GetBlockEntries("/common/intersection_bearings/class_id_to_ranges/diff_blocks"));
 
         util::RangeTable<16, storage::Ownership::View> bearing_range_table(
             bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size()));
@@ -682,9 +595,9 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
 
         auto entry_class_ptr = layout.GetBlockPtr<util::guidance::EntryClass, true>(
-            memory_ptr, storage::DataLayout::ENTRY_CLASS);
+            memory_ptr, "/common/entry_classes");
         util::vector_view<util::guidance::EntryClass> entry_classes(
-            entry_class_ptr, layout.GetBlockEntries(storage::DataLayout::ENTRY_CLASS));
+            entry_class_ptr, layout.GetBlockEntries("/common/entry_classes"));
 
         extractor::files::readIntersections(
             config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
@@ -695,25 +608,22 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         {
             auto graph_nodes_ptr =
                 layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
-                    memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST);
+                    memory_ptr, "/ch/contracted_graph/node_array");
             auto graph_edges_ptr =
                 layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
-                    memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST);
-            auto checksum =
-                layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
+                    memory_ptr, "/ch/contracted_graph/edge_array");
+            auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
 
             util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
-                graph_nodes_ptr, layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_NODE_LIST));
+                graph_nodes_ptr, layout.GetBlockEntries("/ch/contracted_graph/node_array"));
             util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
-                graph_edges_ptr, layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_EDGE_LIST));
+                graph_edges_ptr, layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
 
             std::vector<util::vector_view<bool>> edge_filter;
             for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
             {
-                auto block_id =
-                    static_cast<DataLayout::BlockID>(storage::DataLayout::CH_EDGE_FILTER_0 + index);
-                auto data_ptr =
-                    layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, block_id);
+                auto block_id = "/ch/edge_filter/" + std::to_string(index);
+                auto data_ptr = layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, block_id);
                 auto num_entries = layout.GetBlockEntries(block_id);
                 edge_filter.emplace_back(data_ptr, num_entries);
             }
@@ -736,35 +646,35 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         }
         else
         {
-            layout.GetBlockPtr<unsigned, true>(memory_ptr, DataLayout::HSGR_CHECKSUM);
+            layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
             layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
-                memory_ptr, DataLayout::CH_GRAPH_NODE_LIST);
+                memory_ptr, "/ch/contracted_graph/node_array");
             layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
-                memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST);
+                memory_ptr, "/ch/contracted_graph/edge_array");
         }
     }
 
     { // Loading MLD Data
         if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
         {
-            BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0);
-            BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_TO_CHILDREN) > 0);
-            BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_PARTITION) > 0);
+            BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
+            BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
+            BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
 
             auto level_data =
                 layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
-                    memory_ptr, storage::DataLayout::MLD_LEVEL_DATA);
+                    memory_ptr, "/mld/multilevelpartition/level_data");
 
             auto mld_partition_ptr = layout.GetBlockPtr<PartitionID, true>(
-                memory_ptr, storage::DataLayout::MLD_PARTITION);
+                memory_ptr, "/mld/multilevelpartition/partition");
             auto partition_entries_count =
-                layout.GetBlockEntries(storage::DataLayout::MLD_PARTITION);
+                layout.GetBlockEntries("/mld/multilevelpartition/partition");
             util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
 
             auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
-                memory_ptr, storage::DataLayout::MLD_CELL_TO_CHILDREN);
+                memory_ptr, "/mld/multilevelpartition/cell_to_children");
             auto children_entries_count =
-                layout.GetBlockEntries(storage::DataLayout::MLD_CELL_TO_CHILDREN);
+                layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
             util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
 
             partitioner::MultiLevelPartitionView mlp{
@@ -774,25 +684,25 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
         if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
         {
-            BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
-            BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
+            BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
+            BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
 
-            auto mld_source_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
-                memory_ptr, storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
+            auto mld_source_boundary_ptr =
+                layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/source_boundary");
             auto mld_destination_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
-                memory_ptr, storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
+                memory_ptr, "/mld/cellstorage/destination_boundary");
             auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>(
-                memory_ptr, storage::DataLayout::MLD_CELLS);
+                memory_ptr, "/mld/cellstorage/cells");
             auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
-                memory_ptr, storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
+                memory_ptr, "/mld/cellstorage/level_to_cell_offset");
 
             auto source_boundary_entries_count =
-                layout.GetBlockEntries(storage::DataLayout::MLD_CELL_SOURCE_BOUNDARY);
+                layout.GetBlockEntries("/mld/cellstorage/source_boundary");
             auto destination_boundary_entries_count =
-                layout.GetBlockEntries(storage::DataLayout::MLD_CELL_DESTINATION_BOUNDARY);
-            auto cells_entries_counts = layout.GetBlockEntries(storage::DataLayout::MLD_CELLS);
+                layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
+            auto cells_entries_counts = layout.GetBlockEntries("/mld/cellstorage/cells");
             auto cell_level_offsets_entries_count =
-                layout.GetBlockEntries(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS);
+                layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
 
             util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
                                                       source_boundary_entries_count);
@@ -812,17 +722,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
         if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
         {
-            BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELLS) > 0);
-            BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_CELL_LEVEL_OFFSETS) > 0);
+            BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
+            BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
 
             std::vector<customizer::CellMetricView> metrics;
 
             for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
             {
-                auto weights_block_id = static_cast<DataLayout::BlockID>(
-                    storage::DataLayout::MLD_CELL_WEIGHTS_0 + index);
-                auto durations_block_id = static_cast<DataLayout::BlockID>(
-                    storage::DataLayout::MLD_CELL_DURATIONS_0 + index);
+                auto weights_block_id = "/mld/metrics/" + std::to_string(index) + "/weights";
+                auto durations_block_id = "/mld/metrics/" + std::to_string(index) + "/durations";
 
                 auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
                 auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
@@ -846,21 +754,21 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
             auto graph_nodes_ptr =
                 layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>(
-                    memory_ptr, storage::DataLayout::MLD_GRAPH_NODE_LIST);
+                    memory_ptr, "/mld/multilevelgraph/node_array");
             auto graph_edges_ptr =
                 layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>(
-                    memory_ptr, storage::DataLayout::MLD_GRAPH_EDGE_LIST);
+                    memory_ptr, "/mld/multilevelgraph/edge_array");
             auto graph_node_to_offset_ptr =
                 layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>(
-                    memory_ptr, storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET);
+                    memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
 
             util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list(
-                graph_nodes_ptr, layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_LIST));
+                graph_nodes_ptr, layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
             util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list(
-                graph_edges_ptr, layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_EDGE_LIST));
+                graph_edges_ptr, layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
             util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset(
                 graph_node_to_offset_ptr,
-                layout.GetBlockEntries(storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET));
+                layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
 
             std::uint32_t graph_connectivity_checksum = 0;
             customizer::MultiLevelEdgeBasedGraphView graph_view(
@@ -882,15 +790,16 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         {
             const auto maneuver_overrides_ptr =
                 layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
-                    memory_ptr, DataLayout::MANEUVER_OVERRIDES);
+                    memory_ptr, "/common/maneuver_overrides/overrides");
             const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
-                memory_ptr, DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES);
+                memory_ptr, "/common/maneuver_overrides/node_sequences");
 
             util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
-                maneuver_overrides_ptr, layout.GetBlockEntries(DataLayout::MANEUVER_OVERRIDES));
+                maneuver_overrides_ptr,
+                layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
             util::vector_view<NodeID> maneuver_override_node_sequences(
                 maneuver_override_node_sequences_ptr,
-                layout.GetBlockEntries(DataLayout::MANEUVER_OVERRIDE_NODE_SEQUENCES));
+                layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
 
             extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
                                                     maneuver_overrides,
diff --git a/unit_tests/storage/data_layout.cpp b/unit_tests/storage/data_layout.cpp
new file mode 100644
index 00000000000..57e14458308
--- /dev/null
+++ b/unit_tests/storage/data_layout.cpp
@@ -0,0 +1,54 @@
+#include "storage/shared_datatype.hpp"
+
+#include "../common/range_tools.hpp"
+#include "../common/temporary_file.hpp"
+
+#include <boost/filesystem.hpp>
+#include <boost/test/unit_test.hpp>
+
+#include <cmath>
+
+BOOST_AUTO_TEST_SUITE(serialization)
+
+using namespace osrm;
+using namespace osrm::storage;
+
+BOOST_AUTO_TEST_CASE(layout_write_test)
+{
+    DataLayout layout;
+
+    Block block_1 {20, 8*20};
+    Block block_2 {1, 4*1};
+    Block block_3 {100, static_cast<std::uint64_t>(std::ceil(100/8.))};
+
+    layout.SetBlock("block1", block_1);
+    layout.SetBlock("block2", block_2);
+    layout.SetBlock("block3", block_3);
+
+    // Canary and alignment change layout size
+    BOOST_CHECK_GT(layout.GetSizeOfLayout(), block_1.byte_size + block_2.byte_size + block_3.byte_size);
+
+    BOOST_CHECK_EQUAL(layout.GetBlockSize("block1"), block_1.byte_size);
+    BOOST_CHECK_EQUAL(layout.GetBlockSize("block2"), block_2.byte_size);
+    BOOST_CHECK_EQUAL(layout.GetBlockSize("block3"), block_3.byte_size);
+
+    std::vector<char> buffer(layout.GetSizeOfLayout());
+    auto smallest_addr = buffer.data();
+    auto biggest_addr = buffer.data() + buffer.size();
+
+    {
+        auto block_1_ptr = layout.GetBlockPtr<std::uint64_t, true>(buffer.data(), "block1");
+        auto block_2_ptr = layout.GetBlockPtr<std::uint32_t, true>(buffer.data(), "block2");
+        auto block_3_ptr = layout.GetBlockPtr<unsigned char, true>(buffer.data(), "block3");
+
+        BOOST_CHECK_LT(smallest_addr, block_1_ptr);
+        BOOST_CHECK_GT(biggest_addr, block_1_ptr + layout.GetBlockEntries("block1"));
+        BOOST_CHECK_LT(smallest_addr, block_2_ptr);
+        BOOST_CHECK_GT(biggest_addr, block_2_ptr + layout.GetBlockEntries("block3"));
+        BOOST_CHECK_LT(smallest_addr, block_3_ptr);
+        BOOST_CHECK_GT(biggest_addr, block_3_ptr + layout.GetBlockEntries("block3"));
+    }
+
+}
+
+BOOST_AUTO_TEST_SUITE_END()

From ab94c93d7cbc75a74978ed42c8283cd9cbaa8ebe Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Fri, 23 Mar 2018 12:20:05 +0000
Subject: [PATCH 02/19] Add datalayout tests

---
 unit_tests/storage/data_layout.cpp   | 40 ++++++++++++++++++----------
 unit_tests/storage/serialization.cpp | 29 ++++++++++++++++++++
 2 files changed, 55 insertions(+), 14 deletions(-)

diff --git a/unit_tests/storage/data_layout.cpp b/unit_tests/storage/data_layout.cpp
index 57e14458308..6b628ecbea4 100644
--- a/unit_tests/storage/data_layout.cpp
+++ b/unit_tests/storage/data_layout.cpp
@@ -8,7 +8,7 @@
 
 #include <cmath>
 
-BOOST_AUTO_TEST_SUITE(serialization)
+BOOST_AUTO_TEST_SUITE(data_layout)
 
 using namespace osrm;
 using namespace osrm::storage;
@@ -17,16 +17,17 @@ BOOST_AUTO_TEST_CASE(layout_write_test)
 {
     DataLayout layout;
 
-    Block block_1 {20, 8*20};
-    Block block_2 {1, 4*1};
-    Block block_3 {100, static_cast<std::uint64_t>(std::ceil(100/8.))};
+    Block block_1{20, 8 * 20};
+    Block block_2{1, 4 * 1};
+    Block block_3{100, static_cast<std::uint64_t>(std::ceil(100 / 64.))};
 
     layout.SetBlock("block1", block_1);
     layout.SetBlock("block2", block_2);
     layout.SetBlock("block3", block_3);
 
     // Canary and alignment change layout size
-    BOOST_CHECK_GT(layout.GetSizeOfLayout(), block_1.byte_size + block_2.byte_size + block_3.byte_size);
+    BOOST_CHECK_GT(layout.GetSizeOfLayout(),
+                   block_1.byte_size + block_2.byte_size + block_3.byte_size);
 
     BOOST_CHECK_EQUAL(layout.GetBlockSize("block1"), block_1.byte_size);
     BOOST_CHECK_EQUAL(layout.GetBlockSize("block2"), block_2.byte_size);
@@ -39,16 +40,27 @@ BOOST_AUTO_TEST_CASE(layout_write_test)
     {
         auto block_1_ptr = layout.GetBlockPtr<std::uint64_t, true>(buffer.data(), "block1");
         auto block_2_ptr = layout.GetBlockPtr<std::uint32_t, true>(buffer.data(), "block2");
-        auto block_3_ptr = layout.GetBlockPtr<unsigned char, true>(buffer.data(), "block3");
-
-        BOOST_CHECK_LT(smallest_addr, block_1_ptr);
-        BOOST_CHECK_GT(biggest_addr, block_1_ptr + layout.GetBlockEntries("block1"));
-        BOOST_CHECK_LT(smallest_addr, block_2_ptr);
-        BOOST_CHECK_GT(biggest_addr, block_2_ptr + layout.GetBlockEntries("block3"));
-        BOOST_CHECK_LT(smallest_addr, block_3_ptr);
-        BOOST_CHECK_GT(biggest_addr, block_3_ptr + layout.GetBlockEntries("block3"));
-    }
+        auto block_3_ptr = layout.GetBlockPtr<std::uint64_t, true>(buffer.data(), "block3");
+
+        BOOST_CHECK_LT(reinterpret_cast<std::size_t>(smallest_addr),
+                       reinterpret_cast<std::size_t>(block_1_ptr));
+        BOOST_CHECK_GT(
+            reinterpret_cast<std::size_t>(biggest_addr),
+            reinterpret_cast<std::size_t>(block_1_ptr + layout.GetBlockEntries("block1")));
 
+        BOOST_CHECK_LT(reinterpret_cast<std::size_t>(smallest_addr),
+                       reinterpret_cast<std::size_t>(block_2_ptr));
+        BOOST_CHECK_GT(
+            reinterpret_cast<std::size_t>(biggest_addr),
+            reinterpret_cast<std::size_t>(block_2_ptr + layout.GetBlockEntries("block2")));
+
+        BOOST_CHECK_LT(reinterpret_cast<std::size_t>(smallest_addr),
+                       reinterpret_cast<std::size_t>(block_3_ptr));
+        BOOST_CHECK_GT(reinterpret_cast<std::size_t>(biggest_addr),
+                       reinterpret_cast<std::size_t>(
+                           block_3_ptr + static_cast<std::size_t>(
+                                             std::ceil(layout.GetBlockEntries("block3") / 64))));
+    }
 }
 
 BOOST_AUTO_TEST_SUITE_END()
diff --git a/unit_tests/storage/serialization.cpp b/unit_tests/storage/serialization.cpp
index 3496d1e9cdc..e8153a92f6a 100644
--- a/unit_tests/storage/serialization.cpp
+++ b/unit_tests/storage/serialization.cpp
@@ -115,4 +115,33 @@ BOOST_AUTO_TEST_CASE(tar_serialize_deallocting_vector)
     }
 }
 
+BOOST_AUTO_TEST_CASE(buffer_serialize_map)
+{
+    std::map<std::string, std::int32_t> map = {
+        {"foo", 1},
+        {"barrrr", 2},
+        {"bal", 3},
+        {"bazbar", 4},
+        {"foofofofo", 5},
+    };
+
+    std::string buffer;
+    {
+    io::BufferWriter writer;
+    storage::serialization::write(writer, map);
+    buffer = writer.GetBuffer();
+    }
+
+    std::map<std::string, std::int32_t> result;
+    {
+        io::BufferReader reader(buffer);
+        storage::serialization::read(reader, result);
+    }
+
+    for (auto &pair : map)
+    {
+        BOOST_CHECK_EQUAL(pair.second, result[pair.first]);
+    }
+}
+
 BOOST_AUTO_TEST_SUITE_END()

From 383e19e67b231eea5d318358c2d10cb11aacc126 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Mon, 26 Mar 2018 10:46:07 +0000
Subject: [PATCH 03/19] More refactor

---
 include/storage/shared_datatype.hpp | 57 ++++++++++++++++++-----------
 src/storage/storage.cpp             | 24 ++++++------
 2 files changed, 47 insertions(+), 34 deletions(-)

diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index 7d72ad912a0..fbd6f579d59 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -39,24 +39,12 @@ class DataLayout
 
     inline uint64_t GetBlockEntries(const std::string &name) const
     {
-        auto iter = blocks.find(name);
-        if (iter == blocks.end())
-        {
-            throw util::exception("Could not find block " + name);
-        }
-
-        return iter->second.num_entries;
+        return GetBlock(name).num_entries;
     }
 
     inline uint64_t GetBlockSize(const std::string &name) const
     {
-        auto iter = blocks.find(name);
-        if (iter == blocks.end())
-        {
-            throw util::exception("Could not find block " + name);
-        }
-
-        return iter->second.byte_size;
+        return GetBlock(name).byte_size;
     }
 
     inline uint64_t GetSizeOfLayout() const
@@ -80,6 +68,7 @@ class DataLayout
         {
             char *start_canary_ptr = ptr - sizeof(CANARY);
             char *end_canary_ptr = ptr + GetBlockSize(name);
+            std::cout << name << ": " << (long) (start_canary_ptr-shared_memory) << " -> " << (long) (end_canary_ptr-shared_memory) << std::endl;
             std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
             std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
         }
@@ -103,10 +92,34 @@ class DataLayout
         return (T *)ptr;
     }
 
+    template<typename OutIter>
+    void List(const std::string& name_prefix, OutIter out) const
+    {
+        for (const auto& pair : blocks)
+        {
+            // check if string begins with the name prefix
+            if (pair.first.find(name_prefix) == 0)
+            {
+                *out++ = pair.first;
+            }
+        }
+    }
+
   private:
     friend void serialization::read(io::BufferReader &reader, DataLayout &layout);
     friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout);
 
+    const Block& GetBlock(const std::string& name) const
+    {
+        auto iter = blocks.find(name);
+        if (iter == blocks.end())
+        {
+            throw util::exception("Could not find block " + name);
+        }
+
+        return iter->second;
+    }
+
     // Fit aligned storage in buffer to 64 bytes to conform with AVX 512 types
     inline void *align(void *&ptr) const noexcept
     {
@@ -117,11 +130,17 @@ class DataLayout
 
     inline void *GetAlignedBlockPtr(void *ptr, const std::string &name) const
     {
-        for (auto iter = blocks.begin(); iter != blocks.end() && iter->first != name; ++iter)
+        auto block_iter = blocks.find(name);
+        if (block_iter == blocks.end())
+        {
+            throw util::exception("Could not find block " + name);
+        }
+
+        for (auto iter = blocks.begin(); iter != block_iter; ++iter)
         {
             ptr = static_cast<char *>(ptr) + sizeof(CANARY);
             ptr = align(ptr);
-            ptr = static_cast<char *>(ptr) + GetBlockSize(name);
+            ptr = static_cast<char *>(ptr) + iter->second.byte_size;
             ptr = static_cast<char *>(ptr) + sizeof(CANARY);
         }
 
@@ -130,12 +149,6 @@ class DataLayout
         return ptr;
     }
 
-    template <typename T> inline T *GetBlockEnd(char *shared_memory, const std::string &name) const
-    {
-        auto begin = GetBlockPtr<T>(shared_memory, name);
-        return begin + GetBlockEntries(name);
-    }
-
     static constexpr std::size_t BLOCK_ALIGNMENT = 64;
     std::map<std::string, Block> blocks;
 };
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 051dcc0c7b1..5971b612693 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -170,7 +170,7 @@ int Storage::Run(int max_wait)
 
     // Copy memory layout to shared memory and populate data
     char *shared_memory_ptr = static_cast<char *>(data_memory->Ptr());
-    memcpy(shared_memory_ptr, encoded_layout.data(), encoded_layout.size());
+    std::copy_n(encoded_layout.data(), encoded_layout.size(), shared_memory_ptr);
     PopulateData(layout, shared_memory_ptr + encoded_layout.size());
 
     { // Lock for write access shared region mutex
@@ -304,8 +304,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
                   0);
         const auto absolute_file_index_path =
             boost::filesystem::absolute(config.GetPath(".osrm.fileIndex")).string();
-        BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize("/common/rtree/file_index_path")) >=
-                     absolute_file_index_path.size());
+        BOOST_ASSERT(static_cast<std::size_t>(layout.GetBlockSize(
+                         "/common/rtree/file_index_path")) >= absolute_file_index_path.size());
         std::copy(
             absolute_file_index_path.begin(), absolute_file_index_path.end(), file_index_path_ptr);
     }
@@ -386,8 +386,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             turn_instruction_list_ptr,
             layout.GetBlockEntries("/common/turn_data/turn_instructions"));
 
-        const auto entry_class_id_list_ptr = layout.GetBlockPtr<EntryClassID, true>(
-            memory_ptr, "/common/turn_data/entry_class_ids");
+        const auto entry_class_id_list_ptr =
+            layout.GetBlockPtr<EntryClassID, true>(memory_ptr, "/common/turn_data/entry_class_ids");
         util::vector_view<EntryClassID> entry_class_ids(
             entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
 
@@ -620,13 +620,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
                 graph_edges_ptr, layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
 
             std::vector<util::vector_view<bool>> edge_filter;
-            for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
-            {
-                auto block_id = "/ch/edge_filter/" + std::to_string(index);
-                auto data_ptr = layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, block_id);
-                auto num_entries = layout.GetBlockEntries(block_id);
-                edge_filter.emplace_back(data_ptr, num_entries);
-            }
+            layout.List(
+                "/ch/edge_filter", boost::make_function_output_iterator([&](const auto &name) {
+                    auto data_ptr =
+                        layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, name);
+                    auto num_entries = layout.GetBlockEntries(name);
+                    edge_filter.emplace_back(data_ptr, num_entries);
+                }));
 
             std::uint32_t graph_connectivity_checksum = 0;
             contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list));

From e0a914e3f090239b961fe1970fcf29df60fc01d2 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Mon, 26 Mar 2018 12:00:37 +0000
Subject: [PATCH 04/19] Add directory listing

---
 include/storage/shared_datatype.hpp           | 55 +++++++++++++++----
 .../datafacade/shared_memory_allocator.cpp    |  1 +
 src/storage/storage.cpp                       |  8 ++-
 unit_tests/storage/data_layout.cpp            | 38 +++++++++++++
 4 files changed, 89 insertions(+), 13 deletions(-)

diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index fbd6f579d59..5d2b4893f80 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -13,6 +13,7 @@
 #include <array>
 #include <cstdint>
 #include <map>
+#include <unordered_set>
 
 namespace osrm
 {
@@ -27,6 +28,32 @@ inline void read(io::BufferReader &reader, DataLayout &layout);
 inline void write(io::BufferWriter &writer, const DataLayout &layout);
 }
 
+namespace detail
+{
+// Removes the file name if name_prefix is a directory and name is not a file in that directory
+inline std::string trimName(const std::string &name_prefix, const std::string &name)
+{
+    // list directory and
+    if (name_prefix.back() == '/')
+    {
+        auto directory_position = name.find_first_of("/", name_prefix.length());
+        // this is a "file" in the directory of name_prefix
+        if (directory_position == std::string::npos)
+        {
+            return name;
+        }
+        else
+        {
+            return name.substr(0, directory_position);
+        }
+    }
+    else
+    {
+        return name;
+    }
+}
+}
+
 // Added at the start and end of each block as sanity check
 const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
 
@@ -42,10 +69,7 @@ class DataLayout
         return GetBlock(name).num_entries;
     }
 
-    inline uint64_t GetBlockSize(const std::string &name) const
-    {
-        return GetBlock(name).byte_size;
-    }
+    inline uint64_t GetBlockSize(const std::string &name) const { return GetBlock(name).byte_size; }
 
     inline uint64_t GetSizeOfLayout() const
     {
@@ -68,7 +92,8 @@ class DataLayout
         {
             char *start_canary_ptr = ptr - sizeof(CANARY);
             char *end_canary_ptr = ptr + GetBlockSize(name);
-            std::cout << name << ": " << (long) (start_canary_ptr-shared_memory) << " -> " << (long) (end_canary_ptr-shared_memory) << std::endl;
+            std::cout << name << ": " << (long)(start_canary_ptr - shared_memory) << " -> "
+                      << (long)(end_canary_ptr - shared_memory) << std::endl;
             std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
             std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
         }
@@ -92,15 +117,25 @@ class DataLayout
         return (T *)ptr;
     }
 
-    template<typename OutIter>
-    void List(const std::string& name_prefix, OutIter out) const
+    // Depending on the name prefix this function either lists all blocks with the same prefix
+    // or all entries in the sub-directory.
+    // '/ch/edge' -> '/ch/edge_filter/0/blocks', '/ch/edge_filter/1/blocks'
+    // '/ch/edge_filters/' -> '/ch/edge_filter/0', '/ch/edge_filter/1'
+    template <typename OutIter> void List(const std::string &name_prefix, OutIter out) const
     {
-        for (const auto& pair : blocks)
+        std::unordered_set<std::string> returned_name;
+
+        for (const auto &pair : blocks)
         {
             // check if string begins with the name prefix
             if (pair.first.find(name_prefix) == 0)
             {
-                *out++ = pair.first;
+                auto trimmed_name = detail::trimName(name_prefix, pair.first);
+                auto ret = returned_name.insert(trimmed_name);
+                if (ret.second)
+                {
+                    *out++ = trimmed_name;
+                }
             }
         }
     }
@@ -109,7 +144,7 @@ class DataLayout
     friend void serialization::read(io::BufferReader &reader, DataLayout &layout);
     friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout);
 
-    const Block& GetBlock(const std::string& name) const
+    const Block &GetBlock(const std::string &name) const
     {
         auto iter = blocks.find(name);
         if (iter == blocks.end())
diff --git a/src/engine/datafacade/shared_memory_allocator.cpp b/src/engine/datafacade/shared_memory_allocator.cpp
index cb31d091e39..91331acc01d 100644
--- a/src/engine/datafacade/shared_memory_allocator.cpp
+++ b/src/engine/datafacade/shared_memory_allocator.cpp
@@ -23,6 +23,7 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
     storage::io::BufferReader reader(GetMemory());
     storage::serialization::read(reader, data_layout);
     layout_size = reader.GetPosition();
+    util::Log(logDEBUG) << "Data layout has size " << layout_size;
 }
 
 SharedMemoryAllocator::~SharedMemoryAllocator() {}
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 5971b612693..2295b8dd50f 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -727,10 +727,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
             std::vector<customizer::CellMetricView> metrics;
 
-            for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
+            std::vector<std::string> metric_prefix_names;
+            layout.List("/mld/metrics/", std::back_inserter(metric_prefix_names));
+            for (const auto &prefix : metric_prefix_names)
             {
-                auto weights_block_id = "/mld/metrics/" + std::to_string(index) + "/weights";
-                auto durations_block_id = "/mld/metrics/" + std::to_string(index) + "/durations";
+                auto weights_block_id = prefix + "/weights";
+                auto durations_block_id = prefix + "/durations";
 
                 auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
                 auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
diff --git a/unit_tests/storage/data_layout.cpp b/unit_tests/storage/data_layout.cpp
index 6b628ecbea4..b97a2020650 100644
--- a/unit_tests/storage/data_layout.cpp
+++ b/unit_tests/storage/data_layout.cpp
@@ -63,4 +63,42 @@ BOOST_AUTO_TEST_CASE(layout_write_test)
     }
 }
 
+BOOST_AUTO_TEST_CASE(layout_list_test)
+{
+    DataLayout layout;
+
+    Block block_1{20, 8 * 20};
+    Block block_2{1, 4 * 1};
+    Block block_3{100, static_cast<std::uint64_t>(std::ceil(100 / 64.))};
+
+    layout.SetBlock("/ch/edge_filter/block1", block_1);
+    layout.SetBlock("/ch/edge_filter/block2", block_2);
+    layout.SetBlock("/ch/edge_filter/block3", block_3);
+    layout.SetBlock("/mld/metrics/0/durations", block_2);
+    layout.SetBlock("/mld/metrics/0/weights", block_3);
+    layout.SetBlock("/mld/metrics/1/durations", block_2);
+    layout.SetBlock("/mld/metrics/1/weights", block_3);
+
+    std::vector<std::string> results_1;
+    std::vector<std::string> results_2;
+    std::vector<std::string> results_3;
+    layout.List("/ch/edge_filter", std::back_inserter(results_1));
+    layout.List("/ch/edge_filter/", std::back_inserter(results_2));
+    layout.List("/ch/", std::back_inserter(results_3));
+
+    std::vector<std::string> results_4;
+    std::vector<std::string> results_5;
+    std::vector<std::string> results_6;
+    layout.List("/mld/metrics", std::back_inserter(results_4));
+    layout.List("/mld/metrics/", std::back_inserter(results_5));
+    layout.List("/mld/", std::back_inserter(results_6));
+
+    CHECK_EQUAL_RANGE(results_1, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
+    CHECK_EQUAL_RANGE(results_2, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
+    CHECK_EQUAL_RANGE(results_3, "/ch/edge_filter");
+    CHECK_EQUAL_RANGE(results_4, "/mld/metrics/0/durations", "/mld/metrics/0/weights", "/mld/metrics/1/durations", "/mld/metrics/1/weights");
+    CHECK_EQUAL_RANGE(results_5, "/mld/metrics/0", "/mld/metrics/1");
+    CHECK_EQUAL_RANGE(results_6, "/mld/metrics");
+}
+
 BOOST_AUTO_TEST_SUITE_END()

From 2396b029e5f305670bbdb0f9f396076a7e00341e Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Mon, 26 Mar 2018 14:23:52 +0000
Subject: [PATCH 05/19] Use boost array sink to avoid copying buffer for
 BufferReader

---
 include/storage/io.hpp | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/include/storage/io.hpp b/include/storage/io.hpp
index 159c149b708..fe2351ca3d1 100644
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -12,6 +12,8 @@
 
 #include <boost/filesystem/fstream.hpp>
 #include <boost/iostreams/seek.hpp>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/device/array.hpp>
 
 #include <cerrno>
 #include <cstring>
@@ -294,7 +296,11 @@ class FileWriter
 class BufferReader
 {
   public:
-    BufferReader(const std::string &buffer) : input_stream(buffer, std::ios::binary)
+    BufferReader(const std::string &buffer) : BufferReader(buffer.data(), buffer.size())
+    {
+    }
+
+    BufferReader(const char* buffer, const std::size_t size) : input_stream(boost::iostreams::array_source(buffer, size))
     {
         if (!input_stream)
         {
@@ -342,7 +348,7 @@ class BufferReader
     }
 
   private:
-    std::istringstream input_stream;
+    boost::iostreams::stream<boost::iostreams::array_source> input_stream;
 };
 
 class BufferWriter

From de0e31afd9f4e8fc7d4a570a3171b56e11b86162 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Mon, 26 Mar 2018 15:01:48 +0000
Subject: [PATCH 06/19] Fixed reading DataLayout from shared memory

---
 include/engine/engine.hpp                     | 36 +++++++++++--------
 include/storage/shared_datatype.hpp           |  2 ++
 include/storage/shared_memory.hpp             |  2 ++
 .../datafacade/shared_memory_allocator.cpp    |  2 +-
 src/osrm/osrm.cpp                             |  6 ----
 src/storage/storage.cpp                       |  1 +
 6 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp
index 79df2f922f1..dc4e9ae550c 100644
--- a/include/engine/engine.hpp
+++ b/include/engine/engine.hpp
@@ -19,6 +19,9 @@
 #include "engine/plugins/viaroute.hpp"
 #include "engine/routing_algorithms.hpp"
 #include "engine/status.hpp"
+
+#include "storage/serialization.hpp"
+
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/fingerprint.hpp"
@@ -151,9 +154,11 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
         boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
-        auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
-        return layout->GetBlockSize("/ch/contracted_graph/node_array") > 4 &&
-               layout->GetBlockSize("/ch/contracted_graph/edge_array") > 4;
+        storage::DataLayout layout;
+        storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
+        storage::serialization::read(reader, layout);
+        return layout.HasBlock("/ch/contracted_graph/node_array") &&
+               layout.HasBlock("/ch/contracted_graph/edge_array");
     }
     else
     {
@@ -171,21 +176,24 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
         boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
-        auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
+        storage::DataLayout layout;
+        storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
+        storage::serialization::read(reader, layout);
+
         // checks that all the needed memory blocks are populated
         // "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
         // are not checked, because in situations where there are so few nodes in the graph that
         // they all fit into one cell, they can be empty.
-        bool empty_data = layout->GetBlockSize("/mld/multilevelpartition/level_data") > 0 &&
-                          layout->GetBlockSize("/mld/multilevelpartition/partition") > 0 &&
-                          layout->GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0 &&
-                          layout->GetBlockSize("/mld/cellstorage/cells") > 0 &&
-                          layout->GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0 &&
-                          layout->GetBlockSize("/mld/multilevelgraph/node_array") > 0 &&
-                          layout->GetBlockSize("/mld/multilevelgraph/edge_array") > 0 &&
-                          layout->GetBlockSize("/mld/metrics/0/weights") > 0 &&
-                          layout->GetBlockSize("/mld/metrics/0/durations") > 0 &&
-                          layout->GetBlockSize("/mld/multilevelgraph/node_to_edge_offset") > 0;
+        bool empty_data = layout.HasBlock("/mld/multilevelpartition/level_data") &&
+                          layout.HasBlock("/mld/multilevelpartition/partition") &&
+                          layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
+                          layout.HasBlock("/mld/cellstorage/cells") &&
+                          layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
+                          layout.HasBlock("/mld/multilevelgraph/node_array") &&
+                          layout.HasBlock("/mld/multilevelgraph/edge_array") &&
+                          layout.HasBlock("/mld/metrics/0/weights") &&
+                          layout.HasBlock("/mld/metrics/0/durations") &&
+                          layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
         return empty_data;
     }
     else
diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index 5d2b4893f80..b42b6be91d6 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -71,6 +71,8 @@ class DataLayout
 
     inline uint64_t GetBlockSize(const std::string &name) const { return GetBlock(name).byte_size; }
 
+    inline bool HasBlock(const std::string &name) const { return blocks.find(name) != blocks.end(); }
+
     inline uint64_t GetSizeOfLayout() const
     {
         uint64_t result = 0;
diff --git a/include/storage/shared_memory.hpp b/include/storage/shared_memory.hpp
index a3ced80d2e0..86dc3fe556d 100644
--- a/include/storage/shared_memory.hpp
+++ b/include/storage/shared_memory.hpp
@@ -47,6 +47,7 @@ class SharedMemory
 {
   public:
     void *Ptr() const { return region.get_address(); }
+    std::size_t Size() const { return region.get_size(); }
 
     SharedMemory(const SharedMemory &) = delete;
     SharedMemory &operator=(const SharedMemory &) = delete;
@@ -200,6 +201,7 @@ class SharedMemory
 
   public:
     void *Ptr() const { return region.get_address(); }
+    std::size_t Size() const { return region.get_size(); }
 
     SharedMemory(const boost::filesystem::path &lock_file, const int id, const uint64_t size = 0)
     {
diff --git a/src/engine/datafacade/shared_memory_allocator.cpp b/src/engine/datafacade/shared_memory_allocator.cpp
index 91331acc01d..e3ef0f0c1d0 100644
--- a/src/engine/datafacade/shared_memory_allocator.cpp
+++ b/src/engine/datafacade/shared_memory_allocator.cpp
@@ -20,7 +20,7 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
     BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region));
     m_large_memory = storage::makeSharedMemory(data_region);
 
-    storage::io::BufferReader reader(GetMemory());
+    storage::io::BufferReader reader(reinterpret_cast<char *>(m_large_memory->Ptr()), m_large_memory->Size());
     storage::serialization::read(reader, data_layout);
     layout_size = reader.GetPosition();
     util::Log(logDEBUG) << "Data layout has size " << layout_size;
diff --git a/src/osrm/osrm.cpp b/src/osrm/osrm.cpp
index c672c77d50d..9a190a7a7d0 100644
--- a/src/osrm/osrm.cpp
+++ b/src/osrm/osrm.cpp
@@ -32,12 +32,6 @@ OSRM::OSRM(engine::EngineConfig &config)
         storage::SharedMonitor<storage::SharedDataTimestamp> barrier;
         using mutex_type = typename decltype(barrier)::mutex_type;
         boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
-
-        auto mem = storage::makeSharedMemory(barrier.data().region);
-        auto layout = reinterpret_cast<storage::DataLayout *>(mem->Ptr());
-        if (layout->GetBlockSize("/common/names/values") == 0)
-            throw util::exception(
-                "No name data loaded, cannot continue.  Have you run osrm-datastore to load data?");
     }
 
     // Now, check that the algorithm requested can be used with the data
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 2295b8dd50f..622e5c42849 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -165,6 +165,7 @@ int Storage::Run(int max_wait)
 
     // Allocate shared memory block
     auto regions_size = encoded_layout.size() + layout.GetSizeOfLayout();
+    util::Log() << "Data layout has a size of " << encoded_layout.size() << " bytes";
     util::Log() << "Allocating shared memory of " << regions_size << " bytes";
     auto data_memory = makeSharedMemory(next_region, regions_size);
 

From 50ccba4e32719c9015efabab9278e0993aa01d99 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Tue, 27 Mar 2018 09:44:13 +0000
Subject: [PATCH 07/19] Refactor metric storage

---
 include/contractor/contracted_metric.hpp      |  26 ++
 include/contractor/files.hpp                  |  57 +--
 include/contractor/serialization.hpp          |  53 +++
 include/customizer/files.hpp                  |  42 +-
 include/engine/algorithm.hpp                  |   5 +
 .../contiguous_internalmem_datafacade.hpp     |  57 +--
 include/engine/datafacade_factory.hpp         |  33 +-
 include/engine/engine.hpp                     |  47 ++-
 include/storage/serialization.hpp             | 131 +++---
 include/storage/shared_datatype.hpp           |   2 -
 src/contractor/contractor.cpp                 |   8 +-
 src/customize/customizer.cpp                  |   5 +-
 src/storage/storage.cpp                       | 372 +++++++++---------
 unit_tests/contractor/files.cpp               |  33 +-
 14 files changed, 503 insertions(+), 368 deletions(-)
 create mode 100644 include/contractor/contracted_metric.hpp
 create mode 100644 include/contractor/serialization.hpp

diff --git a/include/contractor/contracted_metric.hpp b/include/contractor/contracted_metric.hpp
new file mode 100644
index 00000000000..e8592b32ad6
--- /dev/null
+++ b/include/contractor/contracted_metric.hpp
@@ -0,0 +1,26 @@
+#ifndef OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP
+#define OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP
+
+#include "contractor/query_graph.hpp"
+
+namespace osrm
+{
+namespace contractor
+{
+
+namespace detail
+{
+template<storage::Ownership Ownership>
+struct ContractedMetric
+{
+    detail::QueryGraph<Ownership> graph;
+    std::vector<util::ViewOrVector<bool, Ownership>> edge_filter;
+};
+}
+
+using ContractedMetric = detail::ContractedMetric<storage::Ownership::Container>;
+using ContractedMetricView = detail::ContractedMetric<storage::Ownership::View>;
+}
+}
+
+#endif
diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp
index 39a15acaa9c..b7014f9ab0d 100644
--- a/include/contractor/files.hpp
+++ b/include/contractor/files.hpp
@@ -1,12 +1,9 @@
 #ifndef OSRM_CONTRACTOR_FILES_HPP
 #define OSRM_CONTRACTOR_FILES_HPP
 
-#include "contractor/query_graph.hpp"
+#include "contractor/serialization.hpp"
 
-#include "util/serialization.hpp"
-
-#include "storage/serialization.hpp"
-#include "storage/tar.hpp"
+#include <unordered_map>
 
 namespace osrm
 {
@@ -15,67 +12,51 @@ namespace contractor
 namespace files
 {
 // reads .osrm.hsgr file
-template <typename QueryGraphT, typename EdgeFilterT>
+template <typename ContractedMetricT>
 inline void readGraph(const boost::filesystem::path &path,
                       unsigned &checksum,
-                      QueryGraphT &graph,
-                      std::vector<EdgeFilterT> &edge_filter,
+                      std::unordered_map<std::string, ContractedMetricT> &metrics,
                       std::uint32_t &connectivity_checksum)
 {
-    static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
-                      std::is_same<QueryGraph, QueryGraphT>::value,
-                  "graph must be of type QueryGraph<>");
-    static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
-                      std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
-                  "edge_filter must be a container of vector<bool> or vector_view<bool>");
+    static_assert(std::is_same<ContractedMetric, ContractedMetricT>::value ||
+                      std::is_same<ContractedMetricView, ContractedMetricT>::value,
+                  "metric must be of type ContractedMetric<>");
 
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
     reader.ReadInto("/ch/checksum", checksum);
-    util::serialization::read(reader, "/ch/contracted_graph", graph);
+    reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
 
-    auto count = reader.ReadElementCount64("/ch/edge_filter");
-    edge_filter.resize(count);
-    for (const auto index : util::irange<std::size_t>(0, count))
+    for (auto &pair : metrics)
     {
-        storage::serialization::read(
-            reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
+        serialization::read(reader, "/ch/metrics/" + pair.first, pair.second);
     }
 
-    reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
 }
 
 // writes .osrm.hsgr file
-template <typename QueryGraphT, typename EdgeFilterT>
+template <typename ContractedMetricT>
 inline void writeGraph(const boost::filesystem::path &path,
                        unsigned checksum,
-                       const QueryGraphT &graph,
-                       const std::vector<EdgeFilterT> &edge_filter,
+                       const std::unordered_map<std::string, ContractedMetricT> &metrics,
                        const std::uint32_t connectivity_checksum)
 {
-    static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
-                      std::is_same<QueryGraph, QueryGraphT>::value,
-                  "graph must be of type QueryGraph<>");
-    static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
-                      std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
-                  "edge_filter must be a container of vector<bool> or vector_view<bool>");
+    static_assert(std::is_same<ContractedMetric, ContractedMetricT>::value ||
+                      std::is_same<ContractedMetricView, ContractedMetricT>::value,
+                  "metric must be of type ContractedMetric<>");
     const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
     storage::tar::FileWriter writer{path, fingerprint};
 
     writer.WriteElementCount64("/ch/checksum", 1);
     writer.WriteFrom("/ch/checksum", checksum);
-    util::serialization::write(writer, "/ch/contracted_graph", graph);
+    writer.WriteElementCount64("/ch/connectivity_checksum", 1);
+    writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
 
-    writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
-    for (const auto index : util::irange<std::size_t>(0, edge_filter.size()))
+    for (const auto &pair : metrics)
     {
-        storage::serialization::write(
-            writer, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
+        serialization::write(writer, "/ch/metrics/" + pair.first, pair.second);
     }
-
-    writer.WriteElementCount64("/ch/connectivity_checksum", 1);
-    writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
 }
 }
 }
diff --git a/include/contractor/serialization.hpp b/include/contractor/serialization.hpp
new file mode 100644
index 00000000000..aa86592742d
--- /dev/null
+++ b/include/contractor/serialization.hpp
@@ -0,0 +1,53 @@
+#ifndef OSRM_CONTRACTOR_SERIALIZATION_HPP
+#define OSRM_CONTRACTOR_SERIALIZATION_HPP
+
+#include "contractor/contracted_metric.hpp"
+
+#include "util/serialization.hpp"
+
+#include "storage/serialization.hpp"
+#include "storage/tar.hpp"
+
+namespace osrm
+{
+namespace contractor
+{
+namespace serialization
+{
+
+template <storage::Ownership Ownership>
+void write(storage::tar::FileWriter &writer,
+           const std::string &name,
+           const detail::ContractedMetric<Ownership> &metric)
+{
+    util::serialization::write(writer, name + "/contracted_graph", metric.graph);
+
+    writer.WriteElementCount64(name + "/exclude", metric.edge_filter.size());
+    for (const auto index : util::irange<std::size_t>(0, metric.edge_filter.size()))
+    {
+        storage::serialization::write(writer,
+                                      name + "/exclude/" + std::to_string(index) + "/edge_filter",
+                                      metric.edge_filter[index]);
+    }
+}
+
+template <storage::Ownership Ownership>
+void read(storage::tar::FileReader &reader,
+          const std::string &name,
+          detail::ContractedMetric<Ownership> &metric)
+{
+    util::serialization::read(reader, name + "/contracted_graph", metric.graph);
+
+    metric.edge_filter.resize(reader.ReadElementCount64(name + "/exclude"));
+    for (const auto index : util::irange<std::size_t>(0, metric.edge_filter.size()))
+    {
+        storage::serialization::read(reader,
+                                     name + "/exclude/" + std::to_string(index) + "/edge_filter",
+                                     metric.edge_filter[index]);
+    }
+}
+}
+}
+}
+
+#endif
diff --git a/include/customizer/files.hpp b/include/customizer/files.hpp
index 098752f9c9e..8b690ecd27c 100644
--- a/include/customizer/files.hpp
+++ b/include/customizer/files.hpp
@@ -7,6 +7,8 @@
 
 #include "util/integer_range.hpp"
 
+#include <unordered_map>
+
 namespace osrm
 {
 namespace customizer
@@ -16,7 +18,7 @@ namespace files
 
 // reads .osrm.cell_metrics file
 template <typename CellMetricT>
-inline void readCellMetrics(const boost::filesystem::path &path, std::vector<CellMetricT> &metrics)
+inline void readCellMetrics(const boost::filesystem::path &path, std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
 {
     static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
                       std::is_same<CellMetric, CellMetricT>::value,
@@ -25,20 +27,27 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::vector<Cel
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
-    auto num_metrics = reader.ReadElementCount64("/mld/metrics");
-    metrics.resize(num_metrics);
-
-    auto id = 0;
-    for (auto &metric : metrics)
+    for (auto& pair : metrics)
     {
-        serialization::read(reader, "/mld/metrics/" + std::to_string(id++), metric);
+        const auto& metric_name = pair.first;
+        auto& metric_exclude_classes = pair.second;
+
+        auto prefix = "/mld/metrics/" + metric_name + "/exclude";
+        auto num_exclude_classes = reader.ReadElementCount64(prefix);
+        metric_exclude_classes.resize(num_exclude_classes);
+
+        auto id = 0;
+        for (auto &metric : metric_exclude_classes)
+        {
+            serialization::read(reader, prefix + "/" + std::to_string(id++), metric);
+        }
     }
 }
 
 // writes .osrm.cell_metrics file
 template <typename CellMetricT>
 inline void writeCellMetrics(const boost::filesystem::path &path,
-                             const std::vector<CellMetricT> &metrics)
+                             const std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
 {
     static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
                       std::is_same<CellMetric, CellMetricT>::value,
@@ -47,12 +56,19 @@ inline void writeCellMetrics(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
     storage::tar::FileWriter writer{path, fingerprint};
 
-    writer.WriteElementCount64("/mld/metrics", metrics.size());
-
-    auto id = 0;
-    for (const auto &metric : metrics)
+    for (const auto& pair : metrics)
     {
-        serialization::write(writer, "/mld/metrics/" + std::to_string(id++), metric);
+        const auto& metric_name = pair.first;
+        const auto& metric_exclude_classes = pair.second;
+
+        auto prefix = "/mld/metrics/" + metric_name + "/exclude";
+        writer.WriteElementCount64(prefix, metric_exclude_classes.size());
+
+        auto id = 0;
+        for (auto &exclude_metric : metric_exclude_classes)
+        {
+            serialization::write(writer, prefix + "/" + std::to_string(id++), exclude_metric);
+        }
     }
 }
 }
diff --git a/include/engine/algorithm.hpp b/include/engine/algorithm.hpp
index 751ef6b9b09..8f48b373005 100644
--- a/include/engine/algorithm.hpp
+++ b/include/engine/algorithm.hpp
@@ -30,6 +30,11 @@ template <typename AlgorithmT> const char *name();
 template <> inline const char *name<ch::Algorithm>() { return "CH"; }
 template <> inline const char *name<mld::Algorithm>() { return "MLD"; }
 
+// Algorithm identifier
+template <typename AlgorithmT> const char *identifier();
+template <> inline const char *identifier<ch::Algorithm>() { return "ch"; }
+template <> inline const char *identifier<mld::Algorithm>() { return "mld"; }
+
 template <typename AlgorithmT> struct HasAlternativePathSearch final : std::false_type
 {
 };
diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
index 96b424cd09d..65f4504599a 100644
--- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
@@ -83,23 +83,26 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
 
     void InitializeGraphPointer(const storage::DataLayout &data_layout,
                                 char *memory_block,
+                                const std::string& metric_name,
                                 const std::size_t exclude_index)
     {
+        const std::string metric_prefix = "/ch/metrics/" + metric_name;
         auto graph_nodes_ptr =
-            data_layout.GetBlockPtr<GraphNode>(memory_block, "/ch/contracted_graph/node_array");
+            data_layout.GetBlockPtr<GraphNode>(memory_block, metric_prefix + "/contracted_graph/node_array");
 
         auto graph_edges_ptr =
-            data_layout.GetBlockPtr<GraphEdge>(memory_block, "/ch/contracted_graph/edge_array");
+            data_layout.GetBlockPtr<GraphEdge>(memory_block, metric_prefix + "/contracted_graph/edge_array");
 
-        auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index);
+        auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index);
+        auto filter_block_id = exclude_prefix + "/edge_filter";
 
         auto edge_filter_ptr =
             data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
 
         util::vector_view<GraphNode> node_list(
-            graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array"));
+            graph_nodes_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
         util::vector_view<GraphEdge> edge_list(
-            graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
+            graph_edges_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
 
         util::vector_view<bool> edge_filter(edge_filter_ptr,
                                             data_layout.GetBlockEntries(filter_block_id));
@@ -108,17 +111,18 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
 
   public:
     ContiguousInternalMemoryAlgorithmDataFacade(
-        std::shared_ptr<ContiguousBlockAllocator> allocator_, std::size_t exclude_index)
+        std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, std::size_t exclude_index)
         : allocator(std::move(allocator_))
     {
-        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
+        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
     }
 
     void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
+                                    const std::string& metric_name,
                                     const std::size_t exclude_index)
     {
-        InitializeGraphPointer(data_layout, memory_block, exclude_index);
+        InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index);
     }
 
     // search graph access
@@ -225,8 +229,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
                                             char *memory_block,
+                                            const std::string& metric_name,
                                             const std::size_t exclude_index)
     {
+        // TODO: For multi-metric support we need to have separate exclude classes per metric
+        (void) metric_name;
         m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
             memory_block, "/common/properties");
 
@@ -528,6 +535,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
+                                    const std::string& metric_name,
                                     const std::size_t exclude_index)
     {
         InitializeChecksumPointer(data_layout, memory_block);
@@ -538,7 +546,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         InitializeGeometryPointers(data_layout, memory_block);
         InitializeNamePointers(data_layout, memory_block);
         InitializeTurnLaneDescriptionsPointers(data_layout, memory_block);
-        InitializeProfilePropertiesPointer(data_layout, memory_block, exclude_index);
+        InitializeProfilePropertiesPointer(data_layout, memory_block, metric_name, exclude_index);
         InitializeRTreePointers(data_layout, memory_block);
         InitializeIntersectionClassPointers(data_layout, memory_block);
         InitializeManeuverOverridePointers(data_layout, memory_block);
@@ -548,10 +556,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     // allows switching between process_memory/shared_memory datafacade, based on the type of
     // allocator
     ContiguousInternalMemoryDataFacadeBase(std::shared_ptr<ContiguousBlockAllocator> allocator_,
+                                            const std::string& metric_name,
                                            const std::size_t exclude_index)
         : allocator(std::move(allocator_))
     {
-        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
+        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
     }
 
     // node and edge information access
@@ -959,9 +968,10 @@ class ContiguousInternalMemoryDataFacade<CH>
 {
   public:
     ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
+            const std::string& metric_name,
                                        const std::size_t exclude_index)
-        : ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
-          ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, exclude_index)
+        : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
+          ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, metric_name, exclude_index)
 
     {
     }
@@ -981,14 +991,16 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
 
     void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
+                                    const std::string& metric_name,
                                     const std::size_t exclude_index)
     {
-        InitializeMLDDataPointers(data_layout, memory_block, exclude_index);
+        InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index);
         InitializeGraphPointer(data_layout, memory_block);
     }
 
     void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
                                    char *memory_block,
+                                   const std::string& metric_name,
                                    const std::size_t exclude_index)
     {
         if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
@@ -1016,9 +1028,9 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
                 partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
         }
 
-        const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights";
-        const auto durations_block_id =
-            "/mld/metrics/" + std::to_string(exclude_index) + "/durations";
+        const auto exclude_prefix = "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index);
+        const auto weights_block_id = exclude_prefix + "/weights";
+        const auto durations_block_id = exclude_prefix + "/durations";
 
         if (data_layout.GetBlockSize(weights_block_id) > 0)
         {
@@ -1026,8 +1038,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
                 data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
             auto mld_cell_durations_ptr =
                 data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
-            auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights");
-            auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations");
+            auto weight_entries_count = data_layout.GetBlockEntries(weights_block_id);
+            auto duration_entries_count = data_layout.GetBlockEntries(durations_block_id);
             BOOST_ASSERT(weight_entries_count == duration_entries_count);
             util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
             util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
@@ -1099,10 +1111,10 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
 
   public:
     ContiguousInternalMemoryAlgorithmDataFacade(
-        std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::size_t exclude_index)
+        std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, const std::size_t exclude_index)
         : allocator(std::move(allocator_))
     {
-        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
+        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
     }
 
     const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override
@@ -1156,9 +1168,10 @@ class ContiguousInternalMemoryDataFacade<MLD> final
   private:
   public:
     ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
+                                       const std::string& metric_name,
                                        const std::size_t exclude_index)
-        : ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
-          ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, exclude_index)
+        : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
+          ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, metric_name, exclude_index)
 
     {
     }
diff --git a/include/engine/datafacade_factory.hpp b/include/engine/datafacade_factory.hpp
index 85de49bf5c0..bb313db9040 100644
--- a/include/engine/datafacade_factory.hpp
+++ b/include/engine/datafacade_factory.hpp
@@ -33,6 +33,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
     DataFacadeFactory(std::shared_ptr<AllocatorT> allocator)
         : DataFacadeFactory(allocator, has_exclude_flags)
     {
+        BOOST_ASSERT_MSG(facades.size() >= 1, "At least one datafacade is needed");
     }
 
     template <typename ParameterT> std::shared_ptr<const Facade> Get(const ParameterT &params) const
@@ -45,14 +46,28 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
     template <typename AllocatorT>
     DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::true_type)
     {
-        for (const auto index : util::irange<std::size_t>(0, facades.size()))
+        const auto &layout = allocator->GetLayout();
+        properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
+            allocator->GetMemory(), "/common/properties");
+        const auto &metric_name = properties->GetWeightName();
+
+        std::vector<std::string> exclude_prefixes;
+        auto exclude_path = std::string("/") + routing_algorithms::identifier<AlgorithmT>() +
+                            std::string("/metrics/") + metric_name + "/exclude/";
+        layout.List(exclude_path, std::back_inserter(exclude_prefixes));
+        facades.resize(exclude_prefixes.size());
+
+        for (const auto &exclude_prefix : exclude_prefixes)
         {
-            facades[index] = std::make_shared<const Facade>(allocator, index);
+            auto index_begin = exclude_prefix.find_last_of("/");
+            BOOST_ASSERT_MSG(index_begin != std::string::npos,
+                             "The exclude prefix needs to be a valid data path.");
+            std::size_t index =
+                std::stoi(exclude_prefix.substr(index_begin + 1, exclude_prefix.size()));
+            BOOST_ASSERT(index >= 0 && index < facades.size());
+            facades[index] = std::make_shared<const Facade>(allocator, metric_name, index);
         }
 
-        properties = allocator->GetLayout().template GetBlockPtr<extractor::ProfileProperties>(
-            allocator->GetMemory(), "/common/properties");
-
         for (const auto index : util::irange<std::size_t>(0, properties->class_names.size()))
         {
             const std::string name = properties->GetClassName(index);
@@ -67,7 +82,11 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
     template <typename AllocatorT>
     DataFacadeFactory(std::shared_ptr<AllocatorT> allocator, std::false_type)
     {
-        facades[0] = std::make_shared<const Facade>(allocator, 0);
+        const auto &layout = allocator->GetLayout();
+        properties = layout.template GetBlockPtr<extractor::ProfileProperties>(
+            allocator->GetMemory(), "/common/properties");
+        const auto &metric_name = properties->GetWeightName();
+        facades.push_back(std::make_shared<const Facade>(allocator, metric_name, 0));
     }
 
     std::shared_ptr<const Facade> Get(const api::TileParameters &, std::false_type) const
@@ -124,7 +143,7 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
         return {};
     }
 
-    std::array<std::shared_ptr<const Facade>, extractor::MAX_EXCLUDABLE_CLASSES> facades;
+    std::vector<std::shared_ptr<const Facade>> facades;
     std::unordered_map<std::string, extractor::ClassData> name_to_class;
     const extractor::ProfileProperties *properties = nullptr;
 };
diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp
index dc4e9ae550c..3a2bab2e4c8 100644
--- a/include/engine/engine.hpp
+++ b/include/engine/engine.hpp
@@ -155,10 +155,19 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
         storage::DataLayout layout;
-        storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
+        storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
         storage::serialization::read(reader, layout);
-        return layout.HasBlock("/ch/contracted_graph/node_array") &&
-               layout.HasBlock("/ch/contracted_graph/edge_array");
+
+        std::vector<std::string> metric_prefixes;
+        layout.List("/ch/metrics/", std::back_inserter(metric_prefixes));
+
+        bool has_graph = false;
+        for (const auto &metric_prefix : metric_prefixes)
+        {
+            has_graph |= layout.HasBlock(metric_prefix + "/contracted_graph/node_array") &&
+                         layout.HasBlock(metric_prefix + "/contracted_graph/edge_array");
+        }
+        return has_graph;
     }
     else
     {
@@ -177,24 +186,32 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
 
         auto mem = storage::makeSharedMemory(barrier.data().region);
         storage::DataLayout layout;
-        storage::io::BufferReader reader(reinterpret_cast<const char*>(mem->Ptr()), mem->Size());
+        storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
         storage::serialization::read(reader, layout);
 
+        std::vector<std::string> metric_prefixes;
+        layout.List("/mld/metrics/", std::back_inserter(metric_prefixes));
+
+        bool has_cells = false;
+        for (const auto &metric_prefix : metric_prefixes)
+        {
+            has_cells |= layout.HasBlock(metric_prefix + "/exclude/0/durations") &&
+                         layout.HasBlock(metric_prefix + "/exclude/0/weights");
+        }
+
         // checks that all the needed memory blocks are populated
         // "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
         // are not checked, because in situations where there are so few nodes in the graph that
         // they all fit into one cell, they can be empty.
-        bool empty_data = layout.HasBlock("/mld/multilevelpartition/level_data") &&
-                          layout.HasBlock("/mld/multilevelpartition/partition") &&
-                          layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
-                          layout.HasBlock("/mld/cellstorage/cells") &&
-                          layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
-                          layout.HasBlock("/mld/multilevelgraph/node_array") &&
-                          layout.HasBlock("/mld/multilevelgraph/edge_array") &&
-                          layout.HasBlock("/mld/metrics/0/weights") &&
-                          layout.HasBlock("/mld/metrics/0/durations") &&
-                          layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
-        return empty_data;
+        bool has_data = has_cells && layout.HasBlock("/mld/multilevelpartition/level_data") &&
+                        layout.HasBlock("/mld/multilevelpartition/partition") &&
+                        layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
+                        layout.HasBlock("/mld/cellstorage/cells") &&
+                        layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
+                        layout.HasBlock("/mld/multilevelgraph/node_array") &&
+                        layout.HasBlock("/mld/multilevelgraph/edge_array") &&
+                        layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
+        return has_data;
     }
     else
     {
diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp
index 97cd2d2e5c8..ebc4d864c81 100644
--- a/include/storage/serialization.hpp
+++ b/include/storage/serialization.hpp
@@ -27,6 +27,72 @@ namespace storage
 namespace serialization
 {
 
+namespace detail
+{
+template <typename T, typename BlockT = unsigned char>
+inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
+{
+    static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
+    BlockT value = 0;
+    for (std::size_t bit = 0; bit < count; ++bit, ++index)
+        value = (value << 1) | data[index];
+    return value;
+}
+
+template <typename T, typename BlockT = unsigned char>
+inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
+{
+    static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
+    const BlockT mask = BlockT{1} << (count - 1);
+    for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
+        data[index] = value & mask;
+}
+
+template <typename VectorT>
+void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
+{
+    const auto count = reader.ReadElementCount64(name);
+    data.resize(count);
+    std::uint64_t index = 0;
+
+    constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
+
+    const auto decode = [&](const std::uint64_t block) {
+        auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
+        unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
+        index += WORD_BITS;
+    };
+
+    reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
+}
+
+template <typename VectorT>
+void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
+{
+    const auto count = data.size();
+    writer.WriteElementCount64(name, count);
+    std::uint64_t index = 0;
+
+    constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
+
+    // FIXME on old boost version the function_input_iterator does not work with lambdas
+    // so we need to wrap it in a function here.
+    const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
+        auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
+        auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
+        index += WORD_BITS;
+        return packed;
+    };
+
+    std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
+    writer.WriteStreaming<std::uint64_t>(
+        name,
+        boost::make_function_input_iterator(encode_function, boost::infinite()),
+        number_of_blocks);
+}
+}
+
+
 /* All vector formats here use the same on-disk format.
  * This is important because we want to be able to write from a vector
  * of one kind, but read it into a vector of another kind.
@@ -155,71 +221,6 @@ void write(tar::FileWriter &writer, const std::string &name, const util::vector_
     writer.WriteFrom(name, data.data(), count);
 }
 
-namespace detail
-{
-template <typename T, typename BlockT = unsigned char>
-inline BlockT packBits(const T &data, std::size_t index, std::size_t count)
-{
-    static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
-    BlockT value = 0;
-    for (std::size_t bit = 0; bit < count; ++bit, ++index)
-        value = (value << 1) | data[index];
-    return value;
-}
-
-template <typename T, typename BlockT = unsigned char>
-inline void unpackBits(T &data, std::size_t index, std::size_t count, BlockT value)
-{
-    static_assert(std::is_same<typename T::value_type, bool>::value, "value_type is not bool");
-    const BlockT mask = BlockT{1} << (count - 1);
-    for (std::size_t bit = 0; bit < count; value <<= 1, ++bit, ++index)
-        data[index] = value & mask;
-}
-
-template <typename VectorT>
-void readBoolVector(tar::FileReader &reader, const std::string &name, VectorT &data)
-{
-    const auto count = reader.ReadElementCount64(name);
-    data.resize(count);
-    std::uint64_t index = 0;
-
-    constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
-
-    const auto decode = [&](const std::uint64_t block) {
-        auto read_size = std::min<std::size_t>(count - index, WORD_BITS);
-        unpackBits<VectorT, std::uint64_t>(data, index, read_size, block);
-        index += WORD_BITS;
-    };
-
-    reader.ReadStreaming<std::uint64_t>(name, boost::make_function_output_iterator(decode));
-}
-
-template <typename VectorT>
-void writeBoolVector(tar::FileWriter &writer, const std::string &name, const VectorT &data)
-{
-    const auto count = data.size();
-    writer.WriteElementCount64(name, count);
-    std::uint64_t index = 0;
-
-    constexpr std::uint64_t WORD_BITS = CHAR_BIT * sizeof(std::uint64_t);
-
-    // FIXME on old boost version the function_input_iterator does not work with lambdas
-    // so we need to wrap it in a function here.
-    const std::function<std::uint64_t()> encode_function = [&]() -> std::uint64_t {
-        auto write_size = std::min<std::size_t>(count - index, WORD_BITS);
-        auto packed = packBits<VectorT, std::uint64_t>(data, index, write_size);
-        index += WORD_BITS;
-        return packed;
-    };
-
-    std::uint64_t number_of_blocks = (count + WORD_BITS - 1) / WORD_BITS;
-    writer.WriteStreaming<std::uint64_t>(
-        name,
-        boost::make_function_input_iterator(encode_function, boost::infinite()),
-        number_of_blocks);
-}
-}
-
 template <>
 inline void
 read<bool>(tar::FileReader &reader, const std::string &name, util::vector_view<bool> &data)
diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index b42b6be91d6..c00ece841e7 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -94,8 +94,6 @@ class DataLayout
         {
             char *start_canary_ptr = ptr - sizeof(CANARY);
             char *end_canary_ptr = ptr + GetBlockSize(name);
-            std::cout << name << ": " << (long)(start_canary_ptr - shared_memory) << " -> "
-                      << (long)(end_canary_ptr - shared_memory) << std::endl;
             std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
             std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
         }
diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp
index 45a0ce0361d..951a4420fef 100644
--- a/src/contractor/contractor.cpp
+++ b/src/contractor/contractor.cpp
@@ -79,6 +79,7 @@ int Contractor::Run()
 
     TIMER_START(contraction);
 
+    std::string metric_name;
     std::vector<std::vector<bool>> node_filters;
     {
         extractor::EdgeBasedNodeDataContainer node_data;
@@ -86,6 +87,7 @@ int Contractor::Run()
 
         extractor::ProfileProperties properties;
         extractor::files::readProfileProperties(config.GetPath(".osrm.properties"), properties);
+        metric_name = properties.GetWeightName();
 
         node_filters =
             util::excludeFlagsToNodeFilter(number_of_edge_based_nodes, node_data, properties);
@@ -105,8 +107,10 @@ int Contractor::Run()
     util::Log() << "Contracted graph has " << query_graph.GetNumberOfEdges() << " edges.";
     util::Log() << "Contraction took " << TIMER_SEC(contraction) << " sec";
 
-    files::writeGraph(
-        config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters, connectivity_checksum);
+    std::unordered_map<std::string, ContractedMetric> metrics = {
+        {metric_name, {std::move(query_graph), std::move(edge_filters)}}};
+
+    files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, metrics, connectivity_checksum);
 
     TIMER_STOP(preparing);
 
diff --git a/src/customize/customizer.cpp b/src/customize/customizer.cpp
index 06e6204bfe3..c3292a8127b 100644
--- a/src/customize/customizer.cpp
+++ b/src/customize/customizer.cpp
@@ -153,7 +153,10 @@ int Customizer::Run(const CustomizationConfig &config)
     util::Log() << "Cells customization took " << TIMER_SEC(cell_customize) << " seconds";
 
     TIMER_START(writing_mld_data);
-    files::writeCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
+    std::unordered_map<std::string, std::vector<CellMetric>> metric_exclude_classes = {
+        {properties.GetWeightName(), std::move(metrics)},
+    };
+    files::writeCellMetrics(config.GetPath(".osrm.cell_metrics"), metric_exclude_classes);
     TIMER_STOP(writing_mld_data);
     util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
 
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 622e5c42849..27632fe4dc1 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -554,12 +554,16 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
     }
 
+    // FIXME we only need to get the weight name
+    std::string metric_name;
     // load profile properties
     {
         const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>(
             memory_ptr, "/common/properties");
         extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
                                                 *profile_properties_ptr);
+
+        metric_name = profile_properties_ptr->GetWeightName();
     }
 
     // Load intersection data
@@ -604,211 +608,199 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
     }
 
-    { // Load the HSGR file
-        if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
-        {
-            auto graph_nodes_ptr =
-                layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
-                    memory_ptr, "/ch/contracted_graph/node_array");
-            auto graph_edges_ptr =
-                layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
-                    memory_ptr, "/ch/contracted_graph/edge_array");
-            auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
-
-            util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
-                graph_nodes_ptr, layout.GetBlockEntries("/ch/contracted_graph/node_array"));
-            util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
-                graph_edges_ptr, layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
-
-            std::vector<util::vector_view<bool>> edge_filter;
-            layout.List(
-                "/ch/edge_filter", boost::make_function_output_iterator([&](const auto &name) {
-                    auto data_ptr =
-                        layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, name);
-                    auto num_entries = layout.GetBlockEntries(name);
-                    edge_filter.emplace_back(data_ptr, num_entries);
-                }));
-
-            std::uint32_t graph_connectivity_checksum = 0;
-            contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list));
-            contractor::files::readGraph(config.GetPath(".osrm.hsgr"),
-                                         *checksum,
-                                         graph_view,
-                                         edge_filter,
-                                         graph_connectivity_checksum);
-            if (turns_connectivity_checksum != graph_connectivity_checksum)
-            {
-                throw util::exception(
-                    "Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
-                    " in " + config.GetPath(".osrm.hsgr").string() +
-                    " does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
-                    " in " + config.GetPath(".osrm.edges").string());
-            }
-        }
-        else
+    if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
+    {
+        const std::string metric_prefix = "/ch/metrics/" + metric_name;
+        auto graph_nodes_ptr = layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
+            memory_ptr, metric_prefix + "/contracted_graph/node_array");
+        auto graph_edges_ptr = layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
+            memory_ptr, metric_prefix + "/contracted_graph/edge_array");
+        auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
+
+        util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
+            graph_nodes_ptr,
+            layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
+        util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
+            graph_edges_ptr,
+            layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
+
+        std::vector<util::vector_view<bool>> edge_filter;
+        layout.List(
+            metric_prefix + "/exclude", boost::make_function_output_iterator([&](const auto &name) {
+                auto data_ptr =
+                    layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, name);
+                auto num_entries = layout.GetBlockEntries(name);
+                edge_filter.emplace_back(data_ptr, num_entries);
+            }));
+
+        std::uint32_t graph_connectivity_checksum = 0;
+        std::unordered_map<std::string, contractor::ContractedMetricView> metrics = {
+            {metric_name, {{std::move(node_list), std::move(edge_list)}, std::move(edge_filter)}}};
+
+        contractor::files::readGraph(
+            config.GetPath(".osrm.hsgr"), *checksum, metrics, graph_connectivity_checksum);
+        if (turns_connectivity_checksum != graph_connectivity_checksum)
         {
-            layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
-            layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
-                memory_ptr, "/ch/contracted_graph/node_array");
-            layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
-                memory_ptr, "/ch/contracted_graph/edge_array");
+            throw util::exception(
+                "Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
+                config.GetPath(".osrm.hsgr").string() + " does not equal to checksum " +
+                std::to_string(turns_connectivity_checksum) + " in " +
+                config.GetPath(".osrm.edges").string());
         }
     }
 
-    { // Loading MLD Data
-        if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
-        {
-            BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
-            BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
-            BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
-
-            auto level_data =
-                layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
-                    memory_ptr, "/mld/multilevelpartition/level_data");
-
-            auto mld_partition_ptr = layout.GetBlockPtr<PartitionID, true>(
-                memory_ptr, "/mld/multilevelpartition/partition");
-            auto partition_entries_count =
-                layout.GetBlockEntries("/mld/multilevelpartition/partition");
-            util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
-
-            auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
-                memory_ptr, "/mld/multilevelpartition/cell_to_children");
-            auto children_entries_count =
-                layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
-            util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
-
-            partitioner::MultiLevelPartitionView mlp{
-                std::move(level_data), std::move(partition), std::move(cell_to_children)};
-            partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
-        }
-
-        if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
-        {
-            BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
-            BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
-
-            auto mld_source_boundary_ptr =
-                layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/source_boundary");
-            auto mld_destination_boundary_ptr = layout.GetBlockPtr<NodeID, true>(
-                memory_ptr, "/mld/cellstorage/destination_boundary");
-            auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>(
-                memory_ptr, "/mld/cellstorage/cells");
-            auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
-                memory_ptr, "/mld/cellstorage/level_to_cell_offset");
-
-            auto source_boundary_entries_count =
-                layout.GetBlockEntries("/mld/cellstorage/source_boundary");
-            auto destination_boundary_entries_count =
-                layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
-            auto cells_entries_counts = layout.GetBlockEntries("/mld/cellstorage/cells");
-            auto cell_level_offsets_entries_count =
-                layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
-
-            util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
-                                                      source_boundary_entries_count);
-            util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
-                                                           destination_boundary_entries_count);
-            util::vector_view<partitioner::CellStorageView::CellData> cells(mld_cells_ptr,
-                                                                            cells_entries_counts);
-            util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
-                                                           cell_level_offsets_entries_count);
-
-            partitioner::CellStorageView storage{std::move(source_boundary),
-                                                 std::move(destination_boundary),
-                                                 std::move(cells),
-                                                 std::move(level_offsets)};
-            partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
-        }
+    if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
+    {
+        BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
+        BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
+        BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
+
+        auto level_data = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
+            memory_ptr, "/mld/multilevelpartition/level_data");
+
+        auto mld_partition_ptr =
+            layout.GetBlockPtr<PartitionID, true>(memory_ptr, "/mld/multilevelpartition/partition");
+        auto partition_entries_count = layout.GetBlockEntries("/mld/multilevelpartition/partition");
+        util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
+
+        auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
+            memory_ptr, "/mld/multilevelpartition/cell_to_children");
+        auto children_entries_count =
+            layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
+        util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
+
+        partitioner::MultiLevelPartitionView mlp{
+            std::move(level_data), std::move(partition), std::move(cell_to_children)};
+        partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
+    }
 
-        if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
-        {
-            BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
-            BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
+    if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
+    {
+        BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
+        BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
+
+        auto mld_source_boundary_ptr =
+            layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/source_boundary");
+        auto mld_destination_boundary_ptr =
+            layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/destination_boundary");
+        auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>(
+            memory_ptr, "/mld/cellstorage/cells");
+        auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
+            memory_ptr, "/mld/cellstorage/level_to_cell_offset");
+
+        auto source_boundary_entries_count =
+            layout.GetBlockEntries("/mld/cellstorage/source_boundary");
+        auto destination_boundary_entries_count =
+            layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
+        auto cells_entries_counts = layout.GetBlockEntries("/mld/cellstorage/cells");
+        auto cell_level_offsets_entries_count =
+            layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
+
+        util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
+                                                  source_boundary_entries_count);
+        util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
+                                                       destination_boundary_entries_count);
+        util::vector_view<partitioner::CellStorageView::CellData> cells(mld_cells_ptr,
+                                                                        cells_entries_counts);
+        util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
+                                                       cell_level_offsets_entries_count);
+
+        partitioner::CellStorageView storage{std::move(source_boundary),
+                                             std::move(destination_boundary),
+                                             std::move(cells),
+                                             std::move(level_offsets)};
+        partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
+    }
 
-            std::vector<customizer::CellMetricView> metrics;
+    if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
+    {
+        BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
+        BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
 
-            std::vector<std::string> metric_prefix_names;
-            layout.List("/mld/metrics/", std::back_inserter(metric_prefix_names));
-            for (const auto &prefix : metric_prefix_names)
-            {
-                auto weights_block_id = prefix + "/weights";
-                auto durations_block_id = prefix + "/durations";
-
-                auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
-                auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
-                auto mld_cell_weights_ptr =
-                    layout.GetBlockPtr<EdgeWeight, true>(memory_ptr, weights_block_id);
-                auto mld_cell_duration_ptr =
-                    layout.GetBlockPtr<EdgeDuration, true>(memory_ptr, durations_block_id);
-                util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
-                util::vector_view<EdgeDuration> durations(mld_cell_duration_ptr,
-                                                          duration_entries_count);
-
-                metrics.push_back(
-                    customizer::CellMetricView{std::move(weights), std::move(durations)});
-            }
+        std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = {
+            {metric_name, {}},
+        };
 
-            customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
+        std::vector<std::string> metric_prefix_names;
+        layout.List("/mld/metrics/" + metric_name + "/exclude/",
+                    std::back_inserter(metric_prefix_names));
+        for (const auto &prefix : metric_prefix_names)
+        {
+            auto weights_block_id = prefix + "/weights";
+            auto durations_block_id = prefix + "/durations";
+
+            auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
+            auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
+            auto mld_cell_weights_ptr =
+                layout.GetBlockPtr<EdgeWeight, true>(memory_ptr, weights_block_id);
+            auto mld_cell_duration_ptr =
+                layout.GetBlockPtr<EdgeDuration, true>(memory_ptr, durations_block_id);
+            util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
+            util::vector_view<EdgeDuration> durations(mld_cell_duration_ptr,
+                                                      duration_entries_count);
+
+            metrics[metric_name].push_back(
+                customizer::CellMetricView{std::move(weights), std::move(durations)});
         }
 
-        if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
-        {
+        customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
+    }
 
-            auto graph_nodes_ptr =
-                layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>(
-                    memory_ptr, "/mld/multilevelgraph/node_array");
-            auto graph_edges_ptr =
-                layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>(
-                    memory_ptr, "/mld/multilevelgraph/edge_array");
-            auto graph_node_to_offset_ptr =
-                layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>(
-                    memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
-
-            util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list(
-                graph_nodes_ptr, layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
-            util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list(
-                graph_edges_ptr, layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
-            util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset(
-                graph_node_to_offset_ptr,
-                layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
-
-            std::uint32_t graph_connectivity_checksum = 0;
-            customizer::MultiLevelEdgeBasedGraphView graph_view(
-                std::move(node_list), std::move(edge_list), std::move(node_to_offset));
-            partitioner::files::readGraph(
-                config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
-
-            if (turns_connectivity_checksum != graph_connectivity_checksum)
-            {
-                throw util::exception(
-                    "Connectivity checksum " + std::to_string(graph_connectivity_checksum) +
-                    " in " + config.GetPath(".osrm.mldgr").string() +
-                    " does not equal to checksum " + std::to_string(turns_connectivity_checksum) +
-                    " in " + config.GetPath(".osrm.edges").string());
-            }
-        }
+    if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
+    {
 
-        // load maneuver overrides
+        auto graph_nodes_ptr =
+            layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>(
+                memory_ptr, "/mld/multilevelgraph/node_array");
+        auto graph_edges_ptr =
+            layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>(
+                memory_ptr, "/mld/multilevelgraph/edge_array");
+        auto graph_node_to_offset_ptr =
+            layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>(
+                memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
+
+        util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list(
+            graph_nodes_ptr, layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
+        util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list(
+            graph_edges_ptr, layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
+        util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset(
+            graph_node_to_offset_ptr,
+            layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
+
+        std::uint32_t graph_connectivity_checksum = 0;
+        customizer::MultiLevelEdgeBasedGraphView graph_view(
+            std::move(node_list), std::move(edge_list), std::move(node_to_offset));
+        partitioner::files::readGraph(
+            config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
+
+        if (turns_connectivity_checksum != graph_connectivity_checksum)
         {
-            const auto maneuver_overrides_ptr =
-                layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
-                    memory_ptr, "/common/maneuver_overrides/overrides");
-            const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
-                memory_ptr, "/common/maneuver_overrides/node_sequences");
-
-            util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
-                maneuver_overrides_ptr,
-                layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
-            util::vector_view<NodeID> maneuver_override_node_sequences(
-                maneuver_override_node_sequences_ptr,
-                layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
-
-            extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
-                                                    maneuver_overrides,
-                                                    maneuver_override_node_sequences);
+            throw util::exception(
+                "Connectivity checksum " + std::to_string(graph_connectivity_checksum) + " in " +
+                config.GetPath(".osrm.mldgr").string() + " does not equal to checksum " +
+                std::to_string(turns_connectivity_checksum) + " in " +
+                config.GetPath(".osrm.edges").string());
         }
     }
+
+    // load maneuver overrides
+    {
+        const auto maneuver_overrides_ptr =
+            layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
+                memory_ptr, "/common/maneuver_overrides/overrides");
+        const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
+            memory_ptr, "/common/maneuver_overrides/node_sequences");
+
+        util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
+            maneuver_overrides_ptr, layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
+        util::vector_view<NodeID> maneuver_override_node_sequences(
+            maneuver_override_node_sequences_ptr,
+            layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
+
+        extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
+                                                maneuver_overrides,
+                                                maneuver_override_node_sequences);
+    }
 }
 }
 }
diff --git a/unit_tests/contractor/files.cpp b/unit_tests/contractor/files.cpp
index 7c357fca1f5..b3898c173e8 100644
--- a/unit_tests/contractor/files.cpp
+++ b/unit_tests/contractor/files.cpp
@@ -32,26 +32,33 @@ BOOST_AUTO_TEST_CASE(read_write_hsgr)
         {true, true, true, true, true, true, true},
     };
 
+    std::unordered_map<std::string, ContractedMetric> reference_metrics = {
+        {"duration", {std::move(reference_graph), std::move(reference_filters)}}};
+
     TemporaryFile tmp{TEST_DATA_DIR "/read_write_hsgr_test.osrm.hsgr"};
-    contractor::files::writeGraph(tmp.path,
-                                  reference_checksum,
-                                  reference_graph,
-                                  reference_filters,
-                                  reference_connectivity_checksum);
+    contractor::files::writeGraph(
+        tmp.path, reference_checksum, reference_metrics, reference_connectivity_checksum);
 
     unsigned checksum;
     unsigned connectivity_checksum;
-    QueryGraph graph;
-    std::vector<std::vector<bool>> filters;
-    contractor::files::readGraph(tmp.path, checksum, graph, filters, connectivity_checksum);
+
+    std::unordered_map<std::string, ContractedMetric> metrics = {
+        {"duration", {}}
+    };
+    contractor::files::readGraph(tmp.path, checksum, metrics, connectivity_checksum);
 
     BOOST_CHECK_EQUAL(checksum, reference_checksum);
     BOOST_CHECK_EQUAL(connectivity_checksum, reference_connectivity_checksum);
-    BOOST_CHECK_EQUAL(filters.size(), reference_filters.size());
-    CHECK_EQUAL_COLLECTIONS(filters[0], reference_filters[0]);
-    CHECK_EQUAL_COLLECTIONS(filters[1], reference_filters[1]);
-    CHECK_EQUAL_COLLECTIONS(filters[2], reference_filters[2]);
-    CHECK_EQUAL_COLLECTIONS(filters[3], reference_filters[3]);
+    BOOST_CHECK_EQUAL(metrics["duration"].edge_filter.size(),
+                      reference_metrics["duration"].edge_filter.size());
+    CHECK_EQUAL_COLLECTIONS(metrics["duration"].edge_filter[0],
+                            reference_metrics["duration"].edge_filter[0]);
+    CHECK_EQUAL_COLLECTIONS(metrics["duration"].edge_filter[1],
+                            reference_metrics["duration"].edge_filter[1]);
+    CHECK_EQUAL_COLLECTIONS(metrics["duration"].edge_filter[2],
+                            reference_metrics["duration"].edge_filter[2]);
+    CHECK_EQUAL_COLLECTIONS(metrics["duration"].edge_filter[3],
+                            reference_metrics["duration"].edge_filter[3]);
 }
 
 BOOST_AUTO_TEST_SUITE_END()

From 00395bc87dbe47d387a2d4400aedbd3eafa21bd1 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Wed, 28 Mar 2018 11:27:07 +0000
Subject: [PATCH 08/19] Remove CheckCompability because it now duplicates logic
 in the datafacade

---
 include/engine/engine.hpp | 87 ---------------------------------------
 src/osrm/osrm.cpp         | 27 ++----------
 src/storage/storage.cpp   |  2 -
 3 files changed, 4 insertions(+), 112 deletions(-)

diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp
index 3a2bab2e4c8..97cd65b0251 100644
--- a/include/engine/engine.hpp
+++ b/include/engine/engine.hpp
@@ -7,8 +7,6 @@
 #include "engine/api/table_parameters.hpp"
 #include "engine/api/tile_parameters.hpp"
 #include "engine/api/trip_parameters.hpp"
-#include "engine/data_watchdog.hpp"
-#include "engine/datafacade/contiguous_block_allocator.hpp"
 #include "engine/datafacade_provider.hpp"
 #include "engine/engine_config.hpp"
 #include "engine/plugins/match.hpp"
@@ -20,11 +18,6 @@
 #include "engine/routing_algorithms.hpp"
 #include "engine/status.hpp"
 
-#include "storage/serialization.hpp"
-
-#include "util/exception.hpp"
-#include "util/exception_utils.hpp"
-#include "util/fingerprint.hpp"
 #include "util/json_container.hpp"
 
 #include <memory>
@@ -126,8 +119,6 @@ template <typename Algorithm> class Engine final : public EngineInterface
         return tile_plugin.HandleRequest(GetAlgorithms(params), params, result);
     }
 
-    static bool CheckCompatibility(const EngineConfig &config);
-
   private:
     template <typename ParametersT> auto GetAlgorithms(const ParametersT &params) const
     {
@@ -143,84 +134,6 @@ template <typename Algorithm> class Engine final : public EngineInterface
     const plugins::MatchPlugin match_plugin;
     const plugins::TilePlugin tile_plugin;
 };
-
-template <>
-bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineConfig &config)
-{
-    if (config.use_shared_memory)
-    {
-        storage::SharedMonitor<storage::SharedDataTimestamp> barrier;
-        using mutex_type = typename decltype(barrier)::mutex_type;
-        boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
-
-        auto mem = storage::makeSharedMemory(barrier.data().region);
-        storage::DataLayout layout;
-        storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
-        storage::serialization::read(reader, layout);
-
-        std::vector<std::string> metric_prefixes;
-        layout.List("/ch/metrics/", std::back_inserter(metric_prefixes));
-
-        bool has_graph = false;
-        for (const auto &metric_prefix : metric_prefixes)
-        {
-            has_graph |= layout.HasBlock(metric_prefix + "/contracted_graph/node_array") &&
-                         layout.HasBlock(metric_prefix + "/contracted_graph/edge_array");
-        }
-        return has_graph;
-    }
-    else
-    {
-        return boost::filesystem::exists(config.storage_config.GetPath(".osrm.hsgr"));
-    }
-}
-
-template <>
-bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const EngineConfig &config)
-{
-    if (config.use_shared_memory)
-    {
-        storage::SharedMonitor<storage::SharedDataTimestamp> barrier;
-        using mutex_type = typename decltype(barrier)::mutex_type;
-        boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
-
-        auto mem = storage::makeSharedMemory(barrier.data().region);
-        storage::DataLayout layout;
-        storage::io::BufferReader reader(reinterpret_cast<const char *>(mem->Ptr()), mem->Size());
-        storage::serialization::read(reader, layout);
-
-        std::vector<std::string> metric_prefixes;
-        layout.List("/mld/metrics/", std::back_inserter(metric_prefixes));
-
-        bool has_cells = false;
-        for (const auto &metric_prefix : metric_prefixes)
-        {
-            has_cells |= layout.HasBlock(metric_prefix + "/exclude/0/durations") &&
-                         layout.HasBlock(metric_prefix + "/exclude/0/weights");
-        }
-
-        // checks that all the needed memory blocks are populated
-        // "/mld/cellstorage/source_boundary" and "/mld/cellstorage/destination_boundary"
-        // are not checked, because in situations where there are so few nodes in the graph that
-        // they all fit into one cell, they can be empty.
-        bool has_data = has_cells && layout.HasBlock("/mld/multilevelpartition/level_data") &&
-                        layout.HasBlock("/mld/multilevelpartition/partition") &&
-                        layout.HasBlock("/mld/multilevelpartition/cell_to_children") &&
-                        layout.HasBlock("/mld/cellstorage/cells") &&
-                        layout.HasBlock("/mld/cellstorage/level_to_cell_offset") &&
-                        layout.HasBlock("/mld/multilevelgraph/node_array") &&
-                        layout.HasBlock("/mld/multilevelgraph/edge_array") &&
-                        layout.HasBlock("/mld/multilevelgraph/node_to_edge_offset");
-        return has_data;
-    }
-    else
-    {
-        return boost::filesystem::exists(config.storage_config.GetPath(".osrm.partition")) &&
-               boost::filesystem::exists(config.storage_config.GetPath(".osrm.cells")) &&
-               boost::filesystem::exists(config.storage_config.GetPath(".osrm.mldgr")) &&
-               boost::filesystem::exists(config.storage_config.GetPath(".osrm.cell_metrics"));
-    }
-}
 }
 }
 
diff --git a/src/osrm/osrm.cpp b/src/osrm/osrm.cpp
index 9a190a7a7d0..77da7b40e57 100644
--- a/src/osrm/osrm.cpp
+++ b/src/osrm/osrm.cpp
@@ -1,4 +1,5 @@
 #include "osrm/osrm.hpp"
+
 #include "engine/algorithm.hpp"
 #include "engine/api/match_parameters.hpp"
 #include "engine/api/nearest_parameters.hpp"
@@ -37,30 +38,10 @@ OSRM::OSRM(engine::EngineConfig &config)
     // Now, check that the algorithm requested can be used with the data
     // that's available.
 
-    if (config.algorithm == EngineConfig::Algorithm::CH ||
-        config.algorithm == EngineConfig::Algorithm::CoreCH)
-    {
-        if (config.algorithm == EngineConfig::Algorithm::CoreCH)
-        {
-            util::Log(logWARNING) << "Using CoreCH is deprecated. Falling back to CH";
-            config.algorithm = EngineConfig::Algorithm::CH;
-        }
-        bool ch_compatible = engine::Engine<CH>::CheckCompatibility(config);
-
-        // throw error if dataset is not usable with CH
-        if (config.algorithm == EngineConfig::Algorithm::CH && !ch_compatible)
-        {
-            throw util::exception("Dataset is not compatible with CH");
-        }
-    }
-    else if (config.algorithm == EngineConfig::Algorithm::MLD)
+    if (config.algorithm == EngineConfig::Algorithm::CoreCH)
     {
-        bool mld_compatible = engine::Engine<MLD>::CheckCompatibility(config);
-        // throw error if dataset is not usable with MLD
-        if (!mld_compatible)
-        {
-            throw util::exception("Dataset is not compatible with MLD.");
-        }
+        util::Log(logWARNING) << "Using CoreCH is deprecated. Falling back to CH";
+        config.algorithm = EngineConfig::Algorithm::CH;
     }
 
     switch (config.algorithm)
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 27632fe4dc1..099dde34e3c 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -89,8 +89,6 @@ inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
 }
 }
 
-static constexpr std::size_t NUM_METRICS = 8;
-
 using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
 using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
 using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;

From 52ea1c12a63c32ff16776c61451c050b2034de47 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Wed, 28 Mar 2018 23:18:09 +0000
Subject: [PATCH 09/19] Fix errors when loading an incompatible dataset.

---
 include/contractor/files.hpp                         |  5 -----
 .../datafacade/contiguous_internalmem_datafacade.hpp |  6 +++---
 include/engine/datafacade/datafacade_base.hpp        |  2 +-
 include/engine/datafacade_factory.hpp                |  7 +++++++
 include/guidance/files.hpp                           |  7 +++++--
 include/guidance/serialization.hpp                   |  9 ++-------
 include/guidance/turn_data_container.hpp             | 12 ++++--------
 include/partitioner/files.hpp                        |  7 +++++--
 include/partitioner/multi_level_graph.hpp            | 12 ++++--------
 include/partitioner/serialization.hpp                |  9 ++-------
 src/contractor/contractor.cpp                        |  5 +----
 src/storage/storage.cpp                              |  8 +++++---
 unit_tests/mocks/mock_datafacade.hpp                 |  2 +-
 13 files changed, 40 insertions(+), 51 deletions(-)

diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp
index b7014f9ab0d..7ba99442329 100644
--- a/include/contractor/files.hpp
+++ b/include/contractor/files.hpp
@@ -14,7 +14,6 @@ namespace files
 // reads .osrm.hsgr file
 template <typename ContractedMetricT>
 inline void readGraph(const boost::filesystem::path &path,
-                      unsigned &checksum,
                       std::unordered_map<std::string, ContractedMetricT> &metrics,
                       std::uint32_t &connectivity_checksum)
 {
@@ -25,7 +24,6 @@ inline void readGraph(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
-    reader.ReadInto("/ch/checksum", checksum);
     reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
 
     for (auto &pair : metrics)
@@ -38,7 +36,6 @@ inline void readGraph(const boost::filesystem::path &path,
 // writes .osrm.hsgr file
 template <typename ContractedMetricT>
 inline void writeGraph(const boost::filesystem::path &path,
-                       unsigned checksum,
                        const std::unordered_map<std::string, ContractedMetricT> &metrics,
                        const std::uint32_t connectivity_checksum)
 {
@@ -48,8 +45,6 @@ inline void writeGraph(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
     storage::tar::FileWriter writer{path, fingerprint};
 
-    writer.WriteElementCount64("/ch/checksum", 1);
-    writer.WriteFrom("/ch/checksum", checksum);
     writer.WriteElementCount64("/ch/connectivity_checksum", 1);
     writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
 
diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
index 65f4504599a..79ec5f36b9e 100644
--- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
@@ -194,7 +194,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     extractor::ProfileProperties *m_profile_properties;
     extractor::Datasources *m_datasources;
 
-    unsigned m_check_sum;
+    std::uint32_t m_check_sum;
     util::vector_view<util::Coordinate> m_coordinate_list;
     extractor::PackedOSMIDsView m_osmnodeid_list;
     util::vector_view<std::uint32_t> m_lane_description_offsets;
@@ -242,7 +242,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeChecksumPointer(const storage::DataLayout &data_layout, char *memory_block)
     {
-        m_check_sum = *data_layout.GetBlockPtr<unsigned>(memory_block, "/ch/checksum");
+        m_check_sum = *data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/connectivity_checksum");
         util::Log() << "set checksum: " << m_check_sum;
     }
 
@@ -779,7 +779,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
             input_coordinate, bearing, bearing_range, approach);
     }
 
-    unsigned GetCheckSum() const override final { return m_check_sum; }
+    std::uint32_t GetCheckSum() const override final { return m_check_sum; }
 
     GeometryID GetGeometryIndex(const NodeID id) const override final
     {
diff --git a/include/engine/datafacade/datafacade_base.hpp b/include/engine/datafacade/datafacade_base.hpp
index 46d8e335020..ed40533c81e 100644
--- a/include/engine/datafacade/datafacade_base.hpp
+++ b/include/engine/datafacade/datafacade_base.hpp
@@ -53,7 +53,7 @@ class BaseDataFacade
     BaseDataFacade() {}
     virtual ~BaseDataFacade() {}
 
-    virtual unsigned GetCheckSum() const = 0;
+    virtual std::uint32_t GetCheckSum() const = 0;
 
     // node and edge information access
     virtual util::Coordinate GetCoordinateOfNode(const NodeID id) const = 0;
diff --git a/include/engine/datafacade_factory.hpp b/include/engine/datafacade_factory.hpp
index bb313db9040..44e1a1f85c7 100644
--- a/include/engine/datafacade_factory.hpp
+++ b/include/engine/datafacade_factory.hpp
@@ -57,6 +57,13 @@ template <template <typename A> class FacadeT, typename AlgorithmT> class DataFa
         layout.List(exclude_path, std::back_inserter(exclude_prefixes));
         facades.resize(exclude_prefixes.size());
 
+        if (facades.empty())
+        {
+            throw util::exception(std::string("Could not find any metrics for ") +
+                                  routing_algorithms::name<AlgorithmT>() +
+                                  " in the data. Did you load the right dataset?");
+        }
+
         for (const auto &exclude_prefix : exclude_prefixes)
         {
             auto index_begin = exclude_prefix.find_last_of("/");
diff --git a/include/guidance/files.hpp b/include/guidance/files.hpp
index 5fd77479d24..c0d1f54e294 100644
--- a/include/guidance/files.hpp
+++ b/include/guidance/files.hpp
@@ -30,7 +30,8 @@ inline void readTurnData(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, "/common/turn_data", turn_data, connectivity_checksum);
+    reader.ReadInto("/common/connectivity_checksum", connectivity_checksum);
+    serialization::read(reader, "/common/turn_data", turn_data);
 }
 
 // writes .osrm.edges
@@ -46,7 +47,9 @@ inline void writeTurnData(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
     storage::tar::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, "/common/turn_data", turn_data, connectivity_checksum);
+    writer.WriteElementCount64("/common/connectivity_checksum", 1);
+    writer.WriteFrom("/common/connectivity_checksum", connectivity_checksum);
+    serialization::write(writer, "/common/turn_data", turn_data);
 }
 }
 }
diff --git a/include/guidance/serialization.hpp b/include/guidance/serialization.hpp
index 2b8ac66f496..6917cb02e5d 100644
--- a/include/guidance/serialization.hpp
+++ b/include/guidance/serialization.hpp
@@ -19,8 +19,7 @@ namespace serialization
 template <storage::Ownership Ownership>
 inline void read(storage::tar::FileReader &reader,
                  const std::string &name,
-                 guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
-                 std::uint32_t &connectivity_checksum)
+                 guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
 {
     storage::serialization::read(
         reader, name + "/turn_instructions", turn_data_container.turn_instructions);
@@ -32,14 +31,12 @@ inline void read(storage::tar::FileReader &reader,
         reader, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings);
     storage::serialization::read(
         reader, name + "/post_turn_bearings", turn_data_container.post_turn_bearings);
-    reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
 }
 
 template <storage::Ownership Ownership>
 inline void write(storage::tar::FileWriter &writer,
                   const std::string &name,
-                  const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
-                  const std::uint32_t connectivity_checksum)
+                  const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
 {
     storage::serialization::write(
         writer, name + "/turn_instructions", turn_data_container.turn_instructions);
@@ -51,8 +48,6 @@ inline void write(storage::tar::FileWriter &writer,
         writer, name + "/pre_turn_bearings", turn_data_container.pre_turn_bearings);
     storage::serialization::write(
         writer, name + "/post_turn_bearings", turn_data_container.post_turn_bearings);
-    writer.WriteElementCount64(name + "/connectivity_checksum", 1);
-    writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
 }
 }
 }
diff --git a/include/guidance/turn_data_container.hpp b/include/guidance/turn_data_container.hpp
index 2f4b68643f8..ae4ecc1737e 100644
--- a/include/guidance/turn_data_container.hpp
+++ b/include/guidance/turn_data_container.hpp
@@ -26,14 +26,12 @@ namespace serialization
 template <storage::Ownership Ownership>
 void read(storage::tar::FileReader &reader,
           const std::string &name,
-          detail::TurnDataContainerImpl<Ownership> &turn_data,
-          std::uint32_t &connectivity_checksum);
+          detail::TurnDataContainerImpl<Ownership> &turn_data);
 
 template <storage::Ownership Ownership>
 void write(storage::tar::FileWriter &writer,
            const std::string &name,
-           const detail::TurnDataContainerImpl<Ownership> &turn_data,
-           const std::uint32_t connectivity_checksum);
+           const detail::TurnDataContainerImpl<Ownership> &turn_data);
 }
 
 struct TurnData
@@ -101,12 +99,10 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
 
     friend void serialization::read<Ownership>(storage::tar::FileReader &reader,
                                                const std::string &name,
-                                               TurnDataContainerImpl &turn_data_container,
-                                               std::uint32_t &connectivity_checksum);
+                                               TurnDataContainerImpl &turn_data_container);
     friend void serialization::write<Ownership>(storage::tar::FileWriter &writer,
                                                 const std::string &name,
-                                                const TurnDataContainerImpl &turn_data_container,
-                                                const std::uint32_t connectivity_checksum);
+                                                const TurnDataContainerImpl &turn_data_container);
 
   private:
     Vector<TurnInstruction> turn_instructions;
diff --git a/include/partitioner/files.hpp b/include/partitioner/files.hpp
index 0414ee3c83c..3a85a8fb0f7 100644
--- a/include/partitioner/files.hpp
+++ b/include/partitioner/files.hpp
@@ -26,7 +26,8 @@ inline void readGraph(const boost::filesystem::path &path,
 
     storage::tar::FileReader reader{path, storage::tar::FileReader::VerifyFingerprint};
 
-    serialization::read(reader, "/mld/multilevelgraph", graph, connectivity_checksum);
+    reader.ReadInto("/mld/connectivity_checksum", connectivity_checksum);
+    serialization::read(reader, "/mld/multilevelgraph", graph);
 }
 
 // writes .osrm.mldgr file
@@ -41,7 +42,9 @@ inline void writeGraph(const boost::filesystem::path &path,
 
     storage::tar::FileWriter writer{path, storage::tar::FileWriter::GenerateFingerprint};
 
-    serialization::write(writer, "/mld/multilevelgraph", graph, connectivity_checksum);
+    writer.WriteElementCount64("/mld/connectivity_checksum", 1);
+    writer.WriteFrom("/mld/connectivity_checksum", connectivity_checksum);
+    serialization::write(writer, "/mld/multilevelgraph", graph);
 }
 
 // read .osrm.partition file
diff --git a/include/partitioner/multi_level_graph.hpp b/include/partitioner/multi_level_graph.hpp
index 4b952b4608e..2c8c7204cba 100644
--- a/include/partitioner/multi_level_graph.hpp
+++ b/include/partitioner/multi_level_graph.hpp
@@ -26,14 +26,12 @@ namespace serialization
 template <typename EdgeDataT, storage::Ownership Ownership>
 void read(storage::tar::FileReader &reader,
           const std::string &name,
-          MultiLevelGraph<EdgeDataT, Ownership> &graph,
-          std::uint32_t &connectivity_checksum);
+          MultiLevelGraph<EdgeDataT, Ownership> &graph);
 
 template <typename EdgeDataT, storage::Ownership Ownership>
 void write(storage::tar::FileWriter &writer,
            const std::string &name,
-           const MultiLevelGraph<EdgeDataT, Ownership> &graph,
-           const std::uint32_t connectivity_checksum);
+           const MultiLevelGraph<EdgeDataT, Ownership> &graph);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
@@ -206,13 +204,11 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
     friend void
     serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
                                               const std::string &name,
-                                              MultiLevelGraph<EdgeDataT, Ownership> &graph,
-                                              std::uint32_t &connectivity_checksum);
+                                              MultiLevelGraph<EdgeDataT, Ownership> &graph);
     friend void
     serialization::write<EdgeDataT, Ownership>(storage::tar::FileWriter &writer,
                                                const std::string &name,
-                                               const MultiLevelGraph<EdgeDataT, Ownership> &graph,
-                                               const std::uint32_t connectivity_checksum);
+                                               const MultiLevelGraph<EdgeDataT, Ownership> &graph);
 
     Vector<EdgeOffset> node_to_edge_offset;
     std::uint32_t connectivity_checksum;
diff --git a/include/partitioner/serialization.hpp b/include/partitioner/serialization.hpp
index 4fefefa7b7a..b8d57bb6dbc 100644
--- a/include/partitioner/serialization.hpp
+++ b/include/partitioner/serialization.hpp
@@ -22,26 +22,21 @@ namespace serialization
 template <typename EdgeDataT, storage::Ownership Ownership>
 inline void read(storage::tar::FileReader &reader,
                  const std::string &name,
-                 MultiLevelGraph<EdgeDataT, Ownership> &graph,
-                 std::uint32_t &connectivity_checksum)
+                 MultiLevelGraph<EdgeDataT, Ownership> &graph)
 {
     storage::serialization::read(reader, name + "/node_array", graph.node_array);
     storage::serialization::read(reader, name + "/edge_array", graph.edge_array);
     storage::serialization::read(reader, name + "/node_to_edge_offset", graph.node_to_edge_offset);
-    reader.ReadInto(name + "/connectivity_checksum", connectivity_checksum);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
 inline void write(storage::tar::FileWriter &writer,
                   const std::string &name,
-                  const MultiLevelGraph<EdgeDataT, Ownership> &graph,
-                  const std::uint32_t connectivity_checksum)
+                  const MultiLevelGraph<EdgeDataT, Ownership> &graph)
 {
     storage::serialization::write(writer, name + "/node_array", graph.node_array);
     storage::serialization::write(writer, name + "/edge_array", graph.edge_array);
     storage::serialization::write(writer, name + "/node_to_edge_offset", graph.node_to_edge_offset);
-    writer.WriteElementCount64(name + "/connectivity_checksum", 1);
-    writer.WriteFrom(name + "/connectivity_checksum", connectivity_checksum);
 }
 
 template <storage::Ownership Ownership>
diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp
index 951a4420fef..416f4a62b06 100644
--- a/src/contractor/contractor.cpp
+++ b/src/contractor/contractor.cpp
@@ -93,9 +93,6 @@ int Contractor::Run()
             util::excludeFlagsToNodeFilter(number_of_edge_based_nodes, node_data, properties);
     }
 
-    RangebasedCRC32 crc32_calculator;
-    const unsigned checksum = crc32_calculator(edge_based_edge_list);
-
     QueryGraph query_graph;
     std::vector<std::vector<bool>> edge_filters;
     std::vector<std::vector<bool>> cores;
@@ -110,7 +107,7 @@ int Contractor::Run()
     std::unordered_map<std::string, ContractedMetric> metrics = {
         {metric_name, {std::move(query_graph), std::move(edge_filters)}}};
 
-    files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, metrics, connectivity_checksum);
+    files::writeGraph(config.GetPath(".osrm.hsgr"), metrics, connectivity_checksum);
 
     TIMER_STOP(preparing);
 
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 099dde34e3c..8b73b658f51 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -405,9 +405,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
                                          std::move(entry_class_ids),
                                          std::move(pre_turn_bearings),
                                          std::move(post_turn_bearings));
+        auto connectivity_checksum_ptr = layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/connectivity_checksum");
 
         guidance::files::readTurnData(
-            config.GetPath(".osrm.edges"), turn_data, turns_connectivity_checksum);
+            config.GetPath(".osrm.edges"), turn_data, *connectivity_checksum_ptr);
+
+        turns_connectivity_checksum = *connectivity_checksum_ptr;
     }
 
     // load compressed geometry
@@ -613,7 +616,6 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             memory_ptr, metric_prefix + "/contracted_graph/node_array");
         auto graph_edges_ptr = layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
             memory_ptr, metric_prefix + "/contracted_graph/edge_array");
-        auto checksum = layout.GetBlockPtr<unsigned, true>(memory_ptr, "/ch/checksum");
 
         util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
             graph_nodes_ptr,
@@ -636,7 +638,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             {metric_name, {{std::move(node_list), std::move(edge_list)}, std::move(edge_filter)}}};
 
         contractor::files::readGraph(
-            config.GetPath(".osrm.hsgr"), *checksum, metrics, graph_connectivity_checksum);
+            config.GetPath(".osrm.hsgr"), metrics, graph_connectivity_checksum);
         if (turns_connectivity_checksum != graph_connectivity_checksum)
         {
             throw util::exception(
diff --git a/unit_tests/mocks/mock_datafacade.hpp b/unit_tests/mocks/mock_datafacade.hpp
index 830dc41386a..f618730af0f 100644
--- a/unit_tests/mocks/mock_datafacade.hpp
+++ b/unit_tests/mocks/mock_datafacade.hpp
@@ -197,7 +197,7 @@ class MockBaseDataFacade : public engine::datafacade::BaseDataFacade
         return {};
     }
 
-    unsigned GetCheckSum() const override { return 0; }
+    std::uint32_t GetCheckSum() const override { return 0; }
 
     extractor::TravelMode GetTravelMode(const NodeID /* id */) const override
     {

From f8148845452c00d70210f29c81fcffd1292ba536 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Wed, 28 Mar 2018 23:18:28 +0000
Subject: [PATCH 10/19] Fix formating

---
 include/contractor/contracted_metric.hpp      |  3 +-
 include/contractor/files.hpp                  |  1 -
 include/customizer/files.hpp                  | 20 ++++---
 .../contiguous_internalmem_datafacade.hpp     | 59 +++++++++++--------
 include/storage/io.hpp                        | 13 ++--
 include/storage/serialization.hpp             | 21 ++-----
 include/storage/shared_datatype.hpp           |  5 +-
 .../datafacade/shared_memory_allocator.cpp    |  8 +--
 src/storage/storage.cpp                       |  3 +-
 unit_tests/contractor/files.cpp               |  4 +-
 unit_tests/storage/data_layout.cpp            | 12 +++-
 unit_tests/storage/serialization.cpp          | 12 ++--
 12 files changed, 80 insertions(+), 81 deletions(-)

diff --git a/include/contractor/contracted_metric.hpp b/include/contractor/contracted_metric.hpp
index e8592b32ad6..f44f49e8320 100644
--- a/include/contractor/contracted_metric.hpp
+++ b/include/contractor/contracted_metric.hpp
@@ -10,8 +10,7 @@ namespace contractor
 
 namespace detail
 {
-template<storage::Ownership Ownership>
-struct ContractedMetric
+template <storage::Ownership Ownership> struct ContractedMetric
 {
     detail::QueryGraph<Ownership> graph;
     std::vector<util::ViewOrVector<bool, Ownership>> edge_filter;
diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp
index 7ba99442329..8dcade795d2 100644
--- a/include/contractor/files.hpp
+++ b/include/contractor/files.hpp
@@ -30,7 +30,6 @@ inline void readGraph(const boost::filesystem::path &path,
     {
         serialization::read(reader, "/ch/metrics/" + pair.first, pair.second);
     }
-
 }
 
 // writes .osrm.hsgr file
diff --git a/include/customizer/files.hpp b/include/customizer/files.hpp
index 8b690ecd27c..c89b5cb7607 100644
--- a/include/customizer/files.hpp
+++ b/include/customizer/files.hpp
@@ -18,7 +18,8 @@ namespace files
 
 // reads .osrm.cell_metrics file
 template <typename CellMetricT>
-inline void readCellMetrics(const boost::filesystem::path &path, std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
+inline void readCellMetrics(const boost::filesystem::path &path,
+                            std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
 {
     static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
                       std::is_same<CellMetric, CellMetricT>::value,
@@ -27,10 +28,10 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::unordered_
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
-    for (auto& pair : metrics)
+    for (auto &pair : metrics)
     {
-        const auto& metric_name = pair.first;
-        auto& metric_exclude_classes = pair.second;
+        const auto &metric_name = pair.first;
+        auto &metric_exclude_classes = pair.second;
 
         auto prefix = "/mld/metrics/" + metric_name + "/exclude";
         auto num_exclude_classes = reader.ReadElementCount64(prefix);
@@ -46,8 +47,9 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::unordered_
 
 // writes .osrm.cell_metrics file
 template <typename CellMetricT>
-inline void writeCellMetrics(const boost::filesystem::path &path,
-                             const std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
+inline void
+writeCellMetrics(const boost::filesystem::path &path,
+                 const std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
 {
     static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
                       std::is_same<CellMetric, CellMetricT>::value,
@@ -56,10 +58,10 @@ inline void writeCellMetrics(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
     storage::tar::FileWriter writer{path, fingerprint};
 
-    for (const auto& pair : metrics)
+    for (const auto &pair : metrics)
     {
-        const auto& metric_name = pair.first;
-        const auto& metric_exclude_classes = pair.second;
+        const auto &metric_name = pair.first;
+        const auto &metric_exclude_classes = pair.second;
 
         auto prefix = "/mld/metrics/" + metric_name + "/exclude";
         writer.WriteElementCount64(prefix, metric_exclude_classes.size());
diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
index 79ec5f36b9e..ec3a4491bbc 100644
--- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
@@ -83,15 +83,15 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
 
     void InitializeGraphPointer(const storage::DataLayout &data_layout,
                                 char *memory_block,
-                                const std::string& metric_name,
+                                const std::string &metric_name,
                                 const std::size_t exclude_index)
     {
         const std::string metric_prefix = "/ch/metrics/" + metric_name;
-        auto graph_nodes_ptr =
-            data_layout.GetBlockPtr<GraphNode>(memory_block, metric_prefix + "/contracted_graph/node_array");
+        auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
+            memory_block, metric_prefix + "/contracted_graph/node_array");
 
-        auto graph_edges_ptr =
-            data_layout.GetBlockPtr<GraphEdge>(memory_block, metric_prefix + "/contracted_graph/edge_array");
+        auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
+            memory_block, metric_prefix + "/contracted_graph/edge_array");
 
         auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index);
         auto filter_block_id = exclude_prefix + "/edge_filter";
@@ -100,9 +100,11 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
             data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
 
         util::vector_view<GraphNode> node_list(
-            graph_nodes_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
+            graph_nodes_ptr,
+            data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
         util::vector_view<GraphEdge> edge_list(
-            graph_edges_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
+            graph_edges_ptr,
+            data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
 
         util::vector_view<bool> edge_filter(edge_filter_ptr,
                                             data_layout.GetBlockEntries(filter_block_id));
@@ -111,15 +113,18 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
 
   public:
     ContiguousInternalMemoryAlgorithmDataFacade(
-        std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, std::size_t exclude_index)
+        std::shared_ptr<ContiguousBlockAllocator> allocator_,
+        const std::string &metric_name,
+        std::size_t exclude_index)
         : allocator(std::move(allocator_))
     {
-        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
+        InitializeInternalPointers(
+            allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
     }
 
     void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
-                                    const std::string& metric_name,
+                                    const std::string &metric_name,
                                     const std::size_t exclude_index)
     {
         InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index);
@@ -229,11 +234,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
                                             char *memory_block,
-                                            const std::string& metric_name,
+                                            const std::string &metric_name,
                                             const std::size_t exclude_index)
     {
         // TODO: For multi-metric support we need to have separate exclude classes per metric
-        (void) metric_name;
+        (void)metric_name;
         m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
             memory_block, "/common/properties");
 
@@ -242,7 +247,8 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeChecksumPointer(const storage::DataLayout &data_layout, char *memory_block)
     {
-        m_check_sum = *data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/connectivity_checksum");
+        m_check_sum =
+            *data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/connectivity_checksum");
         util::Log() << "set checksum: " << m_check_sum;
     }
 
@@ -535,7 +541,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
-                                    const std::string& metric_name,
+                                    const std::string &metric_name,
                                     const std::size_t exclude_index)
     {
         InitializeChecksumPointer(data_layout, memory_block);
@@ -556,11 +562,12 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     // allows switching between process_memory/shared_memory datafacade, based on the type of
     // allocator
     ContiguousInternalMemoryDataFacadeBase(std::shared_ptr<ContiguousBlockAllocator> allocator_,
-                                            const std::string& metric_name,
+                                           const std::string &metric_name,
                                            const std::size_t exclude_index)
         : allocator(std::move(allocator_))
     {
-        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
+        InitializeInternalPointers(
+            allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
     }
 
     // node and edge information access
@@ -946,7 +953,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         auto found_range = std::equal_range(
             m_maneuver_overrides.begin(), m_maneuver_overrides.end(), edge_based_node_id, Comp{});
 
-        std::for_each(found_range.first, found_range.second, [&](const auto &override) {
+        std::for_each(found_range.first, found_range.second, [&](const auto & override) {
             std::vector<NodeID> sequence(
                 m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_begin,
                 m_maneuver_override_node_sequences.begin() + override.node_sequence_offset_end);
@@ -968,7 +975,7 @@ class ContiguousInternalMemoryDataFacade<CH>
 {
   public:
     ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
-            const std::string& metric_name,
+                                       const std::string &metric_name,
                                        const std::size_t exclude_index)
         : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
           ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, metric_name, exclude_index)
@@ -991,7 +998,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
 
     void InitializeInternalPointers(const storage::DataLayout &data_layout,
                                     char *memory_block,
-                                    const std::string& metric_name,
+                                    const std::string &metric_name,
                                     const std::size_t exclude_index)
     {
         InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index);
@@ -1000,7 +1007,7 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
 
     void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
                                    char *memory_block,
-                                   const std::string& metric_name,
+                                   const std::string &metric_name,
                                    const std::size_t exclude_index)
     {
         if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
@@ -1028,7 +1035,8 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
                 partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
         }
 
-        const auto exclude_prefix = "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index);
+        const auto exclude_prefix =
+            "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index);
         const auto weights_block_id = exclude_prefix + "/weights";
         const auto durations_block_id = exclude_prefix + "/durations";
 
@@ -1111,10 +1119,13 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
 
   public:
     ContiguousInternalMemoryAlgorithmDataFacade(
-        std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, const std::size_t exclude_index)
+        std::shared_ptr<ContiguousBlockAllocator> allocator_,
+        const std::string &metric_name,
+        const std::size_t exclude_index)
         : allocator(std::move(allocator_))
     {
-        InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
+        InitializeInternalPointers(
+            allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
     }
 
     const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override
@@ -1168,7 +1179,7 @@ class ContiguousInternalMemoryDataFacade<MLD> final
   private:
   public:
     ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
-                                       const std::string& metric_name,
+                                       const std::string &metric_name,
                                        const std::size_t exclude_index)
         : ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
           ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, metric_name, exclude_index)
diff --git a/include/storage/io.hpp b/include/storage/io.hpp
index fe2351ca3d1..7702c0ddb77 100644
--- a/include/storage/io.hpp
+++ b/include/storage/io.hpp
@@ -11,9 +11,9 @@
 #include "util/version.hpp"
 
 #include <boost/filesystem/fstream.hpp>
+#include <boost/iostreams/device/array.hpp>
 #include <boost/iostreams/seek.hpp>
 #include <boost/iostreams/stream.hpp>
-#include <boost/iostreams/device/array.hpp>
 
 #include <cerrno>
 #include <cstring>
@@ -296,11 +296,10 @@ class FileWriter
 class BufferReader
 {
   public:
-    BufferReader(const std::string &buffer) : BufferReader(buffer.data(), buffer.size())
-    {
-    }
+    BufferReader(const std::string &buffer) : BufferReader(buffer.data(), buffer.size()) {}
 
-    BufferReader(const char* buffer, const std::size_t size) : input_stream(boost::iostreams::array_source(buffer, size))
+    BufferReader(const char *buffer, const std::size_t size)
+        : input_stream(boost::iostreams::array_source(buffer, size))
     {
         if (!input_stream)
         {
@@ -309,9 +308,7 @@ class BufferReader
         }
     }
 
-    std::size_t GetPosition() {
-        return input_stream.tellg();
-    }
+    std::size_t GetPosition() { return input_stream.tellg(); }
 
     template <typename T> void ReadInto(T *dest, const std::size_t count)
     {
diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp
index ebc4d864c81..af0d3ec7716 100644
--- a/include/storage/serialization.hpp
+++ b/include/storage/serialization.hpp
@@ -6,8 +6,8 @@
 #include "util/vector_view.hpp"
 
 #include "storage/io.hpp"
-#include "storage/tar.hpp"
 #include "storage/shared_datatype.hpp"
+#include "storage/tar.hpp"
 
 #include <boost/function_output_iterator.hpp>
 #include <boost/iterator/function_input_iterator.hpp>
@@ -92,7 +92,6 @@ void writeBoolVector(tar::FileWriter &writer, const std::string &name, const Vec
 }
 }
 
-
 /* All vector formats here use the same on-disk format.
  * This is important because we want to be able to write from a vector
  * of one kind, but read it into a vector of another kind.
@@ -149,17 +148,12 @@ template <typename T> void write(io::BufferWriter &writer, const std::vector<T>
     writer.WriteFrom(data.data(), count);
 }
 
-template<typename T>
-inline void write(io::BufferWriter &writer, const T &data)
+template <typename T> inline void write(io::BufferWriter &writer, const T &data)
 {
     writer.WriteFrom(data);
 }
 
-template<typename T>
-inline void read(io::BufferReader &reader, T &data)
-{
-    reader.ReadInto(data);
-}
+template <typename T> inline void read(io::BufferReader &reader, T &data) { reader.ReadInto(data); }
 
 inline void write(io::BufferWriter &writer, const std::string &data)
 {
@@ -254,7 +248,7 @@ template <typename K, typename V> void read(io::BufferReader &reader, std::map<K
     std::pair<K, V> pair;
     for (auto index : util::irange<std::size_t>(0, count))
     {
-        (void) index;
+        (void)index;
         read(reader, pair.first);
         read(reader, pair.second);
         data.insert(pair);
@@ -272,17 +266,12 @@ template <typename K, typename V> void write(io::BufferWriter &writer, const std
     }
 }
 
-inline void read(io::BufferReader &reader, DataLayout &layout)
-{
-    read(reader, layout.blocks);
-}
+inline void read(io::BufferReader &reader, DataLayout &layout) { read(reader, layout.blocks); }
 
 inline void write(io::BufferWriter &writer, const DataLayout &layout)
 {
     write(writer, layout.blocks);
 }
-
-
 }
 }
 }
diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index c00ece841e7..041457a7f9c 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -71,7 +71,10 @@ class DataLayout
 
     inline uint64_t GetBlockSize(const std::string &name) const { return GetBlock(name).byte_size; }
 
-    inline bool HasBlock(const std::string &name) const { return blocks.find(name) != blocks.end(); }
+    inline bool HasBlock(const std::string &name) const
+    {
+        return blocks.find(name) != blocks.end();
+    }
 
     inline uint64_t GetSizeOfLayout() const
     {
diff --git a/src/engine/datafacade/shared_memory_allocator.cpp b/src/engine/datafacade/shared_memory_allocator.cpp
index e3ef0f0c1d0..d6d8be0d2f0 100644
--- a/src/engine/datafacade/shared_memory_allocator.cpp
+++ b/src/engine/datafacade/shared_memory_allocator.cpp
@@ -20,7 +20,8 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
     BOOST_ASSERT(storage::SharedMemory::RegionExists(data_region));
     m_large_memory = storage::makeSharedMemory(data_region);
 
-    storage::io::BufferReader reader(reinterpret_cast<char *>(m_large_memory->Ptr()), m_large_memory->Size());
+    storage::io::BufferReader reader(reinterpret_cast<char *>(m_large_memory->Ptr()),
+                                     m_large_memory->Size());
     storage::serialization::read(reader, data_layout);
     layout_size = reader.GetPosition();
     util::Log(logDEBUG) << "Data layout has size " << layout_size;
@@ -28,10 +29,7 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
 
 SharedMemoryAllocator::~SharedMemoryAllocator() {}
 
-const storage::DataLayout &SharedMemoryAllocator::GetLayout()
-{
-    return data_layout;
-}
+const storage::DataLayout &SharedMemoryAllocator::GetLayout() { return data_layout; }
 char *SharedMemoryAllocator::GetMemory()
 {
     return reinterpret_cast<char *>(m_large_memory->Ptr()) + layout_size;
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 8b73b658f51..ad29febfc64 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -405,7 +405,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
                                          std::move(entry_class_ids),
                                          std::move(pre_turn_bearings),
                                          std::move(post_turn_bearings));
-        auto connectivity_checksum_ptr = layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/connectivity_checksum");
+        auto connectivity_checksum_ptr =
+            layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/connectivity_checksum");
 
         guidance::files::readTurnData(
             config.GetPath(".osrm.edges"), turn_data, *connectivity_checksum_ptr);
diff --git a/unit_tests/contractor/files.cpp b/unit_tests/contractor/files.cpp
index b3898c173e8..ffb9a455bd8 100644
--- a/unit_tests/contractor/files.cpp
+++ b/unit_tests/contractor/files.cpp
@@ -42,9 +42,7 @@ BOOST_AUTO_TEST_CASE(read_write_hsgr)
     unsigned checksum;
     unsigned connectivity_checksum;
 
-    std::unordered_map<std::string, ContractedMetric> metrics = {
-        {"duration", {}}
-    };
+    std::unordered_map<std::string, ContractedMetric> metrics = {{"duration", {}}};
     contractor::files::readGraph(tmp.path, checksum, metrics, connectivity_checksum);
 
     BOOST_CHECK_EQUAL(checksum, reference_checksum);
diff --git a/unit_tests/storage/data_layout.cpp b/unit_tests/storage/data_layout.cpp
index b97a2020650..5335ff7bf63 100644
--- a/unit_tests/storage/data_layout.cpp
+++ b/unit_tests/storage/data_layout.cpp
@@ -93,10 +93,16 @@ BOOST_AUTO_TEST_CASE(layout_list_test)
     layout.List("/mld/metrics/", std::back_inserter(results_5));
     layout.List("/mld/", std::back_inserter(results_6));
 
-    CHECK_EQUAL_RANGE(results_1, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
-    CHECK_EQUAL_RANGE(results_2, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
+    CHECK_EQUAL_RANGE(
+        results_1, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
+    CHECK_EQUAL_RANGE(
+        results_2, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
     CHECK_EQUAL_RANGE(results_3, "/ch/edge_filter");
-    CHECK_EQUAL_RANGE(results_4, "/mld/metrics/0/durations", "/mld/metrics/0/weights", "/mld/metrics/1/durations", "/mld/metrics/1/weights");
+    CHECK_EQUAL_RANGE(results_4,
+                      "/mld/metrics/0/durations",
+                      "/mld/metrics/0/weights",
+                      "/mld/metrics/1/durations",
+                      "/mld/metrics/1/weights");
     CHECK_EQUAL_RANGE(results_5, "/mld/metrics/0", "/mld/metrics/1");
     CHECK_EQUAL_RANGE(results_6, "/mld/metrics");
 }
diff --git a/unit_tests/storage/serialization.cpp b/unit_tests/storage/serialization.cpp
index e8153a92f6a..e42f8ea8e5c 100644
--- a/unit_tests/storage/serialization.cpp
+++ b/unit_tests/storage/serialization.cpp
@@ -118,18 +118,14 @@ BOOST_AUTO_TEST_CASE(tar_serialize_deallocting_vector)
 BOOST_AUTO_TEST_CASE(buffer_serialize_map)
 {
     std::map<std::string, std::int32_t> map = {
-        {"foo", 1},
-        {"barrrr", 2},
-        {"bal", 3},
-        {"bazbar", 4},
-        {"foofofofo", 5},
+        {"foo", 1}, {"barrrr", 2}, {"bal", 3}, {"bazbar", 4}, {"foofofofo", 5},
     };
 
     std::string buffer;
     {
-    io::BufferWriter writer;
-    storage::serialization::write(writer, map);
-    buffer = writer.GetBuffer();
+        io::BufferWriter writer;
+        storage::serialization::write(writer, map);
+        buffer = writer.GetBuffer();
     }
 
     std::map<std::string, std::int32_t> result;

From f583d9b159cd7a6b584db7dcb96374e6f51b7232 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Thu, 29 Mar 2018 10:19:22 +0000
Subject: [PATCH 11/19] Fix unit-test compilation due to broken readGraph

---
 unit_tests/contractor/files.cpp | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/unit_tests/contractor/files.cpp b/unit_tests/contractor/files.cpp
index ffb9a455bd8..e4c037f8795 100644
--- a/unit_tests/contractor/files.cpp
+++ b/unit_tests/contractor/files.cpp
@@ -15,7 +15,6 @@ using namespace osrm::unit_test;
 
 BOOST_AUTO_TEST_CASE(read_write_hsgr)
 {
-    auto reference_checksum = 0xFF00FF00;
     auto reference_connectivity_checksum = 0xDEADBEEF;
     std::vector<TestEdge> edges = {TestEdge{0, 1, 3},
                                    TestEdge{0, 5, 1},
@@ -36,16 +35,13 @@ BOOST_AUTO_TEST_CASE(read_write_hsgr)
         {"duration", {std::move(reference_graph), std::move(reference_filters)}}};
 
     TemporaryFile tmp{TEST_DATA_DIR "/read_write_hsgr_test.osrm.hsgr"};
-    contractor::files::writeGraph(
-        tmp.path, reference_checksum, reference_metrics, reference_connectivity_checksum);
+    contractor::files::writeGraph(tmp.path, reference_metrics, reference_connectivity_checksum);
 
-    unsigned checksum;
     unsigned connectivity_checksum;
 
     std::unordered_map<std::string, ContractedMetric> metrics = {{"duration", {}}};
-    contractor::files::readGraph(tmp.path, checksum, metrics, connectivity_checksum);
+    contractor::files::readGraph(tmp.path, metrics, connectivity_checksum);
 
-    BOOST_CHECK_EQUAL(checksum, reference_checksum);
     BOOST_CHECK_EQUAL(connectivity_checksum, reference_connectivity_checksum);
     BOOST_CHECK_EQUAL(metrics["duration"].edge_filter.size(),
                       reference_metrics["duration"].edge_filter.size());

From 79fda85a6743305508391ab0fad0f4890241999f Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Thu, 29 Mar 2018 12:36:50 +0000
Subject: [PATCH 12/19] First step for better vector encapsulation

---
 include/storage/shared_datatype.hpp | 13 ++++++++
 src/storage/storage.cpp             | 52 ++++++++---------------------
 2 files changed, 27 insertions(+), 38 deletions(-)

diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index 041457a7f9c..73162619339 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -4,6 +4,7 @@
 #include "storage/block.hpp"
 #include "storage/io_fwd.hpp"
 
+#include "util/vector_view.hpp"
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/log.hpp"
@@ -86,6 +87,18 @@ class DataLayout
         return result;
     }
 
+    template <typename T>
+    util::vector_view<T> GetVector(char* shared_memory, const std::string& name) const
+    {
+        return util::vector_view<T>(GetBlockPtr<T>(shared_memory, name), GetBlockEntries(name));
+    }
+
+    template <typename T>
+    util::vector_view<T> GetWritableVector(char* shared_memory, const std::string& name) const
+    {
+        return util::vector_view<T>(GetBlockPtr<T, true>(shared_memory, name), GetBlockEntries(name));
+    }
+
     template <typename T, bool WRITE_CANARY = false>
     inline T *GetBlockPtr(char *shared_memory, const std::string &name) const
     {
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index ad29febfc64..a22bfed7a82 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -311,17 +311,11 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Name data
     {
-        const auto name_blocks_ptr =
-            layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference, true>(
+        auto blocks =
+            layout.GetWritableVector<extractor::NameTableView::IndexedData::BlockReference>(
                 memory_ptr, "/common/names/blocks");
-        const auto name_values_ptr =
-            layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType, true>(
-                memory_ptr, "/common/names/values");
-
-        util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
-            name_blocks_ptr, layout.GetBlockEntries("/common/names/blocks"));
-        util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
-            name_values_ptr, layout.GetBlockEntries("/common/names/values"));
+        auto values = layout.GetWritableVector<extractor::NameTableView::IndexedData::ValueType>(
+            memory_ptr, "/common/names/values");
 
         extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
         extractor::NameTableView name_table{index_data_view};
@@ -330,41 +324,28 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Turn lane data
     {
-        const auto turn_lane_data_ptr = layout.GetBlockPtr<util::guidance::LaneTupleIdPair, true>(
+        auto turn_lane_data = layout.GetWritableVector<util::guidance::LaneTupleIdPair>(
             memory_ptr, "/common/turn_lanes/data");
-        util::vector_view<util::guidance::LaneTupleIdPair> turn_lane_data(
-            turn_lane_data_ptr, layout.GetBlockEntries("/common/turn_lanes/data"));
 
         extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
     }
 
     // Turn lane descriptions
     {
-        auto offsets_ptr =
-            layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/turn_lanes/offsets");
-        util::vector_view<std::uint32_t> offsets(
-            offsets_ptr, layout.GetBlockEntries("/common/turn_lanes/offsets"));
-
-        auto masks_ptr = layout.GetBlockPtr<extractor::TurnLaneType::Mask, true>(
+        auto offsets =
+            layout.GetWritableVector<std::uint32_t>(memory_ptr, "/common/turn_lanes/offsets");
+        auto masks = layout.GetWritableVector<extractor::TurnLaneType::Mask>(
             memory_ptr, "/common/turn_lanes/masks");
-        util::vector_view<extractor::TurnLaneType::Mask> masks(
-            masks_ptr, layout.GetBlockEntries("/common/turn_lanes/masks"));
 
         extractor::files::readTurnLaneDescriptions(config.GetPath(".osrm.tls"), offsets, masks);
     }
 
     // Load edge-based nodes data
     {
-        auto edge_based_node_data_list_ptr = layout.GetBlockPtr<extractor::EdgeBasedNode, true>(
+        auto edge_based_node_data = layout.GetWritableVector<extractor::EdgeBasedNode>(
             memory_ptr, "/common/ebg_node_data/nodes");
-        util::vector_view<extractor::EdgeBasedNode> edge_based_node_data(
-            edge_based_node_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes"));
-
-        auto annotation_data_list_ptr =
-            layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation, true>(
-                memory_ptr, "/common/ebg_node_data/annotations");
-        util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data(
-            annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations"));
+        auto  annotation_data = layout.GetWritableVector<extractor::NodeBasedEdgeAnnotation>(
+            memory_ptr, "/common/ebg_node_data/annotations");
 
         extractor::EdgeBasedNodeDataView node_data(std::move(edge_based_node_data),
                                                    std::move(annotation_data));
@@ -374,16 +355,11 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Load original edge data
     {
-        const auto lane_data_id_ptr =
-            layout.GetBlockPtr<LaneDataID, true>(memory_ptr, "/common/turn_data/lane_data_ids");
-        util::vector_view<LaneDataID> lane_data_ids(
-            lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids"));
+        auto lane_data_ids = layout.GetWritableVector<LaneDataID>(
+            memory_ptr, "/common/turn_data/lane_data_ids");
 
-        const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction, true>(
+        const auto turn_instructions = layout.GetWritableVector<guidance::TurnInstruction>(
             memory_ptr, "/common/turn_data/turn_instructions");
-        util::vector_view<guidance::TurnInstruction> turn_instructions(
-            turn_instruction_list_ptr,
-            layout.GetBlockEntries("/common/turn_data/turn_instructions"));
 
         const auto entry_class_id_list_ptr =
             layout.GetBlockPtr<EntryClassID, true>(memory_ptr, "/common/turn_data/entry_class_ids");

From abafa7689f0382348ec22f1c53776e867c009bfc Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Thu, 29 Mar 2018 16:05:02 +0000
Subject: [PATCH 13/19] Refactor setting up vector_view

---
 include/storage/shared_datatype.hpp |  18 +-
 include/util/range_table.hpp        |  10 +-
 src/storage/storage.cpp             | 293 +++++++++-------------------
 3 files changed, 106 insertions(+), 215 deletions(-)

diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index 73162619339..61136a13c0a 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -4,10 +4,10 @@
 #include "storage/block.hpp"
 #include "storage/io_fwd.hpp"
 
-#include "util/vector_view.hpp"
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/log.hpp"
+#include "util/vector_view.hpp"
 
 #include <boost/assert.hpp>
 
@@ -88,15 +88,16 @@ class DataLayout
     }
 
     template <typename T>
-    util::vector_view<T> GetVector(char* shared_memory, const std::string& name) const
+    util::vector_view<T> GetVector(char *shared_memory, const std::string &name) const
     {
         return util::vector_view<T>(GetBlockPtr<T>(shared_memory, name), GetBlockEntries(name));
     }
 
     template <typename T>
-    util::vector_view<T> GetWritableVector(char* shared_memory, const std::string& name) const
+    util::vector_view<T> GetWritableVector(char *shared_memory, const std::string &name) const
     {
-        return util::vector_view<T>(GetBlockPtr<T, true>(shared_memory, name), GetBlockEntries(name));
+        return util::vector_view<T>(GetBlockPtr<T, true>(shared_memory, name),
+                                    GetBlockEntries(name));
     }
 
     template <typename T, bool WRITE_CANARY = false>
@@ -204,6 +205,15 @@ class DataLayout
     std::map<std::string, Block> blocks;
 };
 
+template <>
+inline util::vector_view<bool> DataLayout::GetWritableVector<bool>(char *shared_memory,
+                                                                   const std::string &name) const
+{
+    return util::vector_view<bool>(
+        GetBlockPtr<util::vector_view<bool>::Word, true>(shared_memory, name),
+        GetBlockEntries(name));
+}
+
 enum SharedDataType
 {
     REGION_NONE,
diff --git a/include/util/range_table.hpp b/include/util/range_table.hpp
index cceee671378..fcb732b2445 100644
--- a/include/util/range_table.hpp
+++ b/include/util/range_table.hpp
@@ -55,14 +55,12 @@ template <unsigned BLOCK_SIZE, storage::Ownership Ownership> class RangeTable
     RangeTable() : sum_lengths(0) {}
 
     // for loading from shared memory
-    explicit RangeTable(OffsetContainerT &external_offsets,
-                        BlockContainerT &external_blocks,
+    explicit RangeTable(OffsetContainerT offsets_,
+                        BlockContainerT blocks_,
                         const unsigned sum_lengths)
-        : sum_lengths(sum_lengths)
+        : block_offsets(std::move(offsets_)), diff_blocks(std::move(blocks_)),
+          sum_lengths(sum_lengths)
     {
-        using std::swap;
-        swap(block_offsets, external_offsets);
-        swap(diff_blocks, external_blocks);
     }
 
     // construct table from length vector
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index a22bfed7a82..08320efb322 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -344,7 +344,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     {
         auto edge_based_node_data = layout.GetWritableVector<extractor::EdgeBasedNode>(
             memory_ptr, "/common/ebg_node_data/nodes");
-        auto  annotation_data = layout.GetWritableVector<extractor::NodeBasedEdgeAnnotation>(
+        auto annotation_data = layout.GetWritableVector<extractor::NodeBasedEdgeAnnotation>(
             memory_ptr, "/common/ebg_node_data/annotations");
 
         extractor::EdgeBasedNodeDataView node_data(std::move(edge_based_node_data),
@@ -355,26 +355,20 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Load original edge data
     {
-        auto lane_data_ids = layout.GetWritableVector<LaneDataID>(
-            memory_ptr, "/common/turn_data/lane_data_ids");
+        auto lane_data_ids =
+            layout.GetWritableVector<LaneDataID>(memory_ptr, "/common/turn_data/lane_data_ids");
 
         const auto turn_instructions = layout.GetWritableVector<guidance::TurnInstruction>(
             memory_ptr, "/common/turn_data/turn_instructions");
 
-        const auto entry_class_id_list_ptr =
-            layout.GetBlockPtr<EntryClassID, true>(memory_ptr, "/common/turn_data/entry_class_ids");
-        util::vector_view<EntryClassID> entry_class_ids(
-            entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
+        const auto entry_class_ids =
+            layout.GetWritableVector<EntryClassID>(memory_ptr, "/common/turn_data/entry_class_ids");
 
-        const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>(
+        const auto pre_turn_bearings = layout.GetWritableVector<guidance::TurnBearing>(
             memory_ptr, "/common/turn_data/pre_turn_bearings");
-        util::vector_view<guidance::TurnBearing> pre_turn_bearings(
-            pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings"));
 
-        const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing, true>(
+        const auto post_turn_bearings = layout.GetWritableVector<guidance::TurnBearing>(
             memory_ptr, "/common/turn_data/post_turn_bearings");
-        util::vector_view<guidance::TurnBearing> post_turn_bearings(
-            post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings"));
 
         guidance::TurnDataView turn_data(std::move(turn_instructions),
                                          std::move(lane_data_ids),
@@ -392,73 +386,47 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // load compressed geometry
     {
-        auto geometries_index_ptr =
-            layout.GetBlockPtr<unsigned, true>(memory_ptr, "/common/segment_data/index");
-        util::vector_view<unsigned> geometry_begin_indices(
-            geometries_index_ptr, layout.GetBlockEntries("/common/segment_data/index"));
+        auto geometry_begin_indices =
+            layout.GetWritableVector<unsigned>(memory_ptr, "/common/segment_data/index");
+
+        auto node_list = layout.GetWritableVector<NodeID>(memory_ptr, "/common/segment_data/nodes");
 
         auto num_entries = layout.GetBlockEntries("/common/segment_data/nodes");
 
-        auto geometries_node_list_ptr =
-            layout.GetBlockPtr<NodeID, true>(memory_ptr, "/common/segment_data/nodes");
-        util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
-
-        auto geometries_fwd_weight_list_ptr =
-            layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
-                memory_ptr, "/common/segment_data/forward_weights/packed");
-        extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
-            util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                geometries_fwd_weight_list_ptr,
-                layout.GetBlockEntries("/common/segment_data/forward_weights/packed")),
+        extractor::SegmentDataView::SegmentWeightVector fwd_weight_list(
+            layout.GetWritableVector<extractor::SegmentDataView::SegmentWeightVector::block_type>(
+                memory_ptr, "/common/segment_data/forward_weights/packed"),
             num_entries);
 
-        auto geometries_rev_weight_list_ptr =
-            layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type, true>(
-                memory_ptr, "/common/segment_data/reverse_weights/packed");
-        extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
-            util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                geometries_rev_weight_list_ptr,
-                layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")),
+        extractor::SegmentDataView::SegmentWeightVector rev_weight_list(
+            layout.GetWritableVector<extractor::SegmentDataView::SegmentWeightVector::block_type>(
+                memory_ptr, "/common/segment_data/reverse_weights/packed"),
             num_entries);
 
-        auto geometries_fwd_duration_list_ptr =
-            layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
-                memory_ptr, "/common/segment_data/forward_durations/packed");
-        extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
-            util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                geometries_fwd_duration_list_ptr,
-                layout.GetBlockEntries("/common/segment_data/forward_durations/packed")),
+        extractor::SegmentDataView::SegmentDurationVector fwd_duration_list(
+            layout.GetWritableVector<extractor::SegmentDataView::SegmentDurationVector::block_type>(
+                memory_ptr, "/common/segment_data/forward_durations/packed"),
             num_entries);
 
-        auto geometries_rev_duration_list_ptr =
-            layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type, true>(
-                memory_ptr, "/common/segment_data/reverse_durations/packed");
-        extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
-            util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                geometries_rev_duration_list_ptr,
-                layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")),
+        extractor::SegmentDataView::SegmentDurationVector rev_duration_list(
+            layout.GetWritableVector<extractor::SegmentDataView::SegmentDurationVector::block_type>(
+                memory_ptr, "/common/segment_data/reverse_durations/packed"),
             num_entries);
 
-        auto geometries_fwd_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
+        auto fwd_datasources_list = layout.GetWritableVector<DatasourceID>(
             memory_ptr, "/common/segment_data/forward_data_sources");
-        util::vector_view<DatasourceID> geometry_fwd_datasources_list(
-            geometries_fwd_datasources_list_ptr,
-            layout.GetBlockEntries("/common/segment_data/forward_data_sources"));
 
-        auto geometries_rev_datasources_list_ptr = layout.GetBlockPtr<DatasourceID, true>(
+        auto rev_datasources_list = layout.GetWritableVector<DatasourceID>(
             memory_ptr, "/common/segment_data/reverse_data_sources");
-        util::vector_view<DatasourceID> geometry_rev_datasources_list(
-            geometries_rev_datasources_list_ptr,
-            layout.GetBlockEntries("/common/segment_data/reverse_data_sources"));
 
         extractor::SegmentDataView segment_data{std::move(geometry_begin_indices),
-                                                std::move(geometry_node_list),
-                                                std::move(geometry_fwd_weight_list),
-                                                std::move(geometry_rev_weight_list),
-                                                std::move(geometry_fwd_duration_list),
-                                                std::move(geometry_rev_duration_list),
-                                                std::move(geometry_fwd_datasources_list),
-                                                std::move(geometry_rev_datasources_list)};
+                                                std::move(node_list),
+                                                std::move(fwd_weight_list),
+                                                std::move(rev_weight_list),
+                                                std::move(fwd_duration_list),
+                                                std::move(rev_duration_list),
+                                                std::move(fwd_datasources_list),
+                                                std::move(rev_datasources_list)};
 
         extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
     }
@@ -472,37 +440,29 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Loading list of coordinates
     {
-        const auto coordinates_ptr =
-            layout.GetBlockPtr<util::Coordinate, true>(memory_ptr, "/common/coordinates");
-        const auto osmnodeid_ptr =
-            layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type, true>(
-                memory_ptr, "/common/osm_node_ids/packed");
-        util::vector_view<util::Coordinate> coordinates(
-            coordinates_ptr, layout.GetBlockEntries("/common/coordinates"));
-        extractor::PackedOSMIDsView osm_node_ids(
-            util::vector_view<extractor::PackedOSMIDsView::block_type>(
-                osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")),
-            layout.GetBlockEntries("/common/coordinates"));
+        auto coordinates =
+            layout.GetWritableVector<util::Coordinate>(memory_ptr, "/common/coordinates");
+
+        auto osm_node_ids = extractor::PackedOSMIDsView{
+            layout.GetWritableVector<extractor::PackedOSMIDsView::block_type>(
+                memory_ptr, "/common/osm_node_ids/packed"),
+            coordinates.size()};
 
         extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
     }
 
     // load turn weight penalties
     {
-        auto turn_duration_penalties_ptr =
-            layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, "/common/turn_penalty/weight");
-        util::vector_view<TurnPenalty> turn_duration_penalties(
-            turn_duration_penalties_ptr, layout.GetBlockEntries("/common/turn_penalty/weight"));
+        auto turn_duration_penalties =
+            layout.GetWritableVector<TurnPenalty>(memory_ptr, "/common/turn_penalty/weight");
         extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
                                                 turn_duration_penalties);
     }
 
     // load turn duration penalties
     {
-        auto turn_duration_penalties_ptr =
-            layout.GetBlockPtr<TurnPenalty, true>(memory_ptr, "/common/turn_penalty/duration");
-        util::vector_view<TurnPenalty> turn_duration_penalties(
-            turn_duration_penalties_ptr, layout.GetBlockEntries("/common/turn_penalty/duration"));
+        auto turn_duration_penalties =
+            layout.GetWritableVector<TurnPenalty>(memory_ptr, "/common/turn_penalty/duration");
         extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
                                                   turn_duration_penalties);
     }
@@ -510,16 +470,11 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // store search tree portion of rtree
     {
 
-        const auto rtree_ptr =
-            layout.GetBlockPtr<RTreeNode, true>(memory_ptr, "/common/rtree/search_tree");
-        util::vector_view<RTreeNode> search_tree(
-            rtree_ptr, layout.GetBlockEntries("/common/rtree/search_tree"));
+        const auto search_tree =
+            layout.GetWritableVector<RTreeNode>(memory_ptr, "/common/rtree/search_tree");
 
-        const auto rtree_levelstarts_ptr = layout.GetBlockPtr<std::uint64_t, true>(
+        const auto rtree_level_starts = layout.GetWritableVector<std::uint64_t>(
             memory_ptr, "/common/rtree/search_tree_level_starts");
-        util::vector_view<std::uint64_t> rtree_level_starts(
-            rtree_levelstarts_ptr,
-            layout.GetBlockEntries("/common/rtree/search_tree_level_starts"));
 
         // we need this purely for the interface
         util::vector_view<util::Coordinate> empty_coords;
@@ -546,42 +501,25 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Load intersection data
     {
-        auto bearing_class_id_ptr = layout.GetBlockPtr<BearingClassID, true>(
-            memory_ptr, "/common/intersection_bearings/node_to_class_id");
-        util::vector_view<BearingClassID> bearing_class_id(
-            bearing_class_id_ptr,
-            layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id"));
-
-        auto bearing_values_ptr = layout.GetBlockPtr<DiscreteBearing, true>(
-            memory_ptr, "/common/intersection_bearings/bearing_values");
-        util::vector_view<DiscreteBearing> bearing_values(
-            bearing_values_ptr,
-            layout.GetBlockEntries("/common/intersection_bearings/bearing_values"));
-
-        auto offsets_ptr = layout.GetBlockPtr<unsigned, true>(
+        auto bearing_offsets = layout.GetWritableVector<unsigned>(
             memory_ptr, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
-        auto blocks_ptr =
-            layout.GetBlockPtr<util::RangeTable<16, storage::Ownership::View>::BlockT, true>(
+        auto bearing_blocks =
+            layout.GetWritableVector<util::RangeTable<16, storage::Ownership::View>::BlockT>(
                 memory_ptr, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
-        util::vector_view<unsigned> bearing_offsets(
-            offsets_ptr,
-            layout.GetBlockEntries(
-                "/common/intersection_bearings/class_id_to_ranges/block_offsets"));
-        util::vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT> bearing_blocks(
-            blocks_ptr,
-            layout.GetBlockEntries("/common/intersection_bearings/class_id_to_ranges/diff_blocks"));
-
+        auto bearing_values = layout.GetWritableVector<DiscreteBearing>(
+            memory_ptr, "/common/intersection_bearings/bearing_values");
         util::RangeTable<16, storage::Ownership::View> bearing_range_table(
-            bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size()));
+            std::move(bearing_offsets),
+            std::move(bearing_blocks),
+            static_cast<unsigned>(bearing_values.size()));
 
+        auto bearing_class_id = layout.GetWritableVector<BearingClassID>(
+            memory_ptr, "/common/intersection_bearings/node_to_class_id");
         extractor::IntersectionBearingsView intersection_bearings_view{
             std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
 
-        auto entry_class_ptr = layout.GetBlockPtr<util::guidance::EntryClass, true>(
+        auto entry_classes = layout.GetWritableVector<util::guidance::EntryClass>(
             memory_ptr, "/common/entry_classes");
-        util::vector_view<util::guidance::EntryClass> entry_classes(
-            entry_class_ptr, layout.GetBlockEntries("/common/entry_classes"));
-
         extractor::files::readIntersections(
             config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
     }
@@ -589,26 +527,17 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
     {
         const std::string metric_prefix = "/ch/metrics/" + metric_name;
-        auto graph_nodes_ptr = layout.GetBlockPtr<contractor::QueryGraphView::NodeArrayEntry, true>(
+
+        auto node_list = layout.GetWritableVector<contractor::QueryGraphView::NodeArrayEntry>(
             memory_ptr, metric_prefix + "/contracted_graph/node_array");
-        auto graph_edges_ptr = layout.GetBlockPtr<contractor::QueryGraphView::EdgeArrayEntry, true>(
+        auto edge_list = layout.GetWritableVector<contractor::QueryGraphView::EdgeArrayEntry>(
             memory_ptr, metric_prefix + "/contracted_graph/edge_array");
 
-        util::vector_view<contractor::QueryGraphView::NodeArrayEntry> node_list(
-            graph_nodes_ptr,
-            layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
-        util::vector_view<contractor::QueryGraphView::EdgeArrayEntry> edge_list(
-            graph_edges_ptr,
-            layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
-
         std::vector<util::vector_view<bool>> edge_filter;
-        layout.List(
-            metric_prefix + "/exclude", boost::make_function_output_iterator([&](const auto &name) {
-                auto data_ptr =
-                    layout.GetBlockPtr<util::vector_view<bool>::Word, true>(memory_ptr, name);
-                auto num_entries = layout.GetBlockEntries(name);
-                edge_filter.emplace_back(data_ptr, num_entries);
-            }));
+        layout.List(metric_prefix + "/exclude",
+                    boost::make_function_output_iterator([&](const auto &name) {
+                        edge_filter.push_back(layout.GetWritableVector<bool>(memory_ptr, name));
+                    }));
 
         std::uint32_t graph_connectivity_checksum = 0;
         std::unordered_map<std::string, contractor::ContractedMetricView> metrics = {
@@ -632,22 +561,16 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
         BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
 
-        auto level_data = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
-            memory_ptr, "/mld/multilevelpartition/level_data");
-
-        auto mld_partition_ptr =
-            layout.GetBlockPtr<PartitionID, true>(memory_ptr, "/mld/multilevelpartition/partition");
-        auto partition_entries_count = layout.GetBlockEntries("/mld/multilevelpartition/partition");
-        util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
-
-        auto mld_chilren_ptr = layout.GetBlockPtr<CellID, true>(
+        auto level_data_ptr =
+            layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
+                memory_ptr, "/mld/multilevelpartition/level_data");
+        auto partition =
+            layout.GetWritableVector<PartitionID>(memory_ptr, "/mld/multilevelpartition/partition");
+        auto cell_to_children = layout.GetWritableVector<CellID>(
             memory_ptr, "/mld/multilevelpartition/cell_to_children");
-        auto children_entries_count =
-            layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
-        util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
 
         partitioner::MultiLevelPartitionView mlp{
-            std::move(level_data), std::move(partition), std::move(cell_to_children)};
+            level_data_ptr, std::move(partition), std::move(cell_to_children)};
         partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
     }
 
@@ -656,32 +579,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
         BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
 
-        auto mld_source_boundary_ptr =
-            layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/source_boundary");
-        auto mld_destination_boundary_ptr =
-            layout.GetBlockPtr<NodeID, true>(memory_ptr, "/mld/cellstorage/destination_boundary");
-        auto mld_cells_ptr = layout.GetBlockPtr<partitioner::CellStorageView::CellData, true>(
+        auto source_boundary =
+            layout.GetWritableVector<NodeID>(memory_ptr, "/mld/cellstorage/source_boundary");
+        auto destination_boundary =
+            layout.GetWritableVector<NodeID>(memory_ptr, "/mld/cellstorage/destination_boundary");
+        auto cells = layout.GetWritableVector<partitioner::CellStorageView::CellData>(
             memory_ptr, "/mld/cellstorage/cells");
-        auto mld_cell_level_offsets_ptr = layout.GetBlockPtr<std::uint64_t, true>(
+        auto level_offsets = layout.GetWritableVector<std::uint64_t>(
             memory_ptr, "/mld/cellstorage/level_to_cell_offset");
 
-        auto source_boundary_entries_count =
-            layout.GetBlockEntries("/mld/cellstorage/source_boundary");
-        auto destination_boundary_entries_count =
-            layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
-        auto cells_entries_counts = layout.GetBlockEntries("/mld/cellstorage/cells");
-        auto cell_level_offsets_entries_count =
-            layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
-
-        util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
-                                                  source_boundary_entries_count);
-        util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
-                                                       destination_boundary_entries_count);
-        util::vector_view<partitioner::CellStorageView::CellData> cells(mld_cells_ptr,
-                                                                        cells_entries_counts);
-        util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
-                                                       cell_level_offsets_entries_count);
-
         partitioner::CellStorageView storage{std::move(source_boundary),
                                              std::move(destination_boundary),
                                              std::move(cells),
@@ -706,15 +612,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             auto weights_block_id = prefix + "/weights";
             auto durations_block_id = prefix + "/durations";
 
-            auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
-            auto duration_entries_count = layout.GetBlockEntries(durations_block_id);
-            auto mld_cell_weights_ptr =
-                layout.GetBlockPtr<EdgeWeight, true>(memory_ptr, weights_block_id);
-            auto mld_cell_duration_ptr =
-                layout.GetBlockPtr<EdgeDuration, true>(memory_ptr, durations_block_id);
-            util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
-            util::vector_view<EdgeDuration> durations(mld_cell_duration_ptr,
-                                                      duration_entries_count);
+            auto weights = layout.GetWritableVector<EdgeWeight>(memory_ptr, weights_block_id);
+            auto durations = layout.GetWritableVector<EdgeDuration>(memory_ptr, durations_block_id);
 
             metrics[metric_name].push_back(
                 customizer::CellMetricView{std::move(weights), std::move(durations)});
@@ -725,25 +624,16 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
     {
-
-        auto graph_nodes_ptr =
-            layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry, true>(
+        auto node_list =
+            layout.GetWritableVector<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>(
                 memory_ptr, "/mld/multilevelgraph/node_array");
-        auto graph_edges_ptr =
-            layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry, true>(
+        auto edge_list =
+            layout.GetWritableVector<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry>(
                 memory_ptr, "/mld/multilevelgraph/edge_array");
-        auto graph_node_to_offset_ptr =
-            layout.GetBlockPtr<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset, true>(
+        auto node_to_offset =
+            layout.GetWritableVector<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset>(
                 memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
 
-        util::vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry> node_list(
-            graph_nodes_ptr, layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
-        util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry> edge_list(
-            graph_edges_ptr, layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
-        util::vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset> node_to_offset(
-            graph_node_to_offset_ptr,
-            layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
-
         std::uint32_t graph_connectivity_checksum = 0;
         customizer::MultiLevelEdgeBasedGraphView graph_view(
             std::move(node_list), std::move(edge_list), std::move(node_to_offset));
@@ -762,18 +652,11 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // load maneuver overrides
     {
-        const auto maneuver_overrides_ptr =
-            layout.GetBlockPtr<extractor::StorageManeuverOverride, true>(
-                memory_ptr, "/common/maneuver_overrides/overrides");
-        const auto maneuver_override_node_sequences_ptr = layout.GetBlockPtr<NodeID, true>(
+        auto maneuver_overrides = layout.GetWritableVector<extractor::StorageManeuverOverride>(
+            memory_ptr, "/common/maneuver_overrides/overrides");
+        auto maneuver_override_node_sequences = layout.GetWritableVector<NodeID>(
             memory_ptr, "/common/maneuver_overrides/node_sequences");
 
-        util::vector_view<extractor::StorageManeuverOverride> maneuver_overrides(
-            maneuver_overrides_ptr, layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
-        util::vector_view<NodeID> maneuver_override_node_sequences(
-            maneuver_override_node_sequences_ptr,
-            layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
-
         extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
                                                 maneuver_overrides,
                                                 maneuver_override_node_sequences);

From 15300e276ece035800f787c71ad58013a1c6b813 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Thu, 29 Mar 2018 16:10:26 +0000
Subject: [PATCH 14/19] Fix checking for non-empty string in prefix check

---
 include/storage/shared_datatype.hpp | 2 +-
 unit_tests/storage/data_layout.cpp  | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index 61136a13c0a..9656653aac1 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -35,7 +35,7 @@ namespace detail
 inline std::string trimName(const std::string &name_prefix, const std::string &name)
 {
     // list directory and
-    if (name_prefix.back() == '/')
+    if (!name_prefix.empty() && name_prefix.back() == '/')
     {
         auto directory_position = name.find_first_of("/", name_prefix.length());
         // this is a "file" in the directory of name_prefix
diff --git a/unit_tests/storage/data_layout.cpp b/unit_tests/storage/data_layout.cpp
index 5335ff7bf63..f25459075c4 100644
--- a/unit_tests/storage/data_layout.cpp
+++ b/unit_tests/storage/data_layout.cpp
@@ -93,6 +93,10 @@ BOOST_AUTO_TEST_CASE(layout_list_test)
     layout.List("/mld/metrics/", std::back_inserter(results_5));
     layout.List("/mld/", std::back_inserter(results_6));
 
+    std::vector<std::string> results_7;
+    layout.List("", std::back_inserter(results_7));
+    BOOST_CHECK_EQUAL(results_7.size(), 7);
+
     CHECK_EQUAL_RANGE(
         results_1, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
     CHECK_EQUAL_RANGE(

From 371526be13bda171adf1289881c2ef593a9ec4dd Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Tue, 3 Apr 2018 09:43:43 +0000
Subject: [PATCH 15/19] Remove canary to get a consistent interface for getting
 a block pointer

---
 include/storage/shared_datatype.hpp |  58 +--------
 include/storage/view_factory.hpp    |  31 +++++
 src/storage/storage.cpp             | 183 ++++++++++++++--------------
 3 files changed, 123 insertions(+), 149 deletions(-)
 create mode 100644 include/storage/view_factory.hpp

diff --git a/include/storage/shared_datatype.hpp b/include/storage/shared_datatype.hpp
index 9656653aac1..7c3e561d387 100644
--- a/include/storage/shared_datatype.hpp
+++ b/include/storage/shared_datatype.hpp
@@ -7,7 +7,6 @@
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/log.hpp"
-#include "util/vector_view.hpp"
 
 #include <boost/assert.hpp>
 
@@ -55,9 +54,6 @@ inline std::string trimName(const std::string &name_prefix, const std::string &n
 }
 }
 
-// Added at the start and end of each block as sanity check
-const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
-
 class DataLayout
 {
   public:
@@ -82,55 +78,17 @@ class DataLayout
         uint64_t result = 0;
         for (const auto &name_and_block : blocks)
         {
-            result += 2 * sizeof(CANARY) + GetBlockSize(name_and_block.first) + BLOCK_ALIGNMENT;
+            result += GetBlockSize(name_and_block.first) + BLOCK_ALIGNMENT;
         }
         return result;
     }
 
-    template <typename T>
-    util::vector_view<T> GetVector(char *shared_memory, const std::string &name) const
-    {
-        return util::vector_view<T>(GetBlockPtr<T>(shared_memory, name), GetBlockEntries(name));
-    }
-
-    template <typename T>
-    util::vector_view<T> GetWritableVector(char *shared_memory, const std::string &name) const
-    {
-        return util::vector_view<T>(GetBlockPtr<T, true>(shared_memory, name),
-                                    GetBlockEntries(name));
-    }
-
-    template <typename T, bool WRITE_CANARY = false>
-    inline T *GetBlockPtr(char *shared_memory, const std::string &name) const
+    template <typename T> inline T *GetBlockPtr(char *shared_memory, const std::string &name) const
     {
         static_assert(BLOCK_ALIGNMENT % std::alignment_of<T>::value == 0,
                       "Datatype does not fit alignment constraints.");
 
         char *ptr = (char *)GetAlignedBlockPtr(shared_memory, name);
-        if (WRITE_CANARY)
-        {
-            char *start_canary_ptr = ptr - sizeof(CANARY);
-            char *end_canary_ptr = ptr + GetBlockSize(name);
-            std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
-            std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
-        }
-        else
-        {
-            char *start_canary_ptr = ptr - sizeof(CANARY);
-            char *end_canary_ptr = ptr + GetBlockSize(name);
-            bool start_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
-            bool end_canary_alive = std::equal(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
-            if (!start_canary_alive)
-            {
-                throw util::exception("Start canary of block corrupted. (" + name + ")" +
-                                      SOURCE_REF);
-            }
-            if (!end_canary_alive)
-            {
-                throw util::exception("End canary of block corrupted. (" + name + ")" + SOURCE_REF);
-            }
-        }
-
         return (T *)ptr;
     }
 
@@ -190,13 +148,10 @@ class DataLayout
 
         for (auto iter = blocks.begin(); iter != block_iter; ++iter)
         {
-            ptr = static_cast<char *>(ptr) + sizeof(CANARY);
             ptr = align(ptr);
             ptr = static_cast<char *>(ptr) + iter->second.byte_size;
-            ptr = static_cast<char *>(ptr) + sizeof(CANARY);
         }
 
-        ptr = static_cast<char *>(ptr) + sizeof(CANARY);
         ptr = align(ptr);
         return ptr;
     }
@@ -205,15 +160,6 @@ class DataLayout
     std::map<std::string, Block> blocks;
 };
 
-template <>
-inline util::vector_view<bool> DataLayout::GetWritableVector<bool>(char *shared_memory,
-                                                                   const std::string &name) const
-{
-    return util::vector_view<bool>(
-        GetBlockPtr<util::vector_view<bool>::Word, true>(shared_memory, name),
-        GetBlockEntries(name));
-}
-
 enum SharedDataType
 {
     REGION_NONE,
diff --git a/include/storage/view_factory.hpp b/include/storage/view_factory.hpp
new file mode 100644
index 00000000000..7522e5cc67b
--- /dev/null
+++ b/include/storage/view_factory.hpp
@@ -0,0 +1,31 @@
+#ifndef OSRM_STOARGE_VIEW_FACTORY_HPP
+#define OSRM_STOARGE_VIEW_FACTORY_HPP
+
+#include "storage/shared_datatype.hpp"
+
+#include "util/vector_view.hpp"
+
+namespace osrm
+{
+namespace storage
+{
+
+template <typename T>
+util::vector_view<T> make_vector_view(char *memory_ptr, DataLayout layout, const std::string &name)
+{
+    return util::vector_view<T>(layout.GetBlockPtr<T>(memory_ptr, name),
+                                layout.GetBlockEntries(name));
+}
+
+template <>
+inline util::vector_view<bool>
+make_vector_view(char *memory_ptr, DataLayout layout, const std::string &name)
+{
+    return util::vector_view<bool>(
+        layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_ptr, name),
+        layout.GetBlockEntries(name));
+}
+}
+}
+
+#endif
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 08320efb322..4ad84c26907 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -5,6 +5,7 @@
 #include "storage/shared_memory.hpp"
 #include "storage/shared_memory_ownership.hpp"
 #include "storage/shared_monitor.hpp"
+#include "storage/view_factory.hpp"
 
 #include "contractor/files.hpp"
 #include "contractor/query_graph.hpp"
@@ -296,7 +297,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // store the filename of the on-disk portion of the RTree
     {
         const auto file_index_path_ptr =
-            layout.GetBlockPtr<char, true>(memory_ptr, "/common/rtree/file_index_path");
+            layout.GetBlockPtr<char>(memory_ptr, "/common/rtree/file_index_path");
         // make sure we have 0 ending
         std::fill(file_index_path_ptr,
                   file_index_path_ptr + layout.GetBlockSize("/common/rtree/file_index_path"),
@@ -311,11 +312,10 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Name data
     {
-        auto blocks =
-            layout.GetWritableVector<extractor::NameTableView::IndexedData::BlockReference>(
-                memory_ptr, "/common/names/blocks");
-        auto values = layout.GetWritableVector<extractor::NameTableView::IndexedData::ValueType>(
-            memory_ptr, "/common/names/values");
+        auto blocks = make_vector_view<extractor::NameTableView::IndexedData::BlockReference>(
+            memory_ptr, layout, "/common/names/blocks");
+        auto values = make_vector_view<extractor::NameTableView::IndexedData::ValueType>(
+            memory_ptr, layout, "/common/names/values");
 
         extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
         extractor::NameTableView name_table{index_data_view};
@@ -324,8 +324,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Turn lane data
     {
-        auto turn_lane_data = layout.GetWritableVector<util::guidance::LaneTupleIdPair>(
-            memory_ptr, "/common/turn_lanes/data");
+        auto turn_lane_data = make_vector_view<util::guidance::LaneTupleIdPair>(
+            memory_ptr, layout, "/common/turn_lanes/data");
 
         extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
     }
@@ -333,19 +333,19 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Turn lane descriptions
     {
         auto offsets =
-            layout.GetWritableVector<std::uint32_t>(memory_ptr, "/common/turn_lanes/offsets");
-        auto masks = layout.GetWritableVector<extractor::TurnLaneType::Mask>(
-            memory_ptr, "/common/turn_lanes/masks");
+            make_vector_view<std::uint32_t>(memory_ptr, layout, "/common/turn_lanes/offsets");
+        auto masks = make_vector_view<extractor::TurnLaneType::Mask>(
+            memory_ptr, layout, "/common/turn_lanes/masks");
 
         extractor::files::readTurnLaneDescriptions(config.GetPath(".osrm.tls"), offsets, masks);
     }
 
     // Load edge-based nodes data
     {
-        auto edge_based_node_data = layout.GetWritableVector<extractor::EdgeBasedNode>(
-            memory_ptr, "/common/ebg_node_data/nodes");
-        auto annotation_data = layout.GetWritableVector<extractor::NodeBasedEdgeAnnotation>(
-            memory_ptr, "/common/ebg_node_data/annotations");
+        auto edge_based_node_data = make_vector_view<extractor::EdgeBasedNode>(
+            memory_ptr, layout, "/common/ebg_node_data/nodes");
+        auto annotation_data = make_vector_view<extractor::NodeBasedEdgeAnnotation>(
+            memory_ptr, layout, "/common/ebg_node_data/annotations");
 
         extractor::EdgeBasedNodeDataView node_data(std::move(edge_based_node_data),
                                                    std::move(annotation_data));
@@ -356,19 +356,19 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Load original edge data
     {
         auto lane_data_ids =
-            layout.GetWritableVector<LaneDataID>(memory_ptr, "/common/turn_data/lane_data_ids");
+            make_vector_view<LaneDataID>(memory_ptr, layout, "/common/turn_data/lane_data_ids");
 
-        const auto turn_instructions = layout.GetWritableVector<guidance::TurnInstruction>(
-            memory_ptr, "/common/turn_data/turn_instructions");
+        const auto turn_instructions = make_vector_view<guidance::TurnInstruction>(
+            memory_ptr, layout, "/common/turn_data/turn_instructions");
 
         const auto entry_class_ids =
-            layout.GetWritableVector<EntryClassID>(memory_ptr, "/common/turn_data/entry_class_ids");
+            make_vector_view<EntryClassID>(memory_ptr, layout, "/common/turn_data/entry_class_ids");
 
-        const auto pre_turn_bearings = layout.GetWritableVector<guidance::TurnBearing>(
-            memory_ptr, "/common/turn_data/pre_turn_bearings");
+        const auto pre_turn_bearings = make_vector_view<guidance::TurnBearing>(
+            memory_ptr, layout, "/common/turn_data/pre_turn_bearings");
 
-        const auto post_turn_bearings = layout.GetWritableVector<guidance::TurnBearing>(
-            memory_ptr, "/common/turn_data/post_turn_bearings");
+        const auto post_turn_bearings = make_vector_view<guidance::TurnBearing>(
+            memory_ptr, layout, "/common/turn_data/post_turn_bearings");
 
         guidance::TurnDataView turn_data(std::move(turn_instructions),
                                          std::move(lane_data_ids),
@@ -376,7 +376,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
                                          std::move(pre_turn_bearings),
                                          std::move(post_turn_bearings));
         auto connectivity_checksum_ptr =
-            layout.GetBlockPtr<std::uint32_t, true>(memory_ptr, "/common/connectivity_checksum");
+            layout.GetBlockPtr<std::uint32_t>(memory_ptr, "/common/connectivity_checksum");
 
         guidance::files::readTurnData(
             config.GetPath(".osrm.edges"), turn_data, *connectivity_checksum_ptr);
@@ -387,37 +387,37 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // load compressed geometry
     {
         auto geometry_begin_indices =
-            layout.GetWritableVector<unsigned>(memory_ptr, "/common/segment_data/index");
+            make_vector_view<unsigned>(memory_ptr, layout, "/common/segment_data/index");
 
-        auto node_list = layout.GetWritableVector<NodeID>(memory_ptr, "/common/segment_data/nodes");
+        auto node_list = make_vector_view<NodeID>(memory_ptr, layout, "/common/segment_data/nodes");
 
         auto num_entries = layout.GetBlockEntries("/common/segment_data/nodes");
 
         extractor::SegmentDataView::SegmentWeightVector fwd_weight_list(
-            layout.GetWritableVector<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                memory_ptr, "/common/segment_data/forward_weights/packed"),
+            make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
+                memory_ptr, layout, "/common/segment_data/forward_weights/packed"),
             num_entries);
 
         extractor::SegmentDataView::SegmentWeightVector rev_weight_list(
-            layout.GetWritableVector<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                memory_ptr, "/common/segment_data/reverse_weights/packed"),
+            make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
+                memory_ptr, layout, "/common/segment_data/reverse_weights/packed"),
             num_entries);
 
         extractor::SegmentDataView::SegmentDurationVector fwd_duration_list(
-            layout.GetWritableVector<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                memory_ptr, "/common/segment_data/forward_durations/packed"),
+            make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
+                memory_ptr, layout, "/common/segment_data/forward_durations/packed"),
             num_entries);
 
         extractor::SegmentDataView::SegmentDurationVector rev_duration_list(
-            layout.GetWritableVector<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                memory_ptr, "/common/segment_data/reverse_durations/packed"),
+            make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
+                memory_ptr, layout, "/common/segment_data/reverse_durations/packed"),
             num_entries);
 
-        auto fwd_datasources_list = layout.GetWritableVector<DatasourceID>(
-            memory_ptr, "/common/segment_data/forward_data_sources");
+        auto fwd_datasources_list = make_vector_view<DatasourceID>(
+            memory_ptr, layout, "/common/segment_data/forward_data_sources");
 
-        auto rev_datasources_list = layout.GetWritableVector<DatasourceID>(
-            memory_ptr, "/common/segment_data/reverse_data_sources");
+        auto rev_datasources_list = make_vector_view<DatasourceID>(
+            memory_ptr, layout, "/common/segment_data/reverse_data_sources");
 
         extractor::SegmentDataView segment_data{std::move(geometry_begin_indices),
                                                 std::move(node_list),
@@ -432,8 +432,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     }
 
     {
-        const auto datasources_names_ptr = layout.GetBlockPtr<extractor::Datasources, true>(
-            memory_ptr, "/common/data_sources_names");
+        const auto datasources_names_ptr =
+            layout.GetBlockPtr<extractor::Datasources>(memory_ptr, "/common/data_sources_names");
         extractor::files::readDatasources(config.GetPath(".osrm.datasource_names"),
                                           *datasources_names_ptr);
     }
@@ -441,12 +441,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // Loading list of coordinates
     {
         auto coordinates =
-            layout.GetWritableVector<util::Coordinate>(memory_ptr, "/common/coordinates");
+            make_vector_view<util::Coordinate>(memory_ptr, layout, "/common/coordinates");
 
-        auto osm_node_ids = extractor::PackedOSMIDsView{
-            layout.GetWritableVector<extractor::PackedOSMIDsView::block_type>(
-                memory_ptr, "/common/osm_node_ids/packed"),
-            coordinates.size()};
+        auto osm_node_ids =
+            extractor::PackedOSMIDsView{make_vector_view<extractor::PackedOSMIDsView::block_type>(
+                                            memory_ptr, layout, "/common/osm_node_ids/packed"),
+                                        coordinates.size()};
 
         extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
     }
@@ -454,7 +454,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // load turn weight penalties
     {
         auto turn_duration_penalties =
-            layout.GetWritableVector<TurnPenalty>(memory_ptr, "/common/turn_penalty/weight");
+            make_vector_view<TurnPenalty>(memory_ptr, layout, "/common/turn_penalty/weight");
         extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
                                                 turn_duration_penalties);
     }
@@ -462,7 +462,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // load turn duration penalties
     {
         auto turn_duration_penalties =
-            layout.GetWritableVector<TurnPenalty>(memory_ptr, "/common/turn_penalty/duration");
+            make_vector_view<TurnPenalty>(memory_ptr, layout, "/common/turn_penalty/duration");
         extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
                                                   turn_duration_penalties);
     }
@@ -471,10 +471,10 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     {
 
         const auto search_tree =
-            layout.GetWritableVector<RTreeNode>(memory_ptr, "/common/rtree/search_tree");
+            make_vector_view<RTreeNode>(memory_ptr, layout, "/common/rtree/search_tree");
 
-        const auto rtree_level_starts = layout.GetWritableVector<std::uint64_t>(
-            memory_ptr, "/common/rtree/search_tree_level_starts");
+        const auto rtree_level_starts = make_vector_view<std::uint64_t>(
+            memory_ptr, layout, "/common/rtree/search_tree_level_starts");
 
         // we need this purely for the interface
         util::vector_view<util::Coordinate> empty_coords;
@@ -491,8 +491,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     std::string metric_name;
     // load profile properties
     {
-        const auto profile_properties_ptr = layout.GetBlockPtr<extractor::ProfileProperties, true>(
-            memory_ptr, "/common/properties");
+        const auto profile_properties_ptr =
+            layout.GetBlockPtr<extractor::ProfileProperties>(memory_ptr, "/common/properties");
         extractor::files::readProfileProperties(config.GetPath(".osrm.properties"),
                                                 *profile_properties_ptr);
 
@@ -501,25 +501,25 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Load intersection data
     {
-        auto bearing_offsets = layout.GetWritableVector<unsigned>(
-            memory_ptr, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
+        auto bearing_offsets = make_vector_view<unsigned>(
+            memory_ptr, layout, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
         auto bearing_blocks =
-            layout.GetWritableVector<util::RangeTable<16, storage::Ownership::View>::BlockT>(
-                memory_ptr, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
-        auto bearing_values = layout.GetWritableVector<DiscreteBearing>(
-            memory_ptr, "/common/intersection_bearings/bearing_values");
+            make_vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT>(
+                memory_ptr, layout, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
+        auto bearing_values = make_vector_view<DiscreteBearing>(
+            memory_ptr, layout, "/common/intersection_bearings/bearing_values");
         util::RangeTable<16, storage::Ownership::View> bearing_range_table(
             std::move(bearing_offsets),
             std::move(bearing_blocks),
             static_cast<unsigned>(bearing_values.size()));
 
-        auto bearing_class_id = layout.GetWritableVector<BearingClassID>(
-            memory_ptr, "/common/intersection_bearings/node_to_class_id");
+        auto bearing_class_id = make_vector_view<BearingClassID>(
+            memory_ptr, layout, "/common/intersection_bearings/node_to_class_id");
         extractor::IntersectionBearingsView intersection_bearings_view{
             std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
 
-        auto entry_classes = layout.GetWritableVector<util::guidance::EntryClass>(
-            memory_ptr, "/common/entry_classes");
+        auto entry_classes = make_vector_view<util::guidance::EntryClass>(
+            memory_ptr, layout, "/common/entry_classes");
         extractor::files::readIntersections(
             config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
     }
@@ -528,15 +528,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     {
         const std::string metric_prefix = "/ch/metrics/" + metric_name;
 
-        auto node_list = layout.GetWritableVector<contractor::QueryGraphView::NodeArrayEntry>(
-            memory_ptr, metric_prefix + "/contracted_graph/node_array");
-        auto edge_list = layout.GetWritableVector<contractor::QueryGraphView::EdgeArrayEntry>(
-            memory_ptr, metric_prefix + "/contracted_graph/edge_array");
+        auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
+            memory_ptr, layout, metric_prefix + "/contracted_graph/node_array");
+        auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>(
+            memory_ptr, layout, metric_prefix + "/contracted_graph/edge_array");
 
         std::vector<util::vector_view<bool>> edge_filter;
         layout.List(metric_prefix + "/exclude",
                     boost::make_function_output_iterator([&](const auto &name) {
-                        edge_filter.push_back(layout.GetWritableVector<bool>(memory_ptr, name));
+                        edge_filter.push_back(make_vector_view<bool>(memory_ptr, layout, name));
                     }));
 
         std::uint32_t graph_connectivity_checksum = 0;
@@ -561,13 +561,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
         BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
 
-        auto level_data_ptr =
-            layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData, true>(
-                memory_ptr, "/mld/multilevelpartition/level_data");
+        auto level_data_ptr = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
+            memory_ptr, "/mld/multilevelpartition/level_data");
         auto partition =
-            layout.GetWritableVector<PartitionID>(memory_ptr, "/mld/multilevelpartition/partition");
-        auto cell_to_children = layout.GetWritableVector<CellID>(
-            memory_ptr, "/mld/multilevelpartition/cell_to_children");
+            make_vector_view<PartitionID>(memory_ptr, layout, "/mld/multilevelpartition/partition");
+        auto cell_to_children = make_vector_view<CellID>(
+            memory_ptr, layout, "/mld/multilevelpartition/cell_to_children");
 
         partitioner::MultiLevelPartitionView mlp{
             level_data_ptr, std::move(partition), std::move(cell_to_children)};
@@ -580,13 +579,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
         BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
 
         auto source_boundary =
-            layout.GetWritableVector<NodeID>(memory_ptr, "/mld/cellstorage/source_boundary");
+            make_vector_view<NodeID>(memory_ptr, layout, "/mld/cellstorage/source_boundary");
         auto destination_boundary =
-            layout.GetWritableVector<NodeID>(memory_ptr, "/mld/cellstorage/destination_boundary");
-        auto cells = layout.GetWritableVector<partitioner::CellStorageView::CellData>(
-            memory_ptr, "/mld/cellstorage/cells");
-        auto level_offsets = layout.GetWritableVector<std::uint64_t>(
-            memory_ptr, "/mld/cellstorage/level_to_cell_offset");
+            make_vector_view<NodeID>(memory_ptr, layout, "/mld/cellstorage/destination_boundary");
+        auto cells = make_vector_view<partitioner::CellStorageView::CellData>(
+            memory_ptr, layout, "/mld/cellstorage/cells");
+        auto level_offsets = make_vector_view<std::uint64_t>(
+            memory_ptr, layout, "/mld/cellstorage/level_to_cell_offset");
 
         partitioner::CellStorageView storage{std::move(source_boundary),
                                              std::move(destination_boundary),
@@ -612,8 +611,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
             auto weights_block_id = prefix + "/weights";
             auto durations_block_id = prefix + "/durations";
 
-            auto weights = layout.GetWritableVector<EdgeWeight>(memory_ptr, weights_block_id);
-            auto durations = layout.GetWritableVector<EdgeDuration>(memory_ptr, durations_block_id);
+            auto weights = make_vector_view<EdgeWeight>(memory_ptr, layout, weights_block_id);
+            auto durations = make_vector_view<EdgeDuration>(memory_ptr, layout, durations_block_id);
 
             metrics[metric_name].push_back(
                 customizer::CellMetricView{std::move(weights), std::move(durations)});
@@ -624,15 +623,13 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
     {
-        auto node_list =
-            layout.GetWritableVector<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>(
-                memory_ptr, "/mld/multilevelgraph/node_array");
-        auto edge_list =
-            layout.GetWritableVector<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry>(
-                memory_ptr, "/mld/multilevelgraph/edge_array");
+        auto node_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>(
+            memory_ptr, layout, "/mld/multilevelgraph/node_array");
+        auto edge_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry>(
+            memory_ptr, layout, "/mld/multilevelgraph/edge_array");
         auto node_to_offset =
-            layout.GetWritableVector<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset>(
-                memory_ptr, "/mld/multilevelgraph/node_to_edge_offset");
+            make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset>(
+                memory_ptr, layout, "/mld/multilevelgraph/node_to_edge_offset");
 
         std::uint32_t graph_connectivity_checksum = 0;
         customizer::MultiLevelEdgeBasedGraphView graph_view(
@@ -652,10 +649,10 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // load maneuver overrides
     {
-        auto maneuver_overrides = layout.GetWritableVector<extractor::StorageManeuverOverride>(
-            memory_ptr, "/common/maneuver_overrides/overrides");
-        auto maneuver_override_node_sequences = layout.GetWritableVector<NodeID>(
-            memory_ptr, "/common/maneuver_overrides/node_sequences");
+        auto maneuver_overrides = make_vector_view<extractor::StorageManeuverOverride>(
+            memory_ptr, layout, "/common/maneuver_overrides/overrides");
+        auto maneuver_override_node_sequences = make_vector_view<NodeID>(
+            memory_ptr, layout, "/common/maneuver_overrides/node_sequences");
 
         extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
                                                 maneuver_overrides,

From 6bac06c30b2136c46eb6db8105df17497dbd24a5 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Tue, 3 Apr 2018 12:24:03 +0000
Subject: [PATCH 16/19] Refactor view creation to own header file

---
 include/storage/view_factory.hpp | 316 ++++++++++++++++++++++++++++++-
 include/util/packed_vector.hpp   |   1 +
 src/storage/storage.cpp          | 260 +++----------------------
 3 files changed, 341 insertions(+), 236 deletions(-)

diff --git a/include/storage/view_factory.hpp b/include/storage/view_factory.hpp
index 7522e5cc67b..91acdf9c764 100644
--- a/include/storage/view_factory.hpp
+++ b/include/storage/view_factory.hpp
@@ -3,6 +3,39 @@
 
 #include "storage/shared_datatype.hpp"
 
+#include "contractor/contracted_metric.hpp"
+#include "contractor/query_graph.hpp"
+
+#include "customizer/edge_based_graph.hpp"
+
+#include "extractor/class_data.hpp"
+#include "extractor/compressed_edge_container.hpp"
+#include "extractor/edge_based_edge.hpp"
+#include "extractor/edge_based_node.hpp"
+#include "extractor/edge_based_node_segment.hpp"
+#include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
+#include "extractor/packed_osm_ids.hpp"
+#include "extractor/query_node.hpp"
+#include "extractor/travel_mode.hpp"
+
+#include "guidance/turn_bearing.hpp"
+#include "guidance/turn_data_container.hpp"
+#include "guidance/turn_instruction.hpp"
+
+#include "partitioner/cell_storage.hpp"
+#include "partitioner/edge_based_graph_reader.hpp"
+#include "partitioner/multi_level_partition.hpp"
+
+#include "util/coordinate.hpp"
+#include "util/packed_vector.hpp"
+#include "util/range_table.hpp"
+#include "util/static_graph.hpp"
+#include "util/static_rtree.hpp"
+#include "util/typedefs.hpp"
+#include "util/vector_view.hpp"
+
+#include "util/packed_vector.hpp"
 #include "util/vector_view.hpp"
 
 namespace osrm
@@ -11,7 +44,8 @@ namespace storage
 {
 
 template <typename T>
-util::vector_view<T> make_vector_view(char *memory_ptr, DataLayout layout, const std::string &name)
+util::vector_view<T>
+make_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     return util::vector_view<T>(layout.GetBlockPtr<T>(memory_ptr, name),
                                 layout.GetBlockEntries(name));
@@ -19,12 +53,290 @@ util::vector_view<T> make_vector_view(char *memory_ptr, DataLayout layout, const
 
 template <>
 inline util::vector_view<bool>
-make_vector_view(char *memory_ptr, DataLayout layout, const std::string &name)
+make_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     return util::vector_view<bool>(
         layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_ptr, name),
         layout.GetBlockEntries(name));
 }
+
+template <typename T, std::size_t Bits>
+util::PackedVectorView<T, Bits>
+make_packed_vector_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    using V = util::PackedVectorView<T, Bits>;
+    auto packed_internal = util::vector_view<typename V::block_type>(
+        layout.GetBlockPtr<typename V::block_type>(memory_ptr, name + "/packed"),
+        layout.GetBlockEntries(name + "/packed"));
+    // the real size needs to come from an external source
+    return V{packed_internal, std::numeric_limits<std::size_t>::max()};
+}
+
+auto make_name_table_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    auto blocks = make_vector_view<extractor::NameTableView::IndexedData::BlockReference>(
+        memory_ptr, layout, name + "/blocks");
+    auto values = make_vector_view<extractor::NameTableView::IndexedData::ValueType>(
+        memory_ptr, layout, name + "/values");
+
+    extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
+    return extractor::NameTableView{index_data_view};
+}
+
+auto make_lane_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    return make_vector_view<util::guidance::LaneTupleIdPair>(memory_ptr, layout, name + "/data");
+}
+
+auto make_turn_lane_description_views(char *memory_ptr,
+                                      const DataLayout &layout,
+                                      const std::string &name)
+{
+    auto offsets = make_vector_view<std::uint32_t>(memory_ptr, layout, name + "/offsets");
+    auto masks =
+        make_vector_view<extractor::TurnLaneType::Mask>(memory_ptr, layout, name + "/masks");
+
+    return std::make_tuple(offsets, masks);
+}
+
+auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    auto edge_based_node_data =
+        make_vector_view<extractor::EdgeBasedNode>(memory_ptr, layout, name + "/nodes");
+    auto annotation_data = make_vector_view<extractor::NodeBasedEdgeAnnotation>(
+        memory_ptr, layout, name + "/annotations");
+
+    return extractor::EdgeBasedNodeDataView(std::move(edge_based_node_data),
+                                            std::move(annotation_data));
+}
+
+auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    auto lane_data_ids = make_vector_view<LaneDataID>(memory_ptr, layout, name + "/lane_data_ids");
+
+    const auto turn_instructions = make_vector_view<guidance::TurnInstruction>(
+        memory_ptr, layout, name + "/turn_instructions");
+
+    const auto entry_class_ids =
+        make_vector_view<EntryClassID>(memory_ptr, layout, name + "/entry_class_ids");
+
+    const auto pre_turn_bearings =
+        make_vector_view<guidance::TurnBearing>(memory_ptr, layout, name + "/pre_turn_bearings");
+
+    const auto post_turn_bearings =
+        make_vector_view<guidance::TurnBearing>(memory_ptr, layout, name + "/post_turn_bearings");
+
+    return guidance::TurnDataView(std::move(turn_instructions),
+                                  std::move(lane_data_ids),
+                                  std::move(entry_class_ids),
+                                  std::move(pre_turn_bearings),
+                                  std::move(post_turn_bearings));
+}
+
+auto make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    auto geometry_begin_indices = make_vector_view<unsigned>(memory_ptr, layout, name + "/index");
+
+    auto node_list = make_vector_view<NodeID>(memory_ptr, layout, name + "/nodes");
+
+    auto num_entries = layout.GetBlockEntries(name + "/nodes");
+
+    extractor::SegmentDataView::SegmentWeightVector fwd_weight_list(
+        make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
+            memory_ptr, layout, name + "/forward_weights/packed"),
+        num_entries);
+
+    extractor::SegmentDataView::SegmentWeightVector rev_weight_list(
+        make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
+            memory_ptr, layout, name + "/reverse_weights/packed"),
+        num_entries);
+
+    extractor::SegmentDataView::SegmentDurationVector fwd_duration_list(
+        make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
+            memory_ptr, layout, name + "/forward_durations/packed"),
+        num_entries);
+
+    extractor::SegmentDataView::SegmentDurationVector rev_duration_list(
+        make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
+            memory_ptr, layout, name + "/reverse_durations/packed"),
+        num_entries);
+
+    auto fwd_datasources_list =
+        make_vector_view<DatasourceID>(memory_ptr, layout, name + "/forward_data_sources");
+
+    auto rev_datasources_list =
+        make_vector_view<DatasourceID>(memory_ptr, layout, name + "/reverse_data_sources");
+
+    return extractor::SegmentDataView{std::move(geometry_begin_indices),
+                                      std::move(node_list),
+                                      std::move(fwd_weight_list),
+                                      std::move(rev_weight_list),
+                                      std::move(fwd_duration_list),
+                                      std::move(rev_duration_list),
+                                      std::move(fwd_datasources_list),
+                                      std::move(rev_datasources_list)};
+}
+
+auto make_coordinates_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    return make_vector_view<util::Coordinate>(memory_ptr, layout, name);
+}
+
+auto make_osm_ids_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    return make_packed_vector_view<extractor::PackedOSMIDsView::value_type,
+                                   extractor::PackedOSMIDsView::value_size>(
+        memory_ptr, layout, name);
+}
+
+auto make_turn_duration_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    return make_vector_view<TurnPenalty>(memory_ptr, layout, name + "/weight");
+}
+
+auto make_turn_penalties_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    return make_vector_view<TurnPenalty>(memory_ptr, layout, name + "/duration");
+}
+
+auto make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    using RTreeLeaf = extractor::EdgeBasedNodeSegment;
+    using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
+
+    const auto search_tree = make_vector_view<RTreeNode>(memory_ptr, layout, name + "/search_tree");
+
+    const auto rtree_level_starts =
+        make_vector_view<std::uint64_t>(memory_ptr, layout, name + "/search_tree_level_starts");
+
+    const auto coordinates = make_coordinates_view(memory_ptr, layout, "/common/coordinates");
+
+    const char *path = layout.GetBlockPtr<char>(memory_ptr, name + "/file_index_path");
+
+    return util::StaticRTree<RTreeLeaf, storage::Ownership::View>{
+        std::move(search_tree), std::move(rtree_level_starts), path, std::move(coordinates)};
+}
+
+auto make_intersection_bearings_view(char *memory_ptr,
+                                     const DataLayout &layout,
+                                     const std::string &name)
+{
+    auto bearing_offsets =
+        make_vector_view<unsigned>(memory_ptr, layout, name + "/class_id_to_ranges/block_offsets");
+    auto bearing_blocks = make_vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT>(
+        memory_ptr, layout, name + "/class_id_to_ranges/diff_blocks");
+    auto bearing_values =
+        make_vector_view<DiscreteBearing>(memory_ptr, layout, name + "/bearing_values");
+    util::RangeTable<16, storage::Ownership::View> bearing_range_table(
+        std::move(bearing_offsets),
+        std::move(bearing_blocks),
+        static_cast<unsigned>(bearing_values.size()));
+
+    auto bearing_class_id =
+        make_vector_view<BearingClassID>(memory_ptr, layout, name + "/node_to_class_id");
+    return extractor::IntersectionBearingsView{
+        std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
+}
+
+auto make_entry_classes_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    return make_vector_view<util::guidance::EntryClass>(memory_ptr, layout, name);
+}
+
+auto make_contracted_metric_view(char *memory_ptr,
+                                 const DataLayout &layout,
+                                 const std::string &name)
+{
+    auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
+        memory_ptr, layout, name + "/contracted_graph/node_array");
+    auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>(
+        memory_ptr, layout, name + "/contracted_graph/edge_array");
+
+    std::vector<util::vector_view<bool>> edge_filter;
+    layout.List(name + "/exclude",
+                boost::make_function_output_iterator([&](const auto &filter_name) {
+                    edge_filter.push_back(make_vector_view<bool>(memory_ptr, layout, filter_name));
+                }));
+
+    return contractor::ContractedMetricView{{std::move(node_list), std::move(edge_list)},
+                                            std::move(edge_filter)};
+}
+
+auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    auto level_data_ptr = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
+        memory_ptr, name + "/level_data");
+    auto partition = make_vector_view<PartitionID>(memory_ptr, layout, name + "/partition");
+    auto cell_to_children =
+        make_vector_view<CellID>(memory_ptr, layout, name + "/cell_to_children");
+
+    return partitioner::MultiLevelPartitionView{
+        level_data_ptr, std::move(partition), std::move(cell_to_children)};
+}
+
+auto make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    auto source_boundary = make_vector_view<NodeID>(memory_ptr, layout, name + "/source_boundary");
+    auto destination_boundary =
+        make_vector_view<NodeID>(memory_ptr, layout, name + "/destination_boundary");
+    auto cells = make_vector_view<partitioner::CellStorageView::CellData>(
+        memory_ptr, layout, name + "/cells");
+    auto level_offsets =
+        make_vector_view<std::uint64_t>(memory_ptr, layout, name + "/level_to_cell_offset");
+
+    return partitioner::CellStorageView{std::move(source_boundary),
+                                        std::move(destination_boundary),
+                                        std::move(cells),
+                                        std::move(level_offsets)};
+}
+
+auto make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    std::vector<customizer::CellMetricView> cell_metric_excludes;
+
+    std::vector<std::string> metric_prefix_names;
+    layout.List(name + "/exclude/", std::back_inserter(metric_prefix_names));
+    for (const auto &prefix : metric_prefix_names)
+    {
+        auto weights_block_id = prefix + "/weights";
+        auto durations_block_id = prefix + "/durations";
+
+        auto weights = make_vector_view<EdgeWeight>(memory_ptr, layout, weights_block_id);
+        auto durations = make_vector_view<EdgeDuration>(memory_ptr, layout, durations_block_id);
+
+        cell_metric_excludes.push_back(
+            customizer::CellMetricView{std::move(weights), std::move(durations)});
+    }
+
+    return cell_metric_excludes;
+}
+
+auto make_multi_level_graph_view(char *memory_ptr,
+                                 const DataLayout &layout,
+                                 const std::string &name)
+{
+    auto node_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>(
+        memory_ptr, layout, name + "/node_array");
+    auto edge_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry>(
+        memory_ptr, layout, name + "/edge_array");
+    auto node_to_offset = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset>(
+        memory_ptr, layout, name + "/node_to_edge_offset");
+
+    return customizer::MultiLevelEdgeBasedGraphView(
+        std::move(node_list), std::move(edge_list), std::move(node_to_offset));
+}
+
+auto make_maneuver_overrides_views(char *memory_ptr,
+                                   const DataLayout &layout,
+                                   const std::string &name)
+{
+    auto maneuver_overrides = make_vector_view<extractor::StorageManeuverOverride>(
+        memory_ptr, layout, name + "/overrides");
+    auto maneuver_override_node_sequences =
+        make_vector_view<NodeID>(memory_ptr, layout, name + "/node_sequences");
+
+    return std::make_tuple(maneuver_overrides, maneuver_override_node_sequences);
+}
 }
 }
 
diff --git a/include/util/packed_vector.hpp b/include/util/packed_vector.hpp
index a3ca47f7d84..94ee4f19f78 100644
--- a/include/util/packed_vector.hpp
+++ b/include/util/packed_vector.hpp
@@ -243,6 +243,7 @@ template <typename T, std::size_t Bits, storage::Ownership Ownership> class Pack
 
   public:
     using value_type = T;
+    static constexpr std::size_t value_size = Bits;
     using block_type = WordT;
 
     class internal_reference
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index 4ad84c26907..eeb54292e0e 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -8,44 +8,15 @@
 #include "storage/view_factory.hpp"
 
 #include "contractor/files.hpp"
-#include "contractor/query_graph.hpp"
-
-#include "customizer/edge_based_graph.hpp"
 #include "customizer/files.hpp"
-
-#include "extractor/class_data.hpp"
-#include "extractor/compressed_edge_container.hpp"
-#include "extractor/edge_based_edge.hpp"
-#include "extractor/edge_based_node.hpp"
 #include "extractor/files.hpp"
-#include "extractor/maneuver_override.hpp"
-#include "extractor/name_table.hpp"
-#include "extractor/packed_osm_ids.hpp"
-#include "extractor/profile_properties.hpp"
-#include "extractor/query_node.hpp"
-#include "extractor/travel_mode.hpp"
-
 #include "guidance/files.hpp"
-#include "guidance/turn_instruction.hpp"
-
-#include "partitioner/cell_storage.hpp"
-#include "partitioner/edge_based_graph_reader.hpp"
 #include "partitioner/files.hpp"
-#include "partitioner/multi_level_partition.hpp"
-
-#include "engine/datafacade/datafacade_base.hpp"
 
-#include "util/coordinate.hpp"
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
 #include "util/fingerprint.hpp"
 #include "util/log.hpp"
-#include "util/packed_vector.hpp"
-#include "util/range_table.hpp"
-#include "util/static_graph.hpp"
-#include "util/static_rtree.hpp"
-#include "util/typedefs.hpp"
-#include "util/vector_view.hpp"
 
 #ifdef __linux__
 #include <sys/mman.h>
@@ -90,11 +61,6 @@ inline void readBlocks(const boost::filesystem::path &path, DataLayout &layout)
 }
 }
 
-using RTreeLeaf = engine::datafacade::BaseDataFacade::RTreeLeaf;
-using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
-using QueryGraph = util::StaticGraph<contractor::QueryEdge::EdgeData>;
-using EdgeBasedGraph = util::StaticGraph<extractor::EdgeBasedEdge::EdgeData>;
-
 using Monitor = SharedMonitor<SharedDataTimestamp>;
 
 Storage::Storage(StorageConfig config_) : config(std::move(config_)) {}
@@ -312,69 +278,33 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Name data
     {
-        auto blocks = make_vector_view<extractor::NameTableView::IndexedData::BlockReference>(
-            memory_ptr, layout, "/common/names/blocks");
-        auto values = make_vector_view<extractor::NameTableView::IndexedData::ValueType>(
-            memory_ptr, layout, "/common/names/values");
-
-        extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
-        extractor::NameTableView name_table{index_data_view};
+        auto name_table = make_name_table_view(memory_ptr, layout, "/common/names");
         extractor::files::readNames(config.GetPath(".osrm.names"), name_table);
     }
 
     // Turn lane data
     {
-        auto turn_lane_data = make_vector_view<util::guidance::LaneTupleIdPair>(
-            memory_ptr, layout, "/common/turn_lanes/data");
-
+        auto turn_lane_data = make_lane_data_view(memory_ptr, layout, "/common/turn_lanes");
         extractor::files::readTurnLaneData(config.GetPath(".osrm.tld"), turn_lane_data);
     }
 
     // Turn lane descriptions
     {
-        auto offsets =
-            make_vector_view<std::uint32_t>(memory_ptr, layout, "/common/turn_lanes/offsets");
-        auto masks = make_vector_view<extractor::TurnLaneType::Mask>(
-            memory_ptr, layout, "/common/turn_lanes/masks");
-
-        extractor::files::readTurnLaneDescriptions(config.GetPath(".osrm.tls"), offsets, masks);
+        auto views = make_turn_lane_description_views(memory_ptr, layout, "/common/turn_lanes");
+        extractor::files::readTurnLaneDescriptions(
+            config.GetPath(".osrm.tls"), std::get<0>(views), std::get<1>(views));
     }
 
     // Load edge-based nodes data
     {
-        auto edge_based_node_data = make_vector_view<extractor::EdgeBasedNode>(
-            memory_ptr, layout, "/common/ebg_node_data/nodes");
-        auto annotation_data = make_vector_view<extractor::NodeBasedEdgeAnnotation>(
-            memory_ptr, layout, "/common/ebg_node_data/annotations");
-
-        extractor::EdgeBasedNodeDataView node_data(std::move(edge_based_node_data),
-                                                   std::move(annotation_data));
-
+        auto node_data = make_ebn_data_view(memory_ptr, layout, "/common/ebg_node_data");
         extractor::files::readNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
     }
 
     // Load original edge data
     {
-        auto lane_data_ids =
-            make_vector_view<LaneDataID>(memory_ptr, layout, "/common/turn_data/lane_data_ids");
+        auto turn_data = make_turn_data_view(memory_ptr, layout, "/common/turn_data");
 
-        const auto turn_instructions = make_vector_view<guidance::TurnInstruction>(
-            memory_ptr, layout, "/common/turn_data/turn_instructions");
-
-        const auto entry_class_ids =
-            make_vector_view<EntryClassID>(memory_ptr, layout, "/common/turn_data/entry_class_ids");
-
-        const auto pre_turn_bearings = make_vector_view<guidance::TurnBearing>(
-            memory_ptr, layout, "/common/turn_data/pre_turn_bearings");
-
-        const auto post_turn_bearings = make_vector_view<guidance::TurnBearing>(
-            memory_ptr, layout, "/common/turn_data/post_turn_bearings");
-
-        guidance::TurnDataView turn_data(std::move(turn_instructions),
-                                         std::move(lane_data_ids),
-                                         std::move(entry_class_ids),
-                                         std::move(pre_turn_bearings),
-                                         std::move(post_turn_bearings));
         auto connectivity_checksum_ptr =
             layout.GetBlockPtr<std::uint32_t>(memory_ptr, "/common/connectivity_checksum");
 
@@ -386,48 +316,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // load compressed geometry
     {
-        auto geometry_begin_indices =
-            make_vector_view<unsigned>(memory_ptr, layout, "/common/segment_data/index");
-
-        auto node_list = make_vector_view<NodeID>(memory_ptr, layout, "/common/segment_data/nodes");
-
-        auto num_entries = layout.GetBlockEntries("/common/segment_data/nodes");
-
-        extractor::SegmentDataView::SegmentWeightVector fwd_weight_list(
-            make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                memory_ptr, layout, "/common/segment_data/forward_weights/packed"),
-            num_entries);
-
-        extractor::SegmentDataView::SegmentWeightVector rev_weight_list(
-            make_vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                memory_ptr, layout, "/common/segment_data/reverse_weights/packed"),
-            num_entries);
-
-        extractor::SegmentDataView::SegmentDurationVector fwd_duration_list(
-            make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                memory_ptr, layout, "/common/segment_data/forward_durations/packed"),
-            num_entries);
-
-        extractor::SegmentDataView::SegmentDurationVector rev_duration_list(
-            make_vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                memory_ptr, layout, "/common/segment_data/reverse_durations/packed"),
-            num_entries);
-
-        auto fwd_datasources_list = make_vector_view<DatasourceID>(
-            memory_ptr, layout, "/common/segment_data/forward_data_sources");
-
-        auto rev_datasources_list = make_vector_view<DatasourceID>(
-            memory_ptr, layout, "/common/segment_data/reverse_data_sources");
-
-        extractor::SegmentDataView segment_data{std::move(geometry_begin_indices),
-                                                std::move(node_list),
-                                                std::move(fwd_weight_list),
-                                                std::move(rev_weight_list),
-                                                std::move(fwd_duration_list),
-                                                std::move(rev_duration_list),
-                                                std::move(fwd_datasources_list),
-                                                std::move(rev_datasources_list)};
-
+        auto segment_data = make_segment_data_view(memory_ptr, layout, "/common/segment_data");
         extractor::files::readSegmentData(config.GetPath(".osrm.geometry"), segment_data);
     }
 
@@ -469,21 +358,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // store search tree portion of rtree
     {
-
-        const auto search_tree =
-            make_vector_view<RTreeNode>(memory_ptr, layout, "/common/rtree/search_tree");
-
-        const auto rtree_level_starts = make_vector_view<std::uint64_t>(
-            memory_ptr, layout, "/common/rtree/search_tree_level_starts");
-
-        // we need this purely for the interface
-        util::vector_view<util::Coordinate> empty_coords;
-
-        util::StaticRTree<RTreeLeaf, storage::Ownership::View> rtree{
-            std::move(search_tree),
-            std::move(rtree_level_starts),
-            config.GetPath(".osrm.fileIndex"),
-            empty_coords};
+        auto rtree = make_search_tree_view(memory_ptr, layout, "/common/rtree");
         extractor::files::readRamIndex(config.GetPath(".osrm.ramIndex"), rtree);
     }
 
@@ -501,25 +376,9 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Load intersection data
     {
-        auto bearing_offsets = make_vector_view<unsigned>(
-            memory_ptr, layout, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
-        auto bearing_blocks =
-            make_vector_view<util::RangeTable<16, storage::Ownership::View>::BlockT>(
-                memory_ptr, layout, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
-        auto bearing_values = make_vector_view<DiscreteBearing>(
-            memory_ptr, layout, "/common/intersection_bearings/bearing_values");
-        util::RangeTable<16, storage::Ownership::View> bearing_range_table(
-            std::move(bearing_offsets),
-            std::move(bearing_blocks),
-            static_cast<unsigned>(bearing_values.size()));
-
-        auto bearing_class_id = make_vector_view<BearingClassID>(
-            memory_ptr, layout, "/common/intersection_bearings/node_to_class_id");
-        extractor::IntersectionBearingsView intersection_bearings_view{
-            std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
-
-        auto entry_classes = make_vector_view<util::guidance::EntryClass>(
-            memory_ptr, layout, "/common/entry_classes");
+        auto intersection_bearings_view =
+            make_intersection_bearings_view(memory_ptr, layout, "/common/intersection_bearings");
+        auto entry_classes = make_entry_classes_view(memory_ptr, layout, "/common/entry_classes");
         extractor::files::readIntersections(
             config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes);
     }
@@ -527,24 +386,14 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     if (boost::filesystem::exists(config.GetPath(".osrm.hsgr")))
     {
         const std::string metric_prefix = "/ch/metrics/" + metric_name;
-
-        auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
-            memory_ptr, layout, metric_prefix + "/contracted_graph/node_array");
-        auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>(
-            memory_ptr, layout, metric_prefix + "/contracted_graph/edge_array");
-
-        std::vector<util::vector_view<bool>> edge_filter;
-        layout.List(metric_prefix + "/exclude",
-                    boost::make_function_output_iterator([&](const auto &name) {
-                        edge_filter.push_back(make_vector_view<bool>(memory_ptr, layout, name));
-                    }));
-
-        std::uint32_t graph_connectivity_checksum = 0;
+        auto contracted_metric = make_contracted_metric_view(memory_ptr, layout, metric_prefix);
         std::unordered_map<std::string, contractor::ContractedMetricView> metrics = {
-            {metric_name, {{std::move(node_list), std::move(edge_list)}, std::move(edge_filter)}}};
+            {metric_name, std::move(contracted_metric)}};
 
+        std::uint32_t graph_connectivity_checksum = 0;
         contractor::files::readGraph(
             config.GetPath(".osrm.hsgr"), metrics, graph_connectivity_checksum);
+
         if (turns_connectivity_checksum != graph_connectivity_checksum)
         {
             throw util::exception(
@@ -557,83 +406,30 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     if (boost::filesystem::exists(config.GetPath(".osrm.partition")))
     {
-        BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
-        BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
-        BOOST_ASSERT(layout.GetBlockSize("/mld/multilevelpartition/partition") > 0);
-
-        auto level_data_ptr = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
-            memory_ptr, "/mld/multilevelpartition/level_data");
-        auto partition =
-            make_vector_view<PartitionID>(memory_ptr, layout, "/mld/multilevelpartition/partition");
-        auto cell_to_children = make_vector_view<CellID>(
-            memory_ptr, layout, "/mld/multilevelpartition/cell_to_children");
-
-        partitioner::MultiLevelPartitionView mlp{
-            level_data_ptr, std::move(partition), std::move(cell_to_children)};
+        auto mlp = make_partition_view(memory_ptr, layout, "/mld/multilevelpartition");
         partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
     }
 
     if (boost::filesystem::exists(config.GetPath(".osrm.cells")))
     {
-        BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
-        BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
-
-        auto source_boundary =
-            make_vector_view<NodeID>(memory_ptr, layout, "/mld/cellstorage/source_boundary");
-        auto destination_boundary =
-            make_vector_view<NodeID>(memory_ptr, layout, "/mld/cellstorage/destination_boundary");
-        auto cells = make_vector_view<partitioner::CellStorageView::CellData>(
-            memory_ptr, layout, "/mld/cellstorage/cells");
-        auto level_offsets = make_vector_view<std::uint64_t>(
-            memory_ptr, layout, "/mld/cellstorage/level_to_cell_offset");
-
-        partitioner::CellStorageView storage{std::move(source_boundary),
-                                             std::move(destination_boundary),
-                                             std::move(cells),
-                                             std::move(level_offsets)};
+        auto storage = make_cell_storage_view(memory_ptr, layout, "/mld/cellstorage");
         partitioner::files::readCells(config.GetPath(".osrm.cells"), storage);
     }
 
     if (boost::filesystem::exists(config.GetPath(".osrm.cell_metrics")))
     {
-        BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/cells") > 0);
-        BOOST_ASSERT(layout.GetBlockSize("/mld/cellstorage/level_to_cell_offset") > 0);
-
+        auto exclude_metrics =
+            make_cell_metric_view(memory_ptr, layout, "/mld/metrics/" + metric_name);
         std::unordered_map<std::string, std::vector<customizer::CellMetricView>> metrics = {
-            {metric_name, {}},
+            {metric_name, std::move(exclude_metrics)},
         };
-
-        std::vector<std::string> metric_prefix_names;
-        layout.List("/mld/metrics/" + metric_name + "/exclude/",
-                    std::back_inserter(metric_prefix_names));
-        for (const auto &prefix : metric_prefix_names)
-        {
-            auto weights_block_id = prefix + "/weights";
-            auto durations_block_id = prefix + "/durations";
-
-            auto weights = make_vector_view<EdgeWeight>(memory_ptr, layout, weights_block_id);
-            auto durations = make_vector_view<EdgeDuration>(memory_ptr, layout, durations_block_id);
-
-            metrics[metric_name].push_back(
-                customizer::CellMetricView{std::move(weights), std::move(durations)});
-        }
-
         customizer::files::readCellMetrics(config.GetPath(".osrm.cell_metrics"), metrics);
     }
 
     if (boost::filesystem::exists(config.GetPath(".osrm.mldgr")))
     {
-        auto node_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>(
-            memory_ptr, layout, "/mld/multilevelgraph/node_array");
-        auto edge_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeArrayEntry>(
-            memory_ptr, layout, "/mld/multilevelgraph/edge_array");
-        auto node_to_offset =
-            make_vector_view<customizer::MultiLevelEdgeBasedGraphView::EdgeOffset>(
-                memory_ptr, layout, "/mld/multilevelgraph/node_to_edge_offset");
-
+        auto graph_view = make_multi_level_graph_view(memory_ptr, layout, "/mld/multilevelgraph");
         std::uint32_t graph_connectivity_checksum = 0;
-        customizer::MultiLevelEdgeBasedGraphView graph_view(
-            std::move(node_list), std::move(edge_list), std::move(node_to_offset));
         partitioner::files::readGraph(
             config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum);
 
@@ -649,14 +445,10 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // load maneuver overrides
     {
-        auto maneuver_overrides = make_vector_view<extractor::StorageManeuverOverride>(
-            memory_ptr, layout, "/common/maneuver_overrides/overrides");
-        auto maneuver_override_node_sequences = make_vector_view<NodeID>(
-            memory_ptr, layout, "/common/maneuver_overrides/node_sequences");
-
-        extractor::files::readManeuverOverrides(config.GetPath(".osrm.maneuver_overrides"),
-                                                maneuver_overrides,
-                                                maneuver_override_node_sequences);
+        auto views =
+            make_maneuver_overrides_views(memory_ptr, layout, "/common/maneuver_overrides");
+        extractor::files::readManeuverOverrides(
+            config.GetPath(".osrm.maneuver_overrides"), std::get<0>(views), std::get<1>(views));
     }
 }
 }

From 28a4397762f361484ccd3afa5538379d2253e4c6 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Tue, 3 Apr 2018 15:16:53 +0000
Subject: [PATCH 17/19] Make use of the view factory in the data facade

---
 .../contiguous_internalmem_datafacade.hpp     | 523 ++----------------
 include/extractor/files.hpp                   |  10 +-
 include/storage/view_factory.hpp              | 111 +++-
 include/util/static_rtree.hpp                 |  10 +-
 src/storage/storage.cpp                       |  16 +-
 5 files changed, 138 insertions(+), 532 deletions(-)

diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
index ec3a4491bbc..4281e295ef8 100644
--- a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
+++ b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
@@ -9,44 +9,13 @@
 #include "engine/approach.hpp"
 #include "engine/geospatial_query.hpp"
 
-#include "customizer/edge_based_graph.hpp"
-
-#include "extractor/datasources.hpp"
-#include "extractor/edge_based_node.hpp"
-#include "extractor/intersection_bearings_container.hpp"
-#include "extractor/maneuver_override.hpp"
-#include "extractor/name_table.hpp"
-#include "extractor/node_data_container.hpp"
-#include "extractor/packed_osm_ids.hpp"
-#include "extractor/profile_properties.hpp"
-#include "extractor/segment_data_container.hpp"
-#include "extractor/turn_lane_types.hpp"
-
-#include "guidance/turn_bearing.hpp"
-#include "guidance/turn_data_container.hpp"
-#include "guidance/turn_instruction.hpp"
-
-#include "contractor/query_graph.hpp"
-
-#include "partitioner/cell_storage.hpp"
-#include "partitioner/multi_level_partition.hpp"
-
 #include "storage/shared_datatype.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/view_factory.hpp"
 
 #include "util/exception.hpp"
 #include "util/exception_utils.hpp"
-#include "util/filtered_graph.hpp"
-#include "util/guidance/bearing_class.hpp"
-#include "util/guidance/entry_class.hpp"
-#include "util/guidance/turn_lanes.hpp"
 #include "util/log.hpp"
-#include "util/packed_vector.hpp"
-#include "util/range_table.hpp"
-#include "util/rectangle.hpp"
-#include "util/static_graph.hpp"
-#include "util/static_rtree.hpp"
-#include "util/typedefs.hpp"
 
 #include <boost/assert.hpp>
 
@@ -81,36 +50,6 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
     // allocator that keeps the allocation data
     std::shared_ptr<ContiguousBlockAllocator> allocator;
 
-    void InitializeGraphPointer(const storage::DataLayout &data_layout,
-                                char *memory_block,
-                                const std::string &metric_name,
-                                const std::size_t exclude_index)
-    {
-        const std::string metric_prefix = "/ch/metrics/" + metric_name;
-        auto graph_nodes_ptr = data_layout.GetBlockPtr<GraphNode>(
-            memory_block, metric_prefix + "/contracted_graph/node_array");
-
-        auto graph_edges_ptr = data_layout.GetBlockPtr<GraphEdge>(
-            memory_block, metric_prefix + "/contracted_graph/edge_array");
-
-        auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index);
-        auto filter_block_id = exclude_prefix + "/edge_filter";
-
-        auto edge_filter_ptr =
-            data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
-
-        util::vector_view<GraphNode> node_list(
-            graph_nodes_ptr,
-            data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
-        util::vector_view<GraphEdge> edge_list(
-            graph_edges_ptr,
-            data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));
-
-        util::vector_view<bool> edge_filter(edge_filter_ptr,
-                                            data_layout.GetBlockEntries(filter_block_id));
-        m_query_graph = QueryGraph({node_list, edge_list}, edge_filter);
-    }
-
   public:
     ContiguousInternalMemoryAlgorithmDataFacade(
         std::shared_ptr<ContiguousBlockAllocator> allocator_,
@@ -127,7 +66,8 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
                                     const std::string &metric_name,
                                     const std::size_t exclude_index)
     {
-        InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index);
+        m_query_graph = make_filtered_graph_view(
+            memory_block, data_layout, "/ch/metrics/" + metric_name, exclude_index);
     }
 
     // search graph access
@@ -218,7 +158,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     util::vector_view<extractor::StorageManeuverOverride> m_maneuver_overrides;
     util::vector_view<NodeID> m_maneuver_override_node_sequences;
 
-    std::unique_ptr<SharedRTree> m_static_rtree;
+    SharedRTree m_static_rtree;
     std::unique_ptr<SharedGeospatialQuery> m_geospatial_query;
     boost::filesystem::path file_index_path;
 
@@ -232,330 +172,55 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
     // allocator that keeps the allocation data
     std::shared_ptr<ContiguousBlockAllocator> allocator;
 
-    void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
-                                            char *memory_block,
-                                            const std::string &metric_name,
-                                            const std::size_t exclude_index)
+    void InitializeInternalPointers(const storage::DataLayout &layout,
+                                    char *memory_ptr,
+                                    const std::string &metric_name,
+                                    const std::size_t exclude_index)
     {
         // TODO: For multi-metric support we need to have separate exclude classes per metric
         (void)metric_name;
-        m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
-            memory_block, "/common/properties");
+
+        m_profile_properties =
+            layout.GetBlockPtr<extractor::ProfileProperties>(memory_ptr, "/common/properties");
 
         exclude_mask = m_profile_properties->excludable_classes[exclude_index];
-    }
 
-    void InitializeChecksumPointer(const storage::DataLayout &data_layout, char *memory_block)
-    {
         m_check_sum =
-            *data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/connectivity_checksum");
-        util::Log() << "set checksum: " << m_check_sum;
-    }
+            *layout.GetBlockPtr<std::uint32_t>(memory_ptr, "/common/connectivity_checksum");
 
-    void InitializeRTreePointers(const storage::DataLayout &data_layout, char *memory_block)
-    {
-        BOOST_ASSERT_MSG(!m_coordinate_list.empty(), "coordinates must be loaded before r-tree");
+        std::tie(m_coordinate_list, m_osmnodeid_list) =
+            make_nbn_data_view(memory_ptr, layout, "/common/nbn_data");
 
-        const auto file_index_ptr =
-            data_layout.GetBlockPtr<char>(memory_block, "/common/rtree/file_index_path");
-        file_index_path = boost::filesystem::path(file_index_ptr);
-        if (!boost::filesystem::exists(file_index_path))
-        {
-            util::Log(logDEBUG) << "Leaf file name " << file_index_path.string();
-            throw util::exception("Could not load " + file_index_path.string() +
-                                  "Is any data loaded into shared memory?" + SOURCE_REF);
-        }
-
-        const auto rtree_ptr =
-            data_layout.GetBlockPtr<RTreeNode>(memory_block, "/common/rtree/search_tree");
-        util::vector_view<RTreeNode> search_tree(
-            rtree_ptr, data_layout.GetBlockEntries("/common/rtree/search_tree"));
-
-        const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr<std::uint64_t>(
-            memory_block, "/common/rtree/search_tree_level_starts");
-        util::vector_view<std::uint64_t> rtree_level_starts(
-            rtree_levelstarts_ptr,
-            data_layout.GetBlockEntries("/common/rtree/search_tree_level_starts"));
-
-        m_static_rtree.reset(new SharedRTree{std::move(search_tree),
-                                             std::move(rtree_level_starts),
-                                             file_index_path,
-                                             m_coordinate_list});
+        m_static_rtree = make_search_tree_view(memory_ptr, layout, "/common/rtree");
         m_geospatial_query.reset(
-            new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
-    }
+            new SharedGeospatialQuery(m_static_rtree, m_coordinate_list, *this));
 
-    void InitializeNodeInformationPointers(const storage::DataLayout &layout, char *memory_ptr)
-    {
-        const auto coordinate_list_ptr =
-            layout.GetBlockPtr<util::Coordinate>(memory_ptr, "/common/coordinates");
-        m_coordinate_list.reset(coordinate_list_ptr, layout.GetBlockEntries("/common/coordinates"));
+        edge_based_node_data = make_ebn_data_view(memory_ptr, layout, "/common/ebg_node_data");
 
-        const auto osmnodeid_ptr = layout.GetBlockPtr<extractor::PackedOSMIDsView::block_type>(
-            memory_ptr, "/common/osm_node_ids/packed");
-        m_osmnodeid_list = extractor::PackedOSMIDsView(
-            util::vector_view<extractor::PackedOSMIDsView::block_type>(
-                osmnodeid_ptr, layout.GetBlockEntries("/common/osm_node_ids/packed")),
-            // We (ab)use the number of coordinates here because we know we have the same amount of
-            // ids
-            layout.GetBlockEntries("/common/coordinates"));
-    }
+        turn_data = make_turn_data_view(memory_ptr, layout, "/common/turn_data");
 
-    void InitializeEdgeBasedNodeDataInformationPointers(const storage::DataLayout &layout,
-                                                        char *memory_ptr)
-    {
-        const auto edge_based_node_list_ptr =
-            layout.GetBlockPtr<extractor::EdgeBasedNode>(memory_ptr, "/common/ebg_node_data/nodes");
-        util::vector_view<extractor::EdgeBasedNode> edge_based_node_data_list(
-            edge_based_node_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/nodes"));
+        m_name_table = make_name_table_view(memory_ptr, layout, "/common/names");
 
-        const auto annotation_data_list_ptr =
-            layout.GetBlockPtr<extractor::NodeBasedEdgeAnnotation>(
-                memory_ptr, "/common/ebg_node_data/annotations");
-        util::vector_view<extractor::NodeBasedEdgeAnnotation> annotation_data(
-            annotation_data_list_ptr, layout.GetBlockEntries("/common/ebg_node_data/annotations"));
+        std::tie(m_lane_description_offsets, m_lane_description_masks) =
+            make_turn_lane_description_views(memory_ptr, layout, "/common/turn_lanes");
+        m_lane_tupel_id_pairs = make_lane_data_view(memory_ptr, layout, "/common/turn_lanes");
 
-        edge_based_node_data = extractor::EdgeBasedNodeDataView(
-            std::move(edge_based_node_data_list), std::move(annotation_data));
-    }
+        m_turn_weight_penalties = make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty");
+        m_turn_duration_penalties =
+            make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty");
 
-    void InitializeEdgeInformationPointers(const storage::DataLayout &layout, char *memory_ptr)
-    {
-        const auto lane_data_id_ptr =
-            layout.GetBlockPtr<LaneDataID>(memory_ptr, "/common/turn_data/lane_data_ids");
-        util::vector_view<LaneDataID> lane_data_ids(
-            lane_data_id_ptr, layout.GetBlockEntries("/common/turn_data/lane_data_ids"));
+        segment_data = make_segment_data_view(memory_ptr, layout, "/common/segment_data");
 
-        const auto turn_instruction_list_ptr = layout.GetBlockPtr<guidance::TurnInstruction>(
-            memory_ptr, "/common/turn_data/turn_instructions");
-        util::vector_view<guidance::TurnInstruction> turn_instructions(
-            turn_instruction_list_ptr,
-            layout.GetBlockEntries("/common/turn_data/turn_instructions"));
+        m_datasources =
+            layout.GetBlockPtr<extractor::Datasources>(memory_ptr, "/common/data_sources_names");
 
-        const auto entry_class_id_list_ptr =
-            layout.GetBlockPtr<EntryClassID>(memory_ptr, "/common/turn_data/entry_class_ids");
-        util::vector_view<EntryClassID> entry_class_ids(
-            entry_class_id_list_ptr, layout.GetBlockEntries("/common/turn_data/entry_class_ids"));
-
-        const auto pre_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>(
-            memory_ptr, "/common/turn_data/pre_turn_bearings");
-        util::vector_view<guidance::TurnBearing> pre_turn_bearings(
-            pre_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/pre_turn_bearings"));
-
-        const auto post_turn_bearing_ptr = layout.GetBlockPtr<guidance::TurnBearing>(
-            memory_ptr, "/common/turn_data/post_turn_bearings");
-        util::vector_view<guidance::TurnBearing> post_turn_bearings(
-            post_turn_bearing_ptr, layout.GetBlockEntries("/common/turn_data/post_turn_bearings"));
-
-        turn_data = guidance::TurnDataView(std::move(turn_instructions),
-                                           std::move(lane_data_ids),
-                                           std::move(entry_class_ids),
-                                           std::move(pre_turn_bearings),
-                                           std::move(post_turn_bearings));
-    }
-
-    void InitializeNamePointers(const storage::DataLayout &data_layout, char *memory_block)
-    {
-        const auto name_blocks_ptr =
-            data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>(
-                memory_block, "/common/names/blocks");
-        const auto name_values_ptr =
-            data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>(
-                memory_block, "/common/names/values");
-
-        util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
-            name_blocks_ptr, data_layout.GetBlockEntries("/common/names/blocks"));
-        util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
-            name_values_ptr, data_layout.GetBlockEntries("/common/names/values"));
-
-        extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
-        m_name_table = extractor::NameTableView{std::move(index_data_view)};
-    }
-
-    void InitializeTurnLaneDescriptionsPointers(const storage::DataLayout &data_layout,
-                                                char *memory_block)
-    {
-        auto offsets_ptr =
-            data_layout.GetBlockPtr<std::uint32_t>(memory_block, "/common/turn_lanes/offsets");
-        util::vector_view<std::uint32_t> offsets(
-            offsets_ptr, data_layout.GetBlockEntries("/common/turn_lanes/offsets"));
-        m_lane_description_offsets = std::move(offsets);
-
-        auto masks_ptr = data_layout.GetBlockPtr<extractor::TurnLaneType::Mask>(
-            memory_block, "/common/turn_lanes/masks");
-
-        util::vector_view<extractor::TurnLaneType::Mask> masks(
-            masks_ptr, data_layout.GetBlockEntries("/common/turn_lanes/masks"));
-        m_lane_description_masks = std::move(masks);
-
-        const auto lane_tupel_id_pair_ptr =
-            data_layout.GetBlockPtr<util::guidance::LaneTupleIdPair>(memory_block,
-                                                                     "/common/turn_lanes/data");
-        util::vector_view<util::guidance::LaneTupleIdPair> lane_tupel_id_pair(
-            lane_tupel_id_pair_ptr, data_layout.GetBlockEntries("/common/turn_lanes/data"));
-        m_lane_tupel_id_pairs = std::move(lane_tupel_id_pair);
-    }
-
-    void InitializeTurnPenalties(const storage::DataLayout &data_layout, char *memory_block)
-    {
-        auto turn_weight_penalties_ptr =
-            data_layout.GetBlockPtr<TurnPenalty>(memory_block, "/common/turn_penalty/weight");
-        m_turn_weight_penalties = util::vector_view<TurnPenalty>(
-            turn_weight_penalties_ptr, data_layout.GetBlockEntries("/common/turn_penalty/weight"));
-        auto turn_duration_penalties_ptr =
-            data_layout.GetBlockPtr<TurnPenalty>(memory_block, "/common/turn_penalty/duration");
-        m_turn_duration_penalties = util::vector_view<TurnPenalty>(
-            turn_duration_penalties_ptr,
-            data_layout.GetBlockEntries("/common/turn_penalty/duration"));
-    }
-
-    void InitializeGeometryPointers(const storage::DataLayout &data_layout, char *memory_block)
-    {
-        auto geometries_index_ptr =
-            data_layout.GetBlockPtr<unsigned>(memory_block, "/common/segment_data/index");
-        util::vector_view<unsigned> geometry_begin_indices(
-            geometries_index_ptr, data_layout.GetBlockEntries("/common/segment_data/index"));
-
-        auto num_entries = data_layout.GetBlockEntries("/common/segment_data/nodes");
-        auto geometries_node_list_ptr =
-            data_layout.GetBlockPtr<NodeID>(memory_block, "/common/segment_data/nodes");
-        util::vector_view<NodeID> geometry_node_list(geometries_node_list_ptr, num_entries);
-
-        auto geometries_fwd_weight_list_ptr =
-            data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                memory_block, "/common/segment_data/forward_weights/packed");
-        extractor::SegmentDataView::SegmentWeightVector geometry_fwd_weight_list(
-            util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                geometries_fwd_weight_list_ptr,
-                data_layout.GetBlockEntries("/common/segment_data/forward_weights/packed")),
-            num_entries);
-
-        auto geometries_rev_weight_list_ptr =
-            data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                memory_block, "/common/segment_data/reverse_weights/packed");
-        extractor::SegmentDataView::SegmentWeightVector geometry_rev_weight_list(
-            util::vector_view<extractor::SegmentDataView::SegmentWeightVector::block_type>(
-                geometries_rev_weight_list_ptr,
-                data_layout.GetBlockEntries("/common/segment_data/reverse_weights/packed")),
-            num_entries);
-
-        auto geometries_fwd_duration_list_ptr =
-            data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                memory_block, "/common/segment_data/forward_durations/packed");
-        extractor::SegmentDataView::SegmentDurationVector geometry_fwd_duration_list(
-            util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                geometries_fwd_duration_list_ptr,
-                data_layout.GetBlockEntries("/common/segment_data/forward_durations/packed")),
-            num_entries);
-
-        auto geometries_rev_duration_list_ptr =
-            data_layout.GetBlockPtr<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                memory_block, "/common/segment_data/reverse_durations/packed");
-        extractor::SegmentDataView::SegmentDurationVector geometry_rev_duration_list(
-            util::vector_view<extractor::SegmentDataView::SegmentDurationVector::block_type>(
-                geometries_rev_duration_list_ptr,
-                data_layout.GetBlockEntries("/common/segment_data/reverse_durations/packed")),
-            num_entries);
-
-        auto geometries_fwd_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
-            memory_block, "/common/segment_data/forward_data_sources");
-        util::vector_view<DatasourceID> geometry_fwd_datasources_list(
-            geometries_fwd_datasources_list_ptr,
-            data_layout.GetBlockEntries("/common/segment_data/forward_data_sources"));
-
-        auto geometries_rev_datasources_list_ptr = data_layout.GetBlockPtr<DatasourceID>(
-            memory_block, "/common/segment_data/reverse_data_sources");
-        util::vector_view<DatasourceID> geometry_rev_datasources_list(
-            geometries_rev_datasources_list_ptr,
-            data_layout.GetBlockEntries("/common/segment_data/reverse_data_sources"));
-
-        segment_data = extractor::SegmentDataView{std::move(geometry_begin_indices),
-                                                  std::move(geometry_node_list),
-                                                  std::move(geometry_fwd_weight_list),
-                                                  std::move(geometry_rev_weight_list),
-                                                  std::move(geometry_fwd_duration_list),
-                                                  std::move(geometry_rev_duration_list),
-                                                  std::move(geometry_fwd_datasources_list),
-                                                  std::move(geometry_rev_datasources_list)};
-
-        m_datasources = data_layout.GetBlockPtr<extractor::Datasources>(
-            memory_block, "/common/data_sources_names");
-    }
-
-    void InitializeIntersectionClassPointers(const storage::DataLayout &data_layout,
-                                             char *memory_block)
-    {
-        auto bearing_class_id_ptr = data_layout.GetBlockPtr<BearingClassID>(
-            memory_block, "/common/intersection_bearings/node_to_class_id");
-        util::vector_view<BearingClassID> bearing_class_id(
-            bearing_class_id_ptr,
-            data_layout.GetBlockEntries("/common/intersection_bearings/node_to_class_id"));
-
-        auto bearing_values_ptr = data_layout.GetBlockPtr<DiscreteBearing>(
-            memory_block, "/common/intersection_bearings/bearing_values");
-        util::vector_view<DiscreteBearing> bearing_values(
-            bearing_values_ptr,
-            data_layout.GetBlockEntries("/common/intersection_bearings/bearing_values"));
-
-        auto offsets_ptr = data_layout.GetBlockPtr<unsigned>(
-            memory_block, "/common/intersection_bearings/class_id_to_ranges/block_offsets");
-        auto blocks_ptr = data_layout.GetBlockPtr<IndexBlock>(
-            memory_block, "/common/intersection_bearings/class_id_to_ranges/diff_blocks");
-        util::vector_view<unsigned> bearing_offsets(
-            offsets_ptr,
-            data_layout.GetBlockEntries(
-                "/common/intersection_bearings/class_id_to_ranges/block_offsets"));
-        util::vector_view<IndexBlock> bearing_blocks(
-            blocks_ptr,
-            data_layout.GetBlockEntries(
-                "/common/intersection_bearings/class_id_to_ranges/diff_blocks"));
-
-        util::RangeTable<16, storage::Ownership::View> bearing_range_table(
-            bearing_offsets, bearing_blocks, static_cast<unsigned>(bearing_values.size()));
-
-        intersection_bearings_view = extractor::IntersectionBearingsView{
-            std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
-
-        auto entry_class_ptr = data_layout.GetBlockPtr<util::guidance::EntryClass>(
-            memory_block, "/common/entry_classes");
-        util::vector_view<util::guidance::EntryClass> entry_class_table(
-            entry_class_ptr, data_layout.GetBlockEntries("/common/entry_classes"));
-        m_entry_class_table = std::move(entry_class_table);
-    }
-
-    void InitializeManeuverOverridePointers(const storage::DataLayout &data_layout,
-                                            char *memory_block)
-    {
-        auto maneuver_overrides_ptr = data_layout.GetBlockPtr<extractor::StorageManeuverOverride>(
-            memory_block, "/common/maneuver_overrides/overrides");
-        m_maneuver_overrides = util::vector_view<extractor::StorageManeuverOverride>(
-            maneuver_overrides_ptr,
-            data_layout.GetBlockEntries("/common/maneuver_overrides/overrides"));
-
-        auto maneuver_override_node_sequences_ptr = data_layout.GetBlockPtr<NodeID>(
-            memory_block, "/common/maneuver_overrides/node_sequences");
-        m_maneuver_override_node_sequences = util::vector_view<NodeID>(
-            maneuver_override_node_sequences_ptr,
-            data_layout.GetBlockEntries("/common/maneuver_overrides/node_sequences"));
-    }
+        intersection_bearings_view =
+            make_intersection_bearings_view(memory_ptr, layout, "/common/intersection_bearings");
 
-    void InitializeInternalPointers(const storage::DataLayout &data_layout,
-                                    char *memory_block,
-                                    const std::string &metric_name,
-                                    const std::size_t exclude_index)
-    {
-        InitializeChecksumPointer(data_layout, memory_block);
-        InitializeNodeInformationPointers(data_layout, memory_block);
-        InitializeEdgeBasedNodeDataInformationPointers(data_layout, memory_block);
-        InitializeEdgeInformationPointers(data_layout, memory_block);
-        InitializeTurnPenalties(data_layout, memory_block);
-        InitializeGeometryPointers(data_layout, memory_block);
-        InitializeNamePointers(data_layout, memory_block);
-        InitializeTurnLaneDescriptionsPointers(data_layout, memory_block);
-        InitializeProfilePropertiesPointer(data_layout, memory_block, metric_name, exclude_index);
-        InitializeRTreePointers(data_layout, memory_block);
-        InitializeIntersectionClassPointers(data_layout, memory_block);
-        InitializeManeuverOverridePointers(data_layout, memory_block);
+        m_entry_class_table = make_entry_classes_view(memory_ptr, layout, "/common/entry_classes");
+
+        std::tie(m_maneuver_overrides, m_maneuver_override_node_sequences) =
+            make_maneuver_overrides_views(memory_ptr, layout, "/common/maneuver_overrides");
     }
 
   public:
@@ -996,122 +661,16 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
 
     QueryGraph query_graph;
 
-    void InitializeInternalPointers(const storage::DataLayout &data_layout,
-                                    char *memory_block,
+    void InitializeInternalPointers(const storage::DataLayout &layout,
+                                    char *memory_ptr,
                                     const std::string &metric_name,
                                     const std::size_t exclude_index)
     {
-        InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index);
-        InitializeGraphPointer(data_layout, memory_block);
-    }
-
-    void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
-                                   char *memory_block,
-                                   const std::string &metric_name,
-                                   const std::size_t exclude_index)
-    {
-        if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
-        {
-            BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/level_data") > 0);
-            BOOST_ASSERT(data_layout.GetBlockSize("/mld/multilevelpartition/cell_to_children") > 0);
-
-            auto level_data =
-                data_layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
-                    memory_block, "/mld/multilevelpartition/level_data");
-
-            auto mld_partition_ptr = data_layout.GetBlockPtr<PartitionID>(
-                memory_block, "/mld/multilevelpartition/partition");
-            auto partition_entries_count =
-                data_layout.GetBlockEntries("/mld/multilevelpartition/partition");
-            util::vector_view<PartitionID> partition(mld_partition_ptr, partition_entries_count);
-
-            auto mld_chilren_ptr = data_layout.GetBlockPtr<CellID>(
-                memory_block, "/mld/multilevelpartition/cell_to_children");
-            auto children_entries_count =
-                data_layout.GetBlockEntries("/mld/multilevelpartition/cell_to_children");
-            util::vector_view<CellID> cell_to_children(mld_chilren_ptr, children_entries_count);
-
-            mld_partition =
-                partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
-        }
-
-        const auto exclude_prefix =
-            "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index);
-        const auto weights_block_id = exclude_prefix + "/weights";
-        const auto durations_block_id = exclude_prefix + "/durations";
-
-        if (data_layout.GetBlockSize(weights_block_id) > 0)
-        {
-            auto mld_cell_weights_ptr =
-                data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
-            auto mld_cell_durations_ptr =
-                data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
-            auto weight_entries_count = data_layout.GetBlockEntries(weights_block_id);
-            auto duration_entries_count = data_layout.GetBlockEntries(durations_block_id);
-            BOOST_ASSERT(weight_entries_count == duration_entries_count);
-            util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
-            util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
-                                                      duration_entries_count);
-
-            mld_cell_metric = customizer::CellMetricView{std::move(weights), std::move(durations)};
-        }
-
-        if (data_layout.GetBlockSize("/mld/cellstorage/cells") > 0)
-        {
-
-            auto mld_source_boundary_ptr =
-                data_layout.GetBlockPtr<NodeID>(memory_block, "/mld/cellstorage/source_boundary");
-            auto mld_destination_boundary_ptr = data_layout.GetBlockPtr<NodeID>(
-                memory_block, "/mld/cellstorage/destination_boundary");
-            auto mld_cells_ptr = data_layout.GetBlockPtr<partitioner::CellStorageView::CellData>(
-                memory_block, "/mld/cellstorage/cells");
-            auto mld_cell_level_offsets_ptr = data_layout.GetBlockPtr<std::uint64_t>(
-                memory_block, "/mld/cellstorage/level_to_cell_offset");
-
-            auto source_boundary_entries_count =
-                data_layout.GetBlockEntries("/mld/cellstorage/source_boundary");
-            auto destination_boundary_entries_count =
-                data_layout.GetBlockEntries("/mld/cellstorage/destination_boundary");
-            auto cells_entries_counts = data_layout.GetBlockEntries("/mld/cellstorage/cells");
-            auto cell_level_offsets_entries_count =
-                data_layout.GetBlockEntries("/mld/cellstorage/level_to_cell_offset");
-
-            util::vector_view<NodeID> source_boundary(mld_source_boundary_ptr,
-                                                      source_boundary_entries_count);
-            util::vector_view<NodeID> destination_boundary(mld_destination_boundary_ptr,
-                                                           destination_boundary_entries_count);
-            util::vector_view<partitioner::CellStorageView::CellData> cells(mld_cells_ptr,
-                                                                            cells_entries_counts);
-            util::vector_view<std::uint64_t> level_offsets(mld_cell_level_offsets_ptr,
-                                                           cell_level_offsets_entries_count);
-
-            mld_cell_storage = partitioner::CellStorageView{std::move(source_boundary),
-                                                            std::move(destination_boundary),
-                                                            std::move(cells),
-                                                            std::move(level_offsets)};
-        }
-    }
-    void InitializeGraphPointer(const storage::DataLayout &data_layout, char *memory_block)
-    {
-        auto graph_nodes_ptr =
-            data_layout.GetBlockPtr<GraphNode>(memory_block, "/mld/multilevelgraph/node_array");
-
-        auto graph_edges_ptr =
-            data_layout.GetBlockPtr<GraphEdge>(memory_block, "/mld/multilevelgraph/edge_array");
-
-        auto graph_node_to_offset_ptr = data_layout.GetBlockPtr<QueryGraph::EdgeOffset>(
-            memory_block, "/mld/multilevelgraph/node_to_edge_offset");
-
-        util::vector_view<GraphNode> node_list(
-            graph_nodes_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/node_array"));
-        util::vector_view<GraphEdge> edge_list(
-            graph_edges_ptr, data_layout.GetBlockEntries("/mld/multilevelgraph/edge_array"));
-        util::vector_view<QueryGraph::EdgeOffset> node_to_offset(
-            graph_node_to_offset_ptr,
-            data_layout.GetBlockEntries("/mld/multilevelgraph/node_to_edge_offset"));
-
-        query_graph =
-            QueryGraph(std::move(node_list), std::move(edge_list), std::move(node_to_offset));
+        mld_partition = make_partition_view(memory_ptr, layout, "/mld/multilevelpartition");
+        mld_cell_metric = make_filtered_cell_metric_view(
+            memory_ptr, layout, "/mld/metrics/" + metric_name, exclude_index);
+        mld_cell_storage = make_cell_storage_view(memory_ptr, layout, "/mld/cellstorage");
+        query_graph = make_multi_level_graph_view(memory_ptr, layout, "/mld/multilevelgraph");
     }
 
     // allocator that keeps the allocation data
diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp
index 9fa805f6bc0..34ac91c627c 100644
--- a/include/extractor/files.hpp
+++ b/include/extractor/files.hpp
@@ -122,8 +122,8 @@ inline void readNodes(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
-    storage::serialization::read(reader, "/common/coordinates", coordinates);
-    util::serialization::read(reader, "/common/osm_node_ids", osm_node_ids);
+    storage::serialization::read(reader, "/common/nbn_data/coordinates", coordinates);
+    util::serialization::read(reader, "/common/nbn_data/osm_node_ids", osm_node_ids);
 }
 
 // reads only coordinates from .osrm.nbg_nodes
@@ -135,7 +135,7 @@ inline void readNodeCoordinates(const boost::filesystem::path &path, Coordinates
     const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
     storage::tar::FileReader reader{path, fingerprint};
 
-    storage::serialization::read(reader, "/common/coordinates", coordinates);
+    storage::serialization::read(reader, "/common/nbn_data/coordinates", coordinates);
 }
 
 // writes .osrm.nbg_nodes
@@ -150,8 +150,8 @@ inline void writeNodes(const boost::filesystem::path &path,
     const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
     storage::tar::FileWriter writer{path, fingerprint};
 
-    storage::serialization::write(writer, "/common/coordinates", coordinates);
-    util::serialization::write(writer, "/common/osm_node_ids", osm_node_ids);
+    storage::serialization::write(writer, "/common/nbn_data/coordinates", coordinates);
+    util::serialization::write(writer, "/common/nbn_data/osm_node_ids", osm_node_ids);
 }
 
 // reads .osrm.cnbg_to_ebg
diff --git a/include/storage/view_factory.hpp b/include/storage/view_factory.hpp
index 91acdf9c764..d39dfbc6255 100644
--- a/include/storage/view_factory.hpp
+++ b/include/storage/view_factory.hpp
@@ -35,6 +35,7 @@
 #include "util/typedefs.hpp"
 #include "util/vector_view.hpp"
 
+#include "util/filtered_graph.hpp"
 #include "util/packed_vector.hpp"
 #include "util/vector_view.hpp"
 
@@ -72,7 +73,8 @@ make_packed_vector_view(char *memory_ptr, const DataLayout &layout, const std::s
     return V{packed_internal, std::numeric_limits<std::size_t>::max()};
 }
 
-auto make_name_table_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto
+make_name_table_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto blocks = make_vector_view<extractor::NameTableView::IndexedData::BlockReference>(
         memory_ptr, layout, name + "/blocks");
@@ -83,14 +85,14 @@ auto make_name_table_view(char *memory_ptr, const DataLayout &layout, const std:
     return extractor::NameTableView{index_data_view};
 }
 
-auto make_lane_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto make_lane_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     return make_vector_view<util::guidance::LaneTupleIdPair>(memory_ptr, layout, name + "/data");
 }
 
-auto make_turn_lane_description_views(char *memory_ptr,
-                                      const DataLayout &layout,
-                                      const std::string &name)
+inline auto make_turn_lane_description_views(char *memory_ptr,
+                                             const DataLayout &layout,
+                                             const std::string &name)
 {
     auto offsets = make_vector_view<std::uint32_t>(memory_ptr, layout, name + "/offsets");
     auto masks =
@@ -99,7 +101,7 @@ auto make_turn_lane_description_views(char *memory_ptr,
     return std::make_tuple(offsets, masks);
 }
 
-auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto edge_based_node_data =
         make_vector_view<extractor::EdgeBasedNode>(memory_ptr, layout, name + "/nodes");
@@ -110,7 +112,7 @@ auto make_ebn_data_view(char *memory_ptr, const DataLayout &layout, const std::s
                                             std::move(annotation_data));
 }
 
-auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto lane_data_ids = make_vector_view<LaneDataID>(memory_ptr, layout, name + "/lane_data_ids");
 
@@ -133,7 +135,8 @@ auto make_turn_data_view(char *memory_ptr, const DataLayout &layout, const std::
                                   std::move(post_turn_bearings));
 }
 
-auto make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto
+make_segment_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto geometry_begin_indices = make_vector_view<unsigned>(memory_ptr, layout, name + "/index");
 
@@ -177,29 +180,39 @@ auto make_segment_data_view(char *memory_ptr, const DataLayout &layout, const st
                                       std::move(rev_datasources_list)};
 }
 
-auto make_coordinates_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto
+make_coordinates_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     return make_vector_view<util::Coordinate>(memory_ptr, layout, name);
 }
 
-auto make_osm_ids_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto make_osm_ids_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     return make_packed_vector_view<extractor::PackedOSMIDsView::value_type,
                                    extractor::PackedOSMIDsView::value_size>(
         memory_ptr, layout, name);
 }
 
-auto make_turn_duration_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto make_nbn_data_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+{
+    return std::make_tuple(make_coordinates_view(memory_ptr, layout, name + "/coordinates"),
+                           make_osm_ids_view(memory_ptr, layout, name + "/osm_node_ids"));
+}
+
+inline auto
+make_turn_weight_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     return make_vector_view<TurnPenalty>(memory_ptr, layout, name + "/weight");
 }
 
-auto make_turn_penalties_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto
+make_turn_duration_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     return make_vector_view<TurnPenalty>(memory_ptr, layout, name + "/duration");
 }
 
-auto make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto
+make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     using RTreeLeaf = extractor::EdgeBasedNodeSegment;
     using RTreeNode = util::StaticRTree<RTreeLeaf, storage::Ownership::View>::TreeNode;
@@ -209,17 +222,23 @@ auto make_search_tree_view(char *memory_ptr, const DataLayout &layout, const std
     const auto rtree_level_starts =
         make_vector_view<std::uint64_t>(memory_ptr, layout, name + "/search_tree_level_starts");
 
-    const auto coordinates = make_coordinates_view(memory_ptr, layout, "/common/coordinates");
+    const auto coordinates =
+        make_coordinates_view(memory_ptr, layout, "/common/nbn_data/coordinates");
 
     const char *path = layout.GetBlockPtr<char>(memory_ptr, name + "/file_index_path");
 
+    if (!boost::filesystem::exists(boost::filesystem::path{path}))
+    {
+        throw util::exception("Could not load " + std::string(path) + "Does the leaf file exist?" +
+                              SOURCE_REF);
+    }
+
     return util::StaticRTree<RTreeLeaf, storage::Ownership::View>{
         std::move(search_tree), std::move(rtree_level_starts), path, std::move(coordinates)};
 }
 
-auto make_intersection_bearings_view(char *memory_ptr,
-                                     const DataLayout &layout,
-                                     const std::string &name)
+inline auto
+make_intersection_bearings_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto bearing_offsets =
         make_vector_view<unsigned>(memory_ptr, layout, name + "/class_id_to_ranges/block_offsets");
@@ -238,14 +257,14 @@ auto make_intersection_bearings_view(char *memory_ptr,
         std::move(bearing_values), std::move(bearing_class_id), std::move(bearing_range_table)};
 }
 
-auto make_entry_classes_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto
+make_entry_classes_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     return make_vector_view<util::guidance::EntryClass>(memory_ptr, layout, name);
 }
 
-auto make_contracted_metric_view(char *memory_ptr,
-                                 const DataLayout &layout,
-                                 const std::string &name)
+inline auto
+make_contracted_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
         memory_ptr, layout, name + "/contracted_graph/node_array");
@@ -262,7 +281,7 @@ auto make_contracted_metric_view(char *memory_ptr,
                                             std::move(edge_filter)};
 }
 
-auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto level_data_ptr = layout.GetBlockPtr<partitioner::MultiLevelPartitionView::LevelData>(
         memory_ptr, name + "/level_data");
@@ -274,7 +293,8 @@ auto make_partition_view(char *memory_ptr, const DataLayout &layout, const std::
         level_data_ptr, std::move(partition), std::move(cell_to_children)};
 }
 
-auto make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto
+make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto source_boundary = make_vector_view<NodeID>(memory_ptr, layout, name + "/source_boundary");
     auto destination_boundary =
@@ -290,7 +310,25 @@ auto make_cell_storage_view(char *memory_ptr, const DataLayout &layout, const st
                                         std::move(level_offsets)};
 }
 
-auto make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
+inline auto make_filtered_cell_metric_view(char *memory_ptr,
+                                           const DataLayout &layout,
+                                           const std::string &name,
+                                           const std::size_t exclude_index)
+{
+    customizer::CellMetricView cell_metric;
+
+    auto prefix = name + "/exclude/" + std::to_string(exclude_index);
+    auto weights_block_id = prefix + "/weights";
+    auto durations_block_id = prefix + "/durations";
+
+    auto weights = make_vector_view<EdgeWeight>(memory_ptr, layout, weights_block_id);
+    auto durations = make_vector_view<EdgeDuration>(memory_ptr, layout, durations_block_id);
+
+    return customizer::CellMetricView{std::move(weights), std::move(durations)};
+}
+
+inline auto
+make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     std::vector<customizer::CellMetricView> cell_metric_excludes;
 
@@ -311,9 +349,8 @@ auto make_cell_metric_view(char *memory_ptr, const DataLayout &layout, const std
     return cell_metric_excludes;
 }
 
-auto make_multi_level_graph_view(char *memory_ptr,
-                                 const DataLayout &layout,
-                                 const std::string &name)
+inline auto
+make_multi_level_graph_view(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto node_list = make_vector_view<customizer::MultiLevelEdgeBasedGraphView::NodeArrayEntry>(
         memory_ptr, layout, name + "/node_array");
@@ -326,9 +363,8 @@ auto make_multi_level_graph_view(char *memory_ptr,
         std::move(node_list), std::move(edge_list), std::move(node_to_offset));
 }
 
-auto make_maneuver_overrides_views(char *memory_ptr,
-                                   const DataLayout &layout,
-                                   const std::string &name)
+inline auto
+make_maneuver_overrides_views(char *memory_ptr, const DataLayout &layout, const std::string &name)
 {
     auto maneuver_overrides = make_vector_view<extractor::StorageManeuverOverride>(
         memory_ptr, layout, name + "/overrides");
@@ -337,6 +373,21 @@ auto make_maneuver_overrides_views(char *memory_ptr,
 
     return std::make_tuple(maneuver_overrides, maneuver_override_node_sequences);
 }
+
+inline auto make_filtered_graph_view(char *memory_ptr,
+                                     const DataLayout &layout,
+                                     const std::string &name,
+                                     const std::size_t exclude_index)
+{
+    auto exclude_prefix = name + "/exclude/" + std::to_string(exclude_index);
+    auto edge_filter = make_vector_view<bool>(memory_ptr, layout, exclude_prefix + "/edge_filter");
+    auto node_list = make_vector_view<contractor::QueryGraphView::NodeArrayEntry>(
+        memory_ptr, layout, name + "/contracted_graph/node_array");
+    auto edge_list = make_vector_view<contractor::QueryGraphView::EdgeArrayEntry>(
+        memory_ptr, layout, name + "/contracted_graph/edge_array");
+
+    return util::FilteredGraphView<contractor::QueryGraphView>({node_list, edge_list}, edge_filter);
+}
 }
 }
 
diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp
index 7ac43e10f56..b0a3c76e32c 100644
--- a/include/util/static_rtree.hpp
+++ b/include/util/static_rtree.hpp
@@ -255,7 +255,7 @@ class StaticRTree
     // Representation of the in-memory search tree
     Vector<TreeNode> m_search_tree;
     // Reference to the actual lon/lat data we need for doing math
-    const Vector<Coordinate> &m_coordinate_list;
+    util::vector_view<const Coordinate> m_coordinate_list;
     // Holds the start indexes of each level in m_search_tree
     Vector<std::uint64_t> m_tree_level_starts;
     // mmap'd .fileIndex file
@@ -264,6 +264,7 @@ class StaticRTree
     util::vector_view<const EdgeDataT> m_objects;
 
   public:
+    StaticRTree() = default;
     StaticRTree(const StaticRTree &) = delete;
     StaticRTree &operator=(const StaticRTree &) = delete;
     StaticRTree(StaticRTree &&) = default;
@@ -273,7 +274,7 @@ class StaticRTree
     explicit StaticRTree(const std::vector<EdgeDataT> &input_data_vector,
                          const Vector<Coordinate> &coordinate_list,
                          const boost::filesystem::path &on_disk_file_name)
-        : m_coordinate_list(coordinate_list)
+        : m_coordinate_list(coordinate_list.data(), coordinate_list.size())
     {
         const auto element_count = input_data_vector.size();
         std::vector<WrappedInputElement> input_wrapper_vector(element_count);
@@ -460,7 +461,7 @@ class StaticRTree
     template <typename = std::enable_if<Ownership == storage::Ownership::Container>>
     explicit StaticRTree(const boost::filesystem::path &on_disk_file_name,
                          const Vector<Coordinate> &coordinate_list)
-        : m_coordinate_list(coordinate_list)
+        : m_coordinate_list(coordinate_list.data(), coordinate_list.size())
     {
         m_objects = mmapFile<EdgeDataT>(on_disk_file_name, m_objects_region);
     }
@@ -475,7 +476,8 @@ class StaticRTree
                          Vector<std::uint64_t> tree_level_starts,
                          const boost::filesystem::path &on_disk_file_name,
                          const Vector<Coordinate> &coordinate_list)
-        : m_search_tree(std::move(search_tree_)), m_coordinate_list(coordinate_list),
+        : m_search_tree(std::move(search_tree_)),
+          m_coordinate_list(coordinate_list.data(), coordinate_list.size()),
           m_tree_level_starts(std::move(tree_level_starts))
     {
         BOOST_ASSERT(m_tree_level_starts.size() >= 2);
diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp
index eeb54292e0e..66aca94850e 100644
--- a/src/storage/storage.cpp
+++ b/src/storage/storage.cpp
@@ -329,21 +329,15 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
     // Loading list of coordinates
     {
-        auto coordinates =
-            make_vector_view<util::Coordinate>(memory_ptr, layout, "/common/coordinates");
-
-        auto osm_node_ids =
-            extractor::PackedOSMIDsView{make_vector_view<extractor::PackedOSMIDsView::block_type>(
-                                            memory_ptr, layout, "/common/osm_node_ids/packed"),
-                                        coordinates.size()};
-
-        extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids);
+        auto views = make_nbn_data_view(memory_ptr, layout, "/common/nbn_data");
+        extractor::files::readNodes(
+            config.GetPath(".osrm.nbg_nodes"), std::get<0>(views), std::get<1>(views));
     }
 
     // load turn weight penalties
     {
         auto turn_duration_penalties =
-            make_vector_view<TurnPenalty>(memory_ptr, layout, "/common/turn_penalty/weight");
+            make_turn_weight_view(memory_ptr, layout, "/common/turn_penalty");
         extractor::files::readTurnWeightPenalty(config.GetPath(".osrm.turn_weight_penalties"),
                                                 turn_duration_penalties);
     }
@@ -351,7 +345,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
     // load turn duration penalties
     {
         auto turn_duration_penalties =
-            make_vector_view<TurnPenalty>(memory_ptr, layout, "/common/turn_penalty/duration");
+            make_turn_duration_view(memory_ptr, layout, "/common/turn_penalty");
         extractor::files::readTurnDurationPenalty(config.GetPath(".osrm.turn_duration_penalties"),
                                                   turn_duration_penalties);
     }

From e28cae76cc9ed7c269e18be0105c5bb6c8931431 Mon Sep 17 00:00:00 2001
From: Patrick Niklaus <patrick.niklaus@student.kit.edu>
Date: Tue, 3 Apr 2018 15:28:53 +0000
Subject: [PATCH 18/19] Fix test compilation

---
 unit_tests/storage/data_layout.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/unit_tests/storage/data_layout.cpp b/unit_tests/storage/data_layout.cpp
index f25459075c4..2aa0b40ee4b 100644
--- a/unit_tests/storage/data_layout.cpp
+++ b/unit_tests/storage/data_layout.cpp
@@ -38,9 +38,9 @@ BOOST_AUTO_TEST_CASE(layout_write_test)
     auto biggest_addr = buffer.data() + buffer.size();
 
     {
-        auto block_1_ptr = layout.GetBlockPtr<std::uint64_t, true>(buffer.data(), "block1");
-        auto block_2_ptr = layout.GetBlockPtr<std::uint32_t, true>(buffer.data(), "block2");
-        auto block_3_ptr = layout.GetBlockPtr<std::uint64_t, true>(buffer.data(), "block3");
+        auto block_1_ptr = layout.GetBlockPtr<std::uint64_t>(buffer.data(), "block1");
+        auto block_2_ptr = layout.GetBlockPtr<std::uint32_t>(buffer.data(), "block2");
+        auto block_3_ptr = layout.GetBlockPtr<std::uint64_t>(buffer.data(), "block3");
 
         BOOST_CHECK_LT(reinterpret_cast<std::size_t>(smallest_addr),
                        reinterpret_cast<std::size_t>(block_1_ptr));

From 9945a711dd8f5bc7de4e5c122094b1961b778f7c Mon Sep 17 00:00:00 2001
From: Michael Krasnyk <michael.krasnyk@gmail.com>
Date: Wed, 4 Apr 2018 11:31:55 +0200
Subject: [PATCH 19/19] Always create a new kv pair to avoid using shared
 strings

---
 include/storage/serialization.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/storage/serialization.hpp b/include/storage/serialization.hpp
index af0d3ec7716..92ee9600c0d 100644
--- a/include/storage/serialization.hpp
+++ b/include/storage/serialization.hpp
@@ -245,10 +245,10 @@ write<bool>(tar::FileWriter &writer, const std::string &name, const std::vector<
 template <typename K, typename V> void read(io::BufferReader &reader, std::map<K, V> &data)
 {
     const auto count = reader.ReadElementCount64();
-    std::pair<K, V> pair;
     for (auto index : util::irange<std::size_t>(0, count))
     {
         (void)index;
+        std::pair<K, V> pair;
         read(reader, pair.first);
         read(reader, pair.second);
         data.insert(pair);