Add directory listing

include/storage/shared_datatype.hpp

@@ -13,6 +13,7 @@
 #include <array>
 #include <cstdint>
 #include <map>
+#include <unordered_set>
 
 namespace osrm
 {
@@ -27,6 +28,32 @@ inline void read(io::BufferReader &reader, DataLayout &layout);
 inline void write(io::BufferWriter &writer, const DataLayout &layout);
 }
 
+namespace detail
+{
+// Removes the file name if name_prefix is a directory and name is not a file in that directory
+inline std::string trimName(const std::string &name_prefix, const std::string &name)
+{
+    // list directory and
+    if (name_prefix.back() == '/')
+    {
+        auto directory_position = name.find_first_of("/", name_prefix.length());
+        // this is a "file" in the directory of name_prefix
+        if (directory_position == std::string::npos)
+        {
+            return name;
+        }
+        else
+        {
+            return name.substr(0, directory_position);
+        }
+    }
+    else
+    {
+        return name;
+    }
+}
+}
+
 // Added at the start and end of each block as sanity check
 const constexpr char CANARY[4] = {'O', 'S', 'R', 'M'};
 
@@ -42,10 +69,7 @@ class DataLayout
         return GetBlock(name).num_entries;
     }
 
-    inline uint64_t GetBlockSize(const std::string &name) const
-    {
-        return GetBlock(name).byte_size;
-    }
+    inline uint64_t GetBlockSize(const std::string &name) const { return GetBlock(name).byte_size; }
 
     inline uint64_t GetSizeOfLayout() const
     {
@@ -68,7 +92,8 @@ class DataLayout
         {
             char *start_canary_ptr = ptr - sizeof(CANARY);
             char *end_canary_ptr = ptr + GetBlockSize(name);
-            std::cout << name << ": " << (long) (start_canary_ptr-shared_memory) << " -> " << (long) (end_canary_ptr-shared_memory) << std::endl;
+            std::cout << name << ": " << (long)(start_canary_ptr - shared_memory) << " -> "
+                      << (long)(end_canary_ptr - shared_memory) << std::endl;
             std::copy(CANARY, CANARY + sizeof(CANARY), start_canary_ptr);
             std::copy(CANARY, CANARY + sizeof(CANARY), end_canary_ptr);
         }
@@ -92,15 +117,25 @@ class DataLayout
         return (T *)ptr;
     }
 
-    template<typename OutIter>
-    void List(const std::string& name_prefix, OutIter out) const
+    // Depending on the name prefix this function either lists all blocks with the same prefix
+    // or all entries in the sub-directory.
+    // '/ch/edge' -> '/ch/edge_filter/0/blocks', '/ch/edge_filter/1/blocks'
+    // '/ch/edge_filters/' -> '/ch/edge_filter/0', '/ch/edge_filter/1'
+    template <typename OutIter> void List(const std::string &name_prefix, OutIter out) const
     {
-        for (const auto& pair : blocks)
+        std::unordered_set<std::string> returned_name;
+
+        for (const auto &pair : blocks)
         {
             // check if string begins with the name prefix
             if (pair.first.find(name_prefix) == 0)
             {
-                *out++ = pair.first;
+                auto trimmed_name = detail::trimName(name_prefix, pair.first);
+                auto ret = returned_name.insert(trimmed_name);
+                if (ret.second)
+                {
+                    *out++ = trimmed_name;
+                }
             }
         }
     }
@@ -109,7 +144,7 @@ class DataLayout
     friend void serialization::read(io::BufferReader &reader, DataLayout &layout);
     friend void serialization::write(io::BufferWriter &writer, const DataLayout &layout);
 
-    const Block& GetBlock(const std::string& name) const
+    const Block &GetBlock(const std::string &name) const
     {
         auto iter = blocks.find(name);
         if (iter == blocks.end())

src/engine/datafacade/shared_memory_allocator.cpp

@@ -23,6 +23,7 @@ SharedMemoryAllocator::SharedMemoryAllocator(storage::SharedDataType data_region
     storage::io::BufferReader reader(GetMemory());
     storage::serialization::read(reader, data_layout);
     layout_size = reader.GetPosition();
+    util::Log(logDEBUG) << "Data layout has size " << layout_size;
 }
 
 SharedMemoryAllocator::~SharedMemoryAllocator() {}

src/storage/storage.cpp

@@ -727,10 +727,12 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr)
 
             std::vector<customizer::CellMetricView> metrics;
 
-            for (auto index : util::irange<std::size_t>(0, NUM_METRICS))
+            std::vector<std::string> metric_prefix_names;
+            layout.List("/mld/metrics/", std::back_inserter(metric_prefix_names));
+            for (const auto &prefix : metric_prefix_names)
             {
-                auto weights_block_id = "/mld/metrics/" + std::to_string(index) + "/weights";
-                auto durations_block_id = "/mld/metrics/" + std::to_string(index) + "/durations";
+                auto weights_block_id = prefix + "/weights";
+                auto durations_block_id = prefix + "/durations";
 
                 auto weight_entries_count = layout.GetBlockEntries(weights_block_id);
                 auto duration_entries_count = layout.GetBlockEntries(durations_block_id);

unit_tests/storage/data_layout.cpp

@@ -63,4 +63,42 @@ BOOST_AUTO_TEST_CASE(layout_write_test)
     }
 }
 
+BOOST_AUTO_TEST_CASE(layout_list_test)
+{
+    DataLayout layout;
+
+    Block block_1{20, 8 * 20};
+    Block block_2{1, 4 * 1};
+    Block block_3{100, static_cast<std::uint64_t>(std::ceil(100 / 64.))};
+
+    layout.SetBlock("/ch/edge_filter/block1", block_1);
+    layout.SetBlock("/ch/edge_filter/block2", block_2);
+    layout.SetBlock("/ch/edge_filter/block3", block_3);
+    layout.SetBlock("/mld/metrics/0/durations", block_2);
+    layout.SetBlock("/mld/metrics/0/weights", block_3);
+    layout.SetBlock("/mld/metrics/1/durations", block_2);
+    layout.SetBlock("/mld/metrics/1/weights", block_3);
+
+    std::vector<std::string> results_1;
+    std::vector<std::string> results_2;
+    std::vector<std::string> results_3;
+    layout.List("/ch/edge_filter", std::back_inserter(results_1));
+    layout.List("/ch/edge_filter/", std::back_inserter(results_2));
+    layout.List("/ch/", std::back_inserter(results_3));
+
+    std::vector<std::string> results_4;
+    std::vector<std::string> results_5;
+    std::vector<std::string> results_6;
+    layout.List("/mld/metrics", std::back_inserter(results_4));
+    layout.List("/mld/metrics/", std::back_inserter(results_5));
+    layout.List("/mld/", std::back_inserter(results_6));
+
+    CHECK_EQUAL_RANGE(results_1, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
+    CHECK_EQUAL_RANGE(results_2, "/ch/edge_filter/block1", "/ch/edge_filter/block2", "/ch/edge_filter/block3");
+    CHECK_EQUAL_RANGE(results_3, "/ch/edge_filter");
+    CHECK_EQUAL_RANGE(results_4, "/mld/metrics/0/durations", "/mld/metrics/0/weights", "/mld/metrics/1/durations", "/mld/metrics/1/weights");
+    CHECK_EQUAL_RANGE(results_5, "/mld/metrics/0", "/mld/metrics/1");
+    CHECK_EQUAL_RANGE(results_6, "/mld/metrics");
+}
+
 BOOST_AUTO_TEST_SUITE_END()