Add support for naming the dataset

CHANGELOG.md

@@ -1,7 +1,7 @@
 # UNRELEASED
   - Changes from 5.16.0:
     - Bugfixes:
-      - fix deduplication of route steps when waypoints are used [#4909](https://github.com/Project-OSRM/osrm-backend/issues/4909)
+      - FIXED: fix deduplication of route steps when waypoints are used [#4909](https://github.com/Project-OSRM/osrm-backend/issues/4909)
       - FIXED #4920: Use smaller range for U-turn angles in map-matching [#4920](https://github.com/Project-OSRM/osrm-backend/pull/4920)
       - FIXED: Remove the last short annotation segment in `trimShortSegments`
       - FIXED: Properly calculate annotations for speeds, durations and distances when waypoints are used with mapmatching [#4949](https://github.com/Project-OSRM/osrm-backend/pull/4949)
@@ -14,9 +14,13 @@
       - CHANGED #4925: Added post process logic to collapse segregated turn instructions [#4925](https://github.com/Project-OSRM/osrm-backend/pull/4925)
       - ADDED: Maneuver relation now supports `straight` as a direction [#4995](https://github.com/Project-OSRM/osrm-backend/pull/4995)
     - Tools:
-      - `osrm-routed` accepts a new property `--memory_file` to store memory in a file on disk.
+      - ADDED: `osrm-datastore` accepts a new parameter `--dataset-name` to select the name of the dataset.
+      - ADDED: `osrm-datastore` accepts a new parameter `--list` to list all datasets loaded into memory.
+      - ADDED: `osrm-routed` accepts a new parameter `--memory_file` to store memory in a file on disk.
+      - ADDED: `osrm-routed` accepts a new parameter `--dataset-name` to select the shared-memory dataset to use.
     - NodeJS:
       - `OSRM` object accepts a new option `memory_file` that stores the memory in a file on disk.
+      - `OSRM` object accepts a new option `dataset_name` to select the shared-memory dataset.
     - Internals
       - CHANGED #4845 #4968: Updated segregated intersection identification [#4845](https://github.com/Project-OSRM/osrm-backend/pull/4845) [#4968](https://github.com/Project-OSRM/osrm-backend/pull/4968)
       - REMOVED: Remove `.timestamp` file since it was unused.

include/engine/data_watchdog.hpp

@@ -36,17 +36,18 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
     using Facade = datafacade::ContiguousInternalMemoryDataFacade<AlgorithmT>;
 
   public:
-    DataWatchdogImpl() : active(true)
+    DataWatchdogImpl(const std::string &dataset_name) : dataset_name(dataset_name), active(true)
     {
         // create the initial facade before launching the watchdog thread
         {
             boost::interprocess::scoped_lock<mutex_type> current_region_lock(barrier.get_mutex());
 
-            auto& shared_register = barrier.data();
-            auto region_id = shared_register.Find("data");
+            auto &shared_register = barrier.data();
+            auto region_id = shared_register.Find(dataset_name + "/data");
             if (region_id == storage::SharedRegionRegister::INVALID_REGION_ID)
             {
-                throw util::exception("Could not find shared memory region. Did you run osrm-datastore?");
+                throw util::exception(
+                    "Could not find shared memory region. Did you run osrm-datastore?");
             }
             shared_region = &shared_register.GetRegion(region_id);
             region = *shared_region;
@@ -101,6 +102,7 @@ class DataWatchdogImpl<AlgorithmT, datafacade::ContiguousInternalMemoryDataFacad
         util::Log() << "DataWatchdog thread stopped";
     }
 
+    const std::string dataset_name;
     storage::SharedMonitor<storage::SharedRegionRegister> barrier;
     std::thread watcher;
     bool active;

include/engine/datafacade_provider.hpp

@@ -83,6 +83,8 @@ class WatchingProvider : public DataFacadeProvider<AlgorithmT, FacadeT>
   public:
     using Facade = typename DataFacadeProvider<AlgorithmT, FacadeT>::Facade;
 
+    WatchingProvider(const std::string &dataset_name) : watchdog(dataset_name) {}
+
     std::shared_ptr<const Facade> Get(const api::TileParameters &params) const override final
     {
         return watchdog.Get(params);

include/engine/engine.hpp

@@ -61,7 +61,7 @@ template <typename Algorithm> class Engine final : public EngineInterface
         {
             util::Log(logDEBUG) << "Using shared memory with algorithm "
                                 << routing_algorithms::name<Algorithm>();
-            facade_provider = std::make_unique<WatchingProvider<Algorithm>>();
+            facade_provider = std::make_unique<WatchingProvider<Algorithm>>(config.dataset_name);
         }
         else if (!config.memory_file.empty())
         {

include/engine/engine_config.hpp

@@ -91,6 +91,7 @@ struct EngineConfig final
     boost::filesystem::path memory_file;
     Algorithm algorithm = Algorithm::CH;
     std::string verbosity;
+    std::string dataset_name;
 };
 }
 }

include/storage/shared_datatype.hpp

@@ -221,6 +221,18 @@ struct SharedRegionRegister
         }
     }
 
+    template<typename OutIter>
+    void List(OutIter out) const
+    {
+        for (const auto& region : regions)
+        {
+            if (!region.IsEmpty())
+            {
+                *out++ = region.name;
+            }
+        }
+    }
+
     void Deregister(const RegionID key) { regions[key] = SharedRegion{}; }
 
     const auto &GetRegion(const RegionID key) const { return regions[key]; }

include/storage/shared_monitor.hpp

@@ -126,7 +126,6 @@ template <typename Data> struct SharedMonitor
         bi::interprocess_condition condition;
     };
 
-
 #else
     // Implement a conditional variable using a queue of semaphores.
     // OSX checks the virtual address of a mutex in pthread_cond_wait and fails with EINVAL
@@ -207,8 +206,10 @@ template <typename Data> struct SharedMonitor
     static_assert(buffer_size >= 2, "buffer size is too small");
 
 #endif
-    static constexpr int rounded_internal_size = ((sizeof(InternalData) + alignof(Data) - 1) / alignof(Data)) * alignof(Data);
-    static_assert(rounded_internal_size < sizeof(InternalData) + sizeof(Data), "Data and internal data need to fit into shared memory");
+    static constexpr int rounded_internal_size =
+        ((sizeof(InternalData) + alignof(Data) - 1) / alignof(Data)) * alignof(Data);
+    static_assert(rounded_internal_size < sizeof(InternalData) + sizeof(Data),
+                  "Data and internal data need to fit into shared memory");
 
     InternalData &internal() const
     {

include/storage/storage.hpp

@@ -44,7 +44,7 @@ class Storage
   public:
     Storage(StorageConfig config);
 
-    int Run(int max_wait);
+    int Run(int max_wait, const std::string &name);
 
     void PopulateLayout(DataLayout &layout);
     void PopulateData(const DataLayout &layout, char *memory_ptr);

src/storage/storage.cpp

@@ -65,7 +65,7 @@ using Monitor = SharedMonitor<SharedRegionRegister>;
 
 Storage::Storage(StorageConfig config_) : config(std::move(config_)) {}
 
-int Storage::Run(int max_wait)
+int Storage::Run(int max_wait, const std::string &dataset_name)
 {
     BOOST_ASSERT_MSG(config.IsValid(), "Invalid storage config");
 
@@ -161,14 +161,14 @@ int Storage::Run(int max_wait)
             lock.lock();
         }
 
-        auto region_id = shared_register.Find("data");
+        auto region_id = shared_register.Find(dataset_name + "/data");
         if (region_id == SharedRegionRegister::INVALID_REGION_ID)
         {
-            region_id = shared_register.Register("data", shm_key);
+            region_id = shared_register.Register(dataset_name + "/data", shm_key);
         }
         else
         {
-            auto& shared_region = shared_register.GetRegion(region_id);
+            auto &shared_region = shared_register.GetRegion(region_id);
             next_timestamp = shared_region.timestamp + 1;
             in_use_key = shared_region.shm_key;
             shared_region.shm_key = shm_key;

src/tools/routed.cpp

@@ -115,6 +115,9 @@ inline unsigned generateServerProgramOptions(const int argc,
         ("memory_file",
          value<boost::filesystem::path>(&config.memory_file),
          "Store data in a memory mapped file rather than in process memory.") //
+        ("dataset-name",
+         value<std::string>(&config.dataset_name),
+         "Name of the shared memory dataset to connect to.") //
         ("algorithm,a",
          value<EngineConfig::Algorithm>(&config.algorithm)
              ->default_value(EngineConfig::Algorithm::CH, "CH"),

src/tools/store.cpp

@@ -25,6 +25,23 @@ void deleteRegion(const storage::SharedRegionRegister::ShmKey key)
     }
 }
 
+void listRegions()
+{
+
+    storage::SharedMonitor<storage::SharedRegionRegister> monitor;
+    std::vector<std::string> names;
+    const auto &shared_register = monitor.data();
+    shared_register.List(std::back_inserter(names));
+    osrm::util::Log() << "name\tshm key\ttimestamp";
+    for (const auto &name : names)
+    {
+        auto id = shared_register.Find(name);
+        auto region = shared_register.GetRegion(id);
+        osrm::util::Log() << name << "\t"
+                          << (int) region.shm_key << "\t" << region.timestamp;
+    }
+}
+
 void springClean()
 {
     osrm::util::Log() << "Releasing all locks";
@@ -55,7 +72,9 @@ bool generateDataStoreOptions(const int argc,
                               const char *argv[],
                               std::string &verbosity,
                               boost::filesystem::path &base_path,
-                              int &max_wait)
+                              int &max_wait,
+                              std::string &dataset_name,
+                              bool &list_datasets)
 {
     // declare a group of options that will be allowed only on command line
     boost::program_options::options_description generic_options("Options");
@@ -69,10 +88,19 @@ bool generateDataStoreOptions(const int argc,
     // declare a group of options that will be allowed both on command line
     // as well as in a config file
     boost::program_options::options_description config_options("Configuration");
-    config_options.add_options()("max-wait",
-                                 boost::program_options::value<int>(&max_wait)->default_value(-1),
-                                 "Maximum number of seconds to wait on a running data update "
-                                 "before aquiring the lock by force.");
+    config_options.add_options() //
+        ("max-wait",
+         boost::program_options::value<int>(&max_wait)->default_value(-1),
+         "Maximum number of seconds to wait on a running data update "
+         "before aquiring the lock by force.") //
+        ("dataset-name",
+         boost::program_options::value<std::string>(&dataset_name)->default_value(""),
+         "Name of the dataset to load into memory. This allows having multiple datasets in memory "
+         "at the same time.") //
+        ("list",
+         boost::program_options::value<bool>(&list_datasets)->default_value(false)->implicit_value(true),
+         "Name of the dataset to load into memory. This allows having multiple datasets in memory "
+         "at the same time.");
 
     // hidden options, will be allowed on command line but will not be shown to the user
     boost::program_options::options_description hidden_options("Hidden options");
@@ -165,13 +193,22 @@ int main(const int argc, const char *argv[]) try
     std::string verbosity;
     boost::filesystem::path base_path;
     int max_wait = -1;
-    if (!generateDataStoreOptions(argc, argv, verbosity, base_path, max_wait))
+    std::string dataset_name;
+    bool list_datasets = false;
+    if (!generateDataStoreOptions(
+            argc, argv, verbosity, base_path, max_wait, dataset_name, list_datasets))
     {
         return EXIT_SUCCESS;
     }
 
     util::LogPolicy::GetInstance().SetLevel(verbosity);
 
+    if (list_datasets)
+    {
+        listRegions();
+        return EXIT_SUCCESS;
+    }
+
     storage::StorageConfig config(base_path);
     if (!config.IsValid())
     {
@@ -180,7 +217,7 @@ int main(const int argc, const char *argv[]) try
     }
     storage::Storage storage(std::move(config));
 
-    return storage.Run(max_wait);
+    return storage.Run(max_wait, dataset_name);
 }
 catch (const osrm::RuntimeError &e)
 {