Skip to content

Commit

Permalink
Refactor metric storage
Browse files Browse the repository at this point in the history
  • Loading branch information
@TheMarex
TheMarex committed Mar 27, 2018
1 parent de0e31a commit 99976aa
Show file tree
Hide file tree
Showing 7 changed files with 202 additions and 155 deletions.
26 changes: 26 additions & 0 deletions include/contractor/contracted_metric.hpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#ifndef OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP
#define OSMR_CONTRACTOR_CONTRACTED_METRIC_HPP

#include "contractor/query_graph.hpp"

namespace osrm
{
namespace contractor
{

namespace detail
{
template<storage::Ownership Ownership>
struct ContractedMetric
{
detail::QueryGraph<Ownership> graph;
std::vector<util::ViewOrVector<bool, Ownership>> edge_filter;
};
}

using ContractedMetric = detail::ContractedMetric<storage::Ownership::Container>;
using ContractedMetricView = detail::ContractedMetric<storage::Ownership::View>;
}
}

#endif
57 changes: 19 additions & 38 deletions include/contractor/files.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
#ifndef OSRM_CONTRACTOR_FILES_HPP
#define OSRM_CONTRACTOR_FILES_HPP

#include "contractor/query_graph.hpp"
#include "contractor/serialization.hpp"

#include "util/serialization.hpp"

#include "storage/serialization.hpp"
#include "storage/tar.hpp"
#include <unordered_map>

namespace osrm
{
Expand All @@ -15,67 +12,51 @@ namespace contractor
namespace files
{
// reads .osrm.hsgr file
template <typename QueryGraphT, typename EdgeFilterT>
template <typename ContractedMetricT>
inline void readGraph(const boost::filesystem::path &path,
unsigned &checksum,
QueryGraphT &graph,
std::vector<EdgeFilterT> &edge_filter,
std::unordered_map<std::string, ContractedMetricT> &metrics,
std::uint32_t &connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
"graph must be of type QueryGraph<>");
static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
"edge_filter must be a container of vector<bool> or vector_view<bool>");
static_assert(std::is_same<ContractedMetric, ContractedMetricT>::value ||
std::is_same<ContractedMetricView, ContractedMetricT>::value,
"metric must be of type ContractedMetric<>");

const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};

reader.ReadInto("/ch/checksum", checksum);
util::serialization::read(reader, "/ch/contracted_graph", graph);
reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);

auto count = reader.ReadElementCount64("/ch/edge_filter");
edge_filter.resize(count);
for (const auto index : util::irange<std::size_t>(0, count))
for (const auto &pair : metrics)
{
storage::serialization::read(
reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
serialization::write("/ch/metrics/" + pair.first, pair.second);
}

reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
}

// writes .osrm.hsgr file
template <typename QueryGraphT, typename EdgeFilterT>
template <typename ContractedMetricT>
inline void writeGraph(const boost::filesystem::path &path,
unsigned checksum,
const QueryGraphT &graph,
const std::vector<EdgeFilterT> &edge_filter,
const std::unordered_map<std::string, ContractedMetricT> &metrics,
const std::uint32_t connectivity_checksum)
{
static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
std::is_same<QueryGraph, QueryGraphT>::value,
"graph must be of type QueryGraph<>");
static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
"edge_filter must be a container of vector<bool> or vector_view<bool>");
static_assert(std::is_same<ContractedMetric, ContractedMetricT>::value ||
std::is_same<ContractedMetricView, ContractedMetricT>::value,
"metric must be of type ContractedMetric<>");
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};

writer.WriteElementCount64("/ch/checksum", 1);
writer.WriteFrom("/ch/checksum", checksum);
util::serialization::write(writer, "/ch/contracted_graph", graph);
writer.WriteElementCount64("/ch/connectivity_checksum", 1);
writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);

writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
for (const auto index : util::irange<std::size_t>(0, edge_filter.size()))
for (const auto &pair : metrics)
{
storage::serialization::write(
writer, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
serialization::write("/ch/metrics/" + pair.first, pair.second);
}

writer.WriteElementCount64("/ch/connectivity_checksum", 1);
writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
}
}
}
Expand Down
42 changes: 29 additions & 13 deletions include/customizer/files.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

#include "util/integer_range.hpp"

#include <unordered_map>

namespace osrm
{
namespace customizer
Expand All @@ -16,7 +18,7 @@ namespace files

// reads .osrm.cell_metrics file
template <typename CellMetricT>
inline void readCellMetrics(const boost::filesystem::path &path, std::vector<CellMetricT> &metrics)
inline void readCellMetrics(const boost::filesystem::path &path, std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
Expand All @@ -25,20 +27,27 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::vector<Cel
const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
storage::tar::FileReader reader{path, fingerprint};

auto num_metrics = reader.ReadElementCount64("/mld/metrics");
metrics.resize(num_metrics);

auto id = 0;
for (auto &metric : metrics)
for (auto& pair : metrics)
{
serialization::read(reader, "/mld/metrics/" + std::to_string(id++), metric);
const auto& metric_name = pair.first;
auto& metric_exclude_classes = pair.second;

auto prefix = "/mld/metrics/" + metric_name + "/exclude";
auto num_exclude_classes = reader.ReadElementCount64(prefix);
metric_exclude_classes.resize(num_exclude_classes);

auto id = 0;
for (auto &metric : metric_exclude_classes)
{
serialization::read(reader, prefix + "/" + std::to_string(id++), metric);
}
}
}

// writes .osrm.cell_metrics file
template <typename CellMetricT>
inline void writeCellMetrics(const boost::filesystem::path &path,
const std::vector<CellMetricT> &metrics)
const std::unordered_map<std::string, std::vector<CellMetricT>> &metrics)
{
static_assert(std::is_same<CellMetricView, CellMetricT>::value ||
std::is_same<CellMetric, CellMetricT>::value,
Expand All @@ -47,12 +56,19 @@ inline void writeCellMetrics(const boost::filesystem::path &path,
const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
storage::tar::FileWriter writer{path, fingerprint};

writer.WriteElementCount64("/mld/metrics", metrics.size());

auto id = 0;
for (const auto &metric : metrics)
for (const auto& pair : metrics)
{
serialization::write(writer, "/mld/metrics/" + std::to_string(id++), metric);
const auto& metric_name = pair.first;
const auto& metric_exclude_classes = pair.second;

auto prefix = "/mld/metrics/" + metric_name + "/exclude";
auto num_metrics = writer.WriteElementCount64(prefix, metric_exclude_classes.size());

auto id = 0;
for (auto &exclude_metric : metric_exclude_classes)
{
serialization::write(writer, prefix + "/" + std::to_string(id++), exclude_metric);
}
}
}
}
Expand Down
57 changes: 35 additions & 22 deletions include/engine/datafacade/contiguous_internalmem_datafacade.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,23 +83,26 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor

void InitializeGraphPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
const std::string metric_prefix = "/ch/metrics/" + metric_name;
auto graph_nodes_ptr =
data_layout.GetBlockPtr<GraphNode>(memory_block, "/ch/contracted_graph/node_array");
data_layout.GetBlockPtr<GraphNode>(memory_block, metric_prefix + "/contracted_graph/node_array");

auto graph_edges_ptr =
data_layout.GetBlockPtr<GraphEdge>(memory_block, "/ch/contracted_graph/edge_array");
data_layout.GetBlockPtr<GraphEdge>(memory_block, metric_prefix + "/contracted_graph/edge_array");

auto filter_block_id = "/ch/edge_filter/" + std::to_string(exclude_index);
auto exclude_prefix = metric_prefix + "/exclude/" + std::to_string(exclude_index);
auto filter_block_id = exclude_prefix + "/edge_filter";

auto edge_filter_ptr =
data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);

util::vector_view<GraphNode> node_list(
graph_nodes_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/node_array"));
graph_nodes_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/node_array"));
util::vector_view<GraphEdge> edge_list(
graph_edges_ptr, data_layout.GetBlockEntries("/ch/contracted_graph/edge_array"));
graph_edges_ptr, data_layout.GetBlockEntries(metric_prefix + "/contracted_graph/edge_array"));

util::vector_view<bool> edge_filter(edge_filter_ptr,
data_layout.GetBlockEntries(filter_block_id));
Expand All @@ -108,17 +111,18 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor

public:
ContiguousInternalMemoryAlgorithmDataFacade(
std::shared_ptr<ContiguousBlockAllocator> allocator_, std::size_t exclude_index)
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}

void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeGraphPointer(data_layout, memory_block, exclude_index);
InitializeGraphPointer(data_layout, memory_block, metric_name, exclude_index);
}

// search graph access
Expand Down Expand Up @@ -225,8 +229,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade

void InitializeProfilePropertiesPointer(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
// TODO: For multi-metric support we need to have separate exclude classes per metric
(void) metric_name;
m_profile_properties = data_layout.GetBlockPtr<extractor::ProfileProperties>(
memory_block, "/common/properties");

Expand Down Expand Up @@ -528,6 +535,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade

void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeChecksumPointer(data_layout, memory_block);
Expand All @@ -538,7 +546,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
InitializeGeometryPointers(data_layout, memory_block);
InitializeNamePointers(data_layout, memory_block);
InitializeTurnLaneDescriptionsPointers(data_layout, memory_block);
InitializeProfilePropertiesPointer(data_layout, memory_block, exclude_index);
InitializeProfilePropertiesPointer(data_layout, memory_block, metric_name, exclude_index);
InitializeRTreePointers(data_layout, memory_block);
InitializeIntersectionClassPointers(data_layout, memory_block);
InitializeManeuverOverridePointers(data_layout, memory_block);
Expand All @@ -548,10 +556,11 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
// allows switching between process_memory/shared_memory datafacade, based on the type of
// allocator
ContiguousInternalMemoryDataFacadeBase(std::shared_ptr<ContiguousBlockAllocator> allocator_,
const std::string& metric_name,
const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}

// node and edge information access
Expand Down Expand Up @@ -959,9 +968,10 @@ class ContiguousInternalMemoryDataFacade<CH>
{
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
const std::string& metric_name,
const std::size_t exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<CH>(allocator, metric_name, exclude_index)

{
}
Expand All @@ -981,14 +991,16 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo

void InitializeInternalPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
InitializeMLDDataPointers(data_layout, memory_block, exclude_index);
InitializeMLDDataPointers(data_layout, memory_block, metric_name, exclude_index);
InitializeGraphPointer(data_layout, memory_block);
}

void InitializeMLDDataPointers(const storage::DataLayout &data_layout,
char *memory_block,
const std::string& metric_name,
const std::size_t exclude_index)
{
if (data_layout.GetBlockSize("/mld/multilevelpartition/partition") > 0)
Expand Down Expand Up @@ -1016,18 +1028,18 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
partitioner::MultiLevelPartitionView{level_data, partition, cell_to_children};
}

const auto weights_block_id = "/mld/metrics/" + std::to_string(exclude_index) + "/weights";
const auto durations_block_id =
"/mld/metrics/" + std::to_string(exclude_index) + "/durations";
const auto exclude_prefix = "/mld/metrics/" + metric_name + "/exclude/" + std::to_string(exclude_index);
const auto weights_block_id = exclude_prefix + "/weights";
const auto durations_block_id = exclude_prefix + "/durations";

if (data_layout.GetBlockSize(weights_block_id) > 0)
{
auto mld_cell_weights_ptr =
data_layout.GetBlockPtr<EdgeWeight>(memory_block, weights_block_id);
auto mld_cell_durations_ptr =
data_layout.GetBlockPtr<EdgeDuration>(memory_block, durations_block_id);
auto weight_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/weights");
auto duration_entries_count = data_layout.GetBlockEntries("/mld/metrics/0/durations");
auto weight_entries_count = data_layout.GetBlockEntries(weights_block_id);
auto duration_entries_count = data_layout.GetBlockEntries(durations_block_id);
BOOST_ASSERT(weight_entries_count == duration_entries_count);
util::vector_view<EdgeWeight> weights(mld_cell_weights_ptr, weight_entries_count);
util::vector_view<EdgeDuration> durations(mld_cell_durations_ptr,
Expand Down Expand Up @@ -1099,10 +1111,10 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo

public:
ContiguousInternalMemoryAlgorithmDataFacade(
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::size_t exclude_index)
std::shared_ptr<ContiguousBlockAllocator> allocator_, const std::string& metric_name, const std::size_t exclude_index)
: allocator(std::move(allocator_))
{
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), exclude_index);
InitializeInternalPointers(allocator->GetLayout(), allocator->GetMemory(), metric_name, exclude_index);
}

const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const override
Expand Down Expand Up @@ -1156,9 +1168,10 @@ class ContiguousInternalMemoryDataFacade<MLD> final
private:
public:
ContiguousInternalMemoryDataFacade(std::shared_ptr<ContiguousBlockAllocator> allocator,
const std::string& metric_name,
const std::size_t exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, exclude_index)
: ContiguousInternalMemoryDataFacadeBase(allocator, metric_name, exclude_index),
ContiguousInternalMemoryAlgorithmDataFacade<MLD>(allocator, metric_name, exclude_index)

{
}
Expand Down
Loading

0 comments on commit 99976aa

Please sign in to comment.