diff --git a/CMakeLists.txt b/CMakeLists.txt index f34b99e9b45..6e1e95a810a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -679,7 +679,8 @@ set(UTIL_LIBRARIES ${CMAKE_THREAD_LIBS_INIT} ${MAYBE_STXXL_LIBRARY} ${TBB_LIBRARIES} - ${MAYBE_COVERAGE_LIBRARIES}) + ${MAYBE_COVERAGE_LIBRARIES} + ${ZLIB_LIBRARY}) # Libraries target_link_libraries(osrm ${ENGINE_LIBRARIES}) diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp index c1ef9a827f8..b342c0fdf1c 100644 --- a/include/contractor/files.hpp +++ b/include/contractor/files.hpp @@ -19,7 +19,8 @@ template inline void readGraph(const boost::filesystem::path &path, unsigned &checksum, QueryGraphT &graph, - std::vector &edge_filter) + std::vector &edge_filter, + std::uint32_t &connectivity_checksum) { static_assert(std::is_same::value || std::is_same::value, @@ -39,6 +40,7 @@ inline void readGraph(const boost::filesystem::path &path, { storage::serialization::read(reader, edge_filter[index]); } + reader.ReadInto(connectivity_checksum); } // writes .osrm.hsgr file @@ -46,7 +48,8 @@ template inline void writeGraph(const boost::filesystem::path &path, unsigned checksum, const QueryGraphT &graph, - const std::vector &edge_filter) + const std::vector &edge_filter, + const std::uint32_t connectivity_checksum) { static_assert(std::is_same::value || std::is_same::value, @@ -64,6 +67,7 @@ inline void writeGraph(const boost::filesystem::path &path, { storage::serialization::write(writer, filter); } + writer.WriteOne(connectivity_checksum); } } } diff --git a/include/extractor/edge_based_graph_factory.hpp b/include/extractor/edge_based_graph_factory.hpp index 53bb93434ac..701031726c7 100644 --- a/include/extractor/edge_based_graph_factory.hpp +++ b/include/extractor/edge_based_graph_factory.hpp @@ -92,6 +92,7 @@ class EdgeBasedGraphFactory void GetEdgeBasedNodeSegments(std::vector &nodes); void GetStartPointMarkers(std::vector &node_is_startpoint); void GetEdgeBasedNodeWeights(std::vector &output_node_weights); + std::uint32_t GetConnectivityChecksum() const; std::uint64_t GetNumberOfEdgeBasedNodes() const; @@ -133,6 +134,7 @@ class EdgeBasedGraphFactory std::vector m_edge_based_node_segments; EdgeBasedNodeDataContainer &m_edge_based_node_container; util::DeallocatingVector m_edge_based_edge_list; + std::uint32_t m_connectivity_checksum; // The number of edge-based nodes is mostly made up out of the edges in the node-based graph. // Any edge in the node-based graph represents a node in the edge-based graph. In addition, we diff --git a/include/extractor/extractor.hpp b/include/extractor/extractor.hpp index 2a1764ba07d..bf544e1b4f3 100644 --- a/include/extractor/extractor.hpp +++ b/include/extractor/extractor.hpp @@ -84,7 +84,8 @@ class Extractor std::vector &edge_based_node_segments, std::vector &node_is_startpoint, std::vector &edge_based_node_weights, - util::DeallocatingVector &edge_based_edge_list); + util::DeallocatingVector &edge_based_edge_list, + std::uint32_t &connectivity_checksum); void FindComponents(unsigned max_edge_id, const util::DeallocatingVector &input_edge_list, diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index bd06787734f..f01ed9f227a 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -78,7 +78,8 @@ inline void writeProfileProperties(const boost::filesystem::path &path, template void writeEdgeBasedGraph(const boost::filesystem::path &path, EdgeID const number_of_edge_based_nodes, - const EdgeBasedEdgeVector &edge_based_edge_list) + const EdgeBasedEdgeVector &edge_based_edge_list, + const std::uint32_t connectivity_checksum) { static_assert(std::is_same::value, ""); @@ -86,12 +87,14 @@ void writeEdgeBasedGraph(const boost::filesystem::path &path, writer.WriteElementCount64(number_of_edge_based_nodes); storage::serialization::write(writer, edge_based_edge_list); + writer.WriteOne(connectivity_checksum); } template void readEdgeBasedGraph(const boost::filesystem::path &path, EdgeID &number_of_edge_based_nodes, - EdgeBasedEdgeVector &edge_based_edge_list) + EdgeBasedEdgeVector &edge_based_edge_list, + std::uint32_t &connectivity_checksum) { static_assert(std::is_same::value, ""); @@ -99,6 +102,7 @@ void readEdgeBasedGraph(const boost::filesystem::path &path, number_of_edge_based_nodes = reader.ReadElementCount64(); storage::serialization::read(reader, edge_based_edge_list); + reader.ReadInto(connectivity_checksum); } // reads .osrm.nodes diff --git a/include/guidance/files.hpp b/include/guidance/files.hpp index 1db91350181..7701040114f 100644 --- a/include/guidance/files.hpp +++ b/include/guidance/files.hpp @@ -19,7 +19,9 @@ namespace files // reads .osrm.edges template -inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_data) +inline void readTurnData(const boost::filesystem::path &path, + TurnDataT &turn_data, + std::uint32_t &connectivity_checksum) { static_assert(std::is_same::value || std::is_same::value || @@ -28,12 +30,14 @@ inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_da const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; - serialization::read(reader, turn_data); + serialization::read(reader, turn_data, connectivity_checksum); } // writes .osrm.edges template -inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &turn_data) +inline void writeTurnData(const boost::filesystem::path &path, + const TurnDataT &turn_data, + const std::uint32_t connectivity_checksum) { static_assert(std::is_same::value || std::is_same::value || @@ -42,7 +46,7 @@ inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT & const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; - serialization::write(writer, turn_data); + serialization::write(writer, turn_data, connectivity_checksum); } } } diff --git a/include/guidance/guidance_processing.hpp b/include/guidance/guidance_processing.hpp index 192a580b41e..0b4034e242e 100644 --- a/include/guidance/guidance_processing.hpp +++ b/include/guidance/guidance_processing.hpp @@ -41,7 +41,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, guidance::TurnDataExternalContainer &turn_data_container, BearingClassesVector &bearing_class_by_node_based_node, BearingClassesMap &bearing_class_hash, - EntryClassesMap &entry_class_hash); + EntryClassesMap &entry_class_hash, + std::uint32_t &connectivity_checksum); } // namespace customizer } // namespace osrm diff --git a/include/guidance/serialization.hpp b/include/guidance/serialization.hpp index d37f32bec01..b41d68927ed 100644 --- a/include/guidance/serialization.hpp +++ b/include/guidance/serialization.hpp @@ -18,24 +18,28 @@ namespace serialization // read/write for turn data file template inline void read(storage::io::FileReader &reader, - guidance::detail::TurnDataContainerImpl &turn_data_container) + guidance::detail::TurnDataContainerImpl &turn_data_container, + std::uint32_t &connectivity_checksum) { storage::serialization::read(reader, turn_data_container.turn_instructions); storage::serialization::read(reader, turn_data_container.lane_data_ids); storage::serialization::read(reader, turn_data_container.entry_class_ids); storage::serialization::read(reader, turn_data_container.pre_turn_bearings); storage::serialization::read(reader, turn_data_container.post_turn_bearings); + reader.ReadInto(connectivity_checksum); } template inline void write(storage::io::FileWriter &writer, - const guidance::detail::TurnDataContainerImpl &turn_data_container) + const guidance::detail::TurnDataContainerImpl &turn_data_container, + const std::uint32_t connectivity_checksum) { storage::serialization::write(writer, turn_data_container.turn_instructions); storage::serialization::write(writer, turn_data_container.lane_data_ids); storage::serialization::write(writer, turn_data_container.entry_class_ids); storage::serialization::write(writer, turn_data_container.pre_turn_bearings); storage::serialization::write(writer, turn_data_container.post_turn_bearings); + writer.WriteOne(connectivity_checksum); } } } diff --git a/include/guidance/turn_data_container.hpp b/include/guidance/turn_data_container.hpp index 91261f3f3b8..7f9501d3810 100644 --- a/include/guidance/turn_data_container.hpp +++ b/include/guidance/turn_data_container.hpp @@ -24,11 +24,14 @@ template class TurnDataContainerImpl; namespace serialization { template -void read(storage::io::FileReader &reader, detail::TurnDataContainerImpl &turn_data); +void read(storage::io::FileReader &reader, + detail::TurnDataContainerImpl &turn_data, + std::uint32_t &connectivity_checksum); template void write(storage::io::FileWriter &writer, - const detail::TurnDataContainerImpl &turn_data); + const detail::TurnDataContainerImpl &turn_data, + const std::uint32_t connectivity_checksum); } struct TurnData @@ -95,9 +98,11 @@ template class TurnDataContainerImpl } friend void serialization::read(storage::io::FileReader &reader, - TurnDataContainerImpl &turn_data_container); + TurnDataContainerImpl &turn_data_container, + std::uint32_t &connectivity_checksum); friend void serialization::write(storage::io::FileWriter &writer, - const TurnDataContainerImpl &turn_data_container); + const TurnDataContainerImpl &turn_data_container, + const std::uint32_t connectivity_checksum); private: Vector turn_instructions; diff --git a/include/partition/edge_based_graph.hpp b/include/partition/edge_based_graph.hpp index 5c8d656c6ba..3168b199b81 100644 --- a/include/partition/edge_based_graph.hpp +++ b/include/partition/edge_based_graph.hpp @@ -36,6 +36,16 @@ struct DynamicEdgeBasedGraph : util::DynamicGraph { using Base = util::DynamicGraph; using Base::Base; + + template + DynamicEdgeBasedGraph(const NodeIterator nodes, + const ContainerT &graph, + std::uint32_t connectivity_checksum) + : Base(nodes, graph), connectivity_checksum(connectivity_checksum) + { + } + + std::uint32_t connectivity_checksum; }; struct DynamicEdgeBasedGraphEdge : DynamicEdgeBasedGraph::InputEdge diff --git a/include/partition/edge_based_graph_reader.hpp b/include/partition/edge_based_graph_reader.hpp index 7456bce612f..1d57f531ef8 100644 --- a/include/partition/edge_based_graph_reader.hpp +++ b/include/partition/edge_based_graph_reader.hpp @@ -187,12 +187,13 @@ inline DynamicEdgeBasedGraph LoadEdgeBasedGraph(const boost::filesystem::path &p { EdgeID number_of_edge_based_nodes; std::vector edges; - extractor::files::readEdgeBasedGraph(path, number_of_edge_based_nodes, edges); + std::uint32_t checksum; + extractor::files::readEdgeBasedGraph(path, number_of_edge_based_nodes, edges, checksum); auto directed = splitBidirectionalEdges(edges); auto tidied = prepareEdgesForUsageInGraph(std::move(directed)); - return DynamicEdgeBasedGraph(number_of_edge_based_nodes, std::move(tidied)); + return DynamicEdgeBasedGraph(number_of_edge_based_nodes, std::move(tidied), checksum); } } // ns partition diff --git a/include/partition/files.hpp b/include/partition/files.hpp index f1b69b16424..251772398dc 100644 --- a/include/partition/files.hpp +++ b/include/partition/files.hpp @@ -16,7 +16,9 @@ namespace files // reads .osrm.mldgr file template -inline void readGraph(const boost::filesystem::path &path, MultiLevelGraphT &graph) +inline void readGraph(const boost::filesystem::path &path, + MultiLevelGraphT &graph, + std::uint32_t &connectivity_checksum) { static_assert(std::is_same::value || std::is_same::value, @@ -25,12 +27,14 @@ inline void readGraph(const boost::filesystem::path &path, MultiLevelGraphT &gra const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; - serialization::read(reader, graph); + serialization::read(reader, graph, connectivity_checksum); } // writes .osrm.mldgr file template -inline void writeGraph(const boost::filesystem::path &path, const MultiLevelGraphT &graph) +inline void writeGraph(const boost::filesystem::path &path, + const MultiLevelGraphT &graph, + const std::uint32_t connectivity_checksum) { static_assert(std::is_same::value || std::is_same::value, @@ -39,7 +43,7 @@ inline void writeGraph(const boost::filesystem::path &path, const MultiLevelGrap const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; - serialization::write(writer, graph); + serialization::write(writer, graph, connectivity_checksum); } // read .osrm.partition file diff --git a/include/partition/multi_level_graph.hpp b/include/partition/multi_level_graph.hpp index c0aa2393bfc..f04dfe17bbf 100644 --- a/include/partition/multi_level_graph.hpp +++ b/include/partition/multi_level_graph.hpp @@ -24,10 +24,14 @@ template class MultiLevelGrap namespace serialization { template -void read(storage::io::FileReader &reader, MultiLevelGraph &graph); +void read(storage::io::FileReader &reader, + MultiLevelGraph &graph, + std::uint32_t &connectivity_checksum); template -void write(storage::io::FileWriter &writer, const MultiLevelGraph &graph); +void write(storage::io::FileWriter &writer, + const MultiLevelGraph &graph, + const std::uint32_t connectivity_checksum); } template @@ -199,12 +203,15 @@ class MultiLevelGraph : public util::StaticGraph friend void serialization::read(storage::io::FileReader &reader, - MultiLevelGraph &graph); + MultiLevelGraph &graph, + std::uint32_t &connectivity_checksum); friend void serialization::write(storage::io::FileWriter &writer, - const MultiLevelGraph &graph); + const MultiLevelGraph &graph, + const std::uint32_t connectivity_checksum); Vector node_to_edge_offset; + std::uint32_t connectivity_checksum; }; } } diff --git a/include/partition/serialization.hpp b/include/partition/serialization.hpp index 1c84557d9d0..8dd66b9b7c8 100644 --- a/include/partition/serialization.hpp +++ b/include/partition/serialization.hpp @@ -18,20 +18,25 @@ namespace serialization { template -inline void read(storage::io::FileReader &reader, MultiLevelGraph &graph) +inline void read(storage::io::FileReader &reader, + MultiLevelGraph &graph, + std::uint32_t &connectivity_checksum) { storage::serialization::read(reader, graph.node_array); storage::serialization::read(reader, graph.edge_array); storage::serialization::read(reader, graph.node_to_edge_offset); + reader.ReadInto(connectivity_checksum); } template inline void write(storage::io::FileWriter &writer, - const MultiLevelGraph &graph) + const MultiLevelGraph &graph, + const std::uint32_t connectivity_checksum) { storage::serialization::write(writer, graph.node_array); storage::serialization::write(writer, graph.edge_array); storage::serialization::write(writer, graph.node_to_edge_offset); + writer.WriteOne(connectivity_checksum); } template diff --git a/include/updater/updater.hpp b/include/updater/updater.hpp index 78f881a9a84..08b451f0c1f 100644 --- a/include/updater/updater.hpp +++ b/include/updater/updater.hpp @@ -17,12 +17,14 @@ class Updater public: Updater(UpdaterConfig config_) : config(std::move(config_)) {} - using NumNodesAndEdges = std::tuple>; + using NumNodesAndEdges = + std::tuple, std::uint32_t>; NumNodesAndEdges LoadAndUpdateEdgeExpandedGraph() const; EdgeID LoadAndUpdateEdgeExpandedGraph(std::vector &edge_based_edge_list, - std::vector &node_weights) const; + std::vector &node_weights, + std::uint32_t &connectivity_checksum) const; private: UpdaterConfig config; diff --git a/include/util/connectivity_checksum.hpp b/include/util/connectivity_checksum.hpp new file mode 100644 index 00000000000..7fbb2dacf48 --- /dev/null +++ b/include/util/connectivity_checksum.hpp @@ -0,0 +1,88 @@ +#ifndef CONNECTIVITY_CHECKSUM_HPP +#define CONNECTIVITY_CHECKSUM_HPP + +#include + +#include + +#include + +namespace osrm +{ +namespace util +{ + +struct ConnectivityChecksum +{ + ConnectivityChecksum() : checksum(0), length(0), byte_number(0), bit_number(0) {} + + void process_byte(unsigned char byte) + { + BOOST_ASSERT(byte_number < buffer.size()); + + if (bit_number > 0) + { + bit_number = 0; + ++byte_number; + ++length; + } + flush_bytes(); + + buffer[byte_number] = byte; + ++byte_number; + ++length; + flush_bytes(); + + buffer[byte_number] = 0; + } + + void process_bit(bool bit) + { + BOOST_ASSERT(byte_number < buffer.size()); + BOOST_ASSERT(bit_number < CHAR_BIT); + + buffer[byte_number] = (buffer[byte_number] << 1) | static_cast(bit); + if (++bit_number >= CHAR_BIT) + { + bit_number = 0; + ++byte_number; + ++length; + flush_bytes(); + buffer[byte_number] = 0; + } + } + + uLong update_checksum(uLong current) + { + if (bit_number > 0) + { + ++byte_number; + ++length; + } + checksum = crc32_z(checksum, buffer.data(), byte_number); + checksum = crc32_combine(current, checksum, length); + length = byte_number = bit_number = 0; + buffer[byte_number] = 0; + return checksum; + } + + private: + void flush_bytes() + { + if (byte_number >= buffer.size()) + { + checksum = crc32_z(checksum, buffer.data(), buffer.size()); + byte_number = 0; + } + } + + std::array buffer; + uLong checksum; + std::size_t length; + std::size_t byte_number; + unsigned char bit_number; +}; +} +} + +#endif diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index 9fc82c967fa..7ff05d4b07f 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -70,8 +70,9 @@ int Contractor::Run() std::vector edge_based_edge_list; updater::Updater updater(config.updater_config); - EdgeID number_of_edge_based_nodes = - updater.LoadAndUpdateEdgeExpandedGraph(edge_based_edge_list, node_weights); + std::uint32_t connectivity_checksum = 0; + EdgeID number_of_edge_based_nodes = updater.LoadAndUpdateEdgeExpandedGraph( + edge_based_edge_list, node_weights, connectivity_checksum); // Contracting the edge-expanded graph @@ -103,7 +104,8 @@ int Contractor::Run() util::Log() << "Contracted graph has " << query_graph.GetNumberOfEdges() << " edges."; util::Log() << "Contraction took " << TIMER_SEC(contraction) << " sec"; - files::writeGraph(config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters); + files::writeGraph( + config.GetPath(".osrm.hsgr"), checksum, query_graph, edge_filters, connectivity_checksum); TIMER_STOP(preparing); diff --git a/src/customize/customizer.cpp b/src/customize/customizer.cpp index 2b926c2444e..d7e6507c594 100644 --- a/src/customize/customizer.cpp +++ b/src/customize/customizer.cpp @@ -79,13 +79,15 @@ void CellStorageStatistics(const Graph &graph, } auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config, - const partition::MultiLevelPartition &mlp) + const partition::MultiLevelPartition &mlp, + std::uint32_t &connectivity_checksum) { updater::Updater updater(config.updater_config); EdgeID num_nodes; std::vector edge_based_edge_list; - std::tie(num_nodes, edge_based_edge_list) = updater.LoadAndUpdateEdgeExpandedGraph(); + std::tie(num_nodes, edge_based_edge_list, connectivity_checksum) = + updater.LoadAndUpdateEdgeExpandedGraph(); auto directed = partition::splitBidirectionalEdges(edge_based_edge_list); auto tidied = @@ -120,7 +122,8 @@ int Customizer::Run(const CustomizationConfig &config) partition::MultiLevelPartition mlp; partition::files::readPartition(config.GetPath(".osrm.partition"), mlp); - auto graph = LoadAndUpdateEdgeExpandedGraph(config, mlp); + std::uint32_t connectivity_checksum = 0; + auto graph = LoadAndUpdateEdgeExpandedGraph(config, mlp, connectivity_checksum); util::Log() << "Loaded edge based graph: " << graph.GetNumberOfEdges() << " edges, " << graph.GetNumberOfNodes() << " nodes"; @@ -148,7 +151,7 @@ int Customizer::Run(const CustomizationConfig &config) util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds"; TIMER_START(writing_graph); - partition::files::writeGraph(config.GetPath(".osrm.mldgr"), graph); + partition::files::writeGraph(config.GetPath(".osrm.mldgr"), graph, connectivity_checksum); TIMER_STOP(writing_graph); util::Log() << "Graph writing took " << TIMER_SEC(writing_graph) << " seconds"; diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index edf548925d6..9a5ac6570db 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -11,6 +11,7 @@ #include "util/assert.hpp" #include "util/bearing.hpp" +#include "util/connectivity_checksum.hpp" #include "util/coordinate.hpp" #include "util/coordinate_calculation.hpp" #include "util/exception.hpp" @@ -20,10 +21,10 @@ #include "util/timing_util.hpp" #include +#include #include #include -#include "boost/unordered_map.hpp" #include #include #include @@ -70,11 +71,12 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory( const util::NameTable &name_table, const std::unordered_set &segregated_edges, const extractor::LaneDescriptionMap &lane_description_map) - : m_edge_based_node_container(node_data_container), m_number_of_edge_based_nodes(0), - m_coordinates(coordinates), m_node_based_graph(std::move(node_based_graph)), - m_barrier_nodes(barrier_nodes), m_traffic_lights(traffic_lights), - m_compressed_edge_container(compressed_edge_container), name_table(name_table), - segregated_edges(segregated_edges), lane_description_map(lane_description_map) + : m_edge_based_node_container(node_data_container), m_connectivity_checksum(0), + m_number_of_edge_based_nodes(0), m_coordinates(coordinates), + m_node_based_graph(std::move(node_based_graph)), m_barrier_nodes(barrier_nodes), + m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container), + name_table(name_table), segregated_edges(segregated_edges), + lane_description_map(lane_description_map) { } @@ -104,6 +106,11 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodeWeights(std::vector &out swap(m_edge_based_node_weights, output_node_weights); } +std::uint32_t EdgeBasedGraphFactory::GetConnectivityChecksum() const +{ + return m_connectivity_checksum; +} + std::uint64_t EdgeBasedGraphFactory::GetNumberOfEdgeBasedNodes() const { return m_number_of_edge_based_nodes; @@ -474,9 +481,13 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( std::vector continuous_data; // may need this std::vector delayed_data; // may need this std::vector conditionals; + + util::ConnectivityChecksum checksum; }; using EdgesPipelineBufferPtr = std::shared_ptr; + m_connectivity_checksum = 0; + // going over all nodes (which form the center of an intersection), we compute all possible // turns along these intersections. NodeID current_node = 0; @@ -486,8 +497,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( // serial final stage time to complete its tasks. const constexpr unsigned GRAINSIZE = 100; - // First part of the pipeline generates iterator ranges of IDs in sets of - // GRAINSIZE + // First part of the pipeline generates iterator ranges of IDs in sets of GRAINSIZE tbb::filter_t> generator_stage( tbb::filter::serial_in_order, [&](tbb::flow_control &fc) { if (current_node < node_count) @@ -669,6 +679,9 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( mergable_road_detector, intersection_node); + buffer->checksum.process_byte(incoming_edges.size()); + buffer->checksum.process_byte(outgoing_edges.size()); + // all nodes in the graph are connected in both directions. We check all // outgoing nodes to find the incoming edge. This is a larger search overhead, // but the cost we need to pay to generate edges here is worth the additional @@ -709,14 +722,18 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( for (const auto &outgoing_edge : outgoing_edges) { - if (!intersection::isTurnAllowed(m_node_based_graph, - m_edge_based_node_container, - node_restriction_map, - m_barrier_nodes, - edge_geometries, - turn_lanes_data, - incoming_edge, - outgoing_edge)) + auto is_turn_allowed = + intersection::isTurnAllowed(m_node_based_graph, + m_edge_based_node_container, + node_restriction_map, + m_barrier_nodes, + edge_geometries, + turn_lanes_data, + incoming_edge, + outgoing_edge); + buffer->checksum.process_bit(is_turn_allowed); + + if (!is_turn_allowed) continue; const auto turn = @@ -954,6 +971,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( routing_progress.PrintAddition(buffer->nodes_processed); + m_connectivity_checksum = buffer->checksum.update_checksum(m_connectivity_checksum); + // Copy data from local buffers into global EBG data std::for_each( buffer->continuous_data.begin(), buffer->continuous_data.end(), transfer_data); diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index ceefd449b2b..1dd7a4e0107 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -215,6 +215,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) util::DeallocatingVector edge_based_edge_list; std::vector node_is_startpoint; std::vector edge_based_node_weights; + std::uint32_t ebg_connectivity_checksum = 0; // Create a node-based graph from the OSRM file NodeBasedGraphFactory node_based_graph_factory(config.GetPath(".osrm"), @@ -293,7 +294,8 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) edge_based_node_segments, node_is_startpoint, edge_based_node_weights, - edge_based_edge_list); + edge_based_edge_list, + ebg_connectivity_checksum); ProcessGuidanceTurns(node_based_graph, edge_based_nodes_container, @@ -339,8 +341,10 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) util::Log() << "Writing edge-based-graph edges ... " << std::flush; TIMER_START(write_edges); - files::writeEdgeBasedGraph( - config.GetPath(".osrm.ebg"), number_of_edge_based_nodes, edge_based_edge_list); + files::writeEdgeBasedGraph(config.GetPath(".osrm.ebg"), + number_of_edge_based_nodes, + edge_based_edge_list, + ebg_connectivity_checksum); TIMER_STOP(write_edges); util::Log() << "ok, after " << TIMER_SEC(write_edges) << "s"; @@ -695,7 +699,8 @@ EdgeID Extractor::BuildEdgeExpandedGraph( std::vector &edge_based_node_segments, std::vector &node_is_startpoint, std::vector &edge_based_node_weights, - util::DeallocatingVector &edge_based_edge_list) + util::DeallocatingVector &edge_based_edge_list, + std::uint32_t &connectivity_checksum) { EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph, edge_based_nodes_container, @@ -741,6 +746,7 @@ EdgeID Extractor::BuildEdgeExpandedGraph( edge_based_graph_factory.GetEdgeBasedNodeSegments(edge_based_node_segments); edge_based_graph_factory.GetStartPointMarkers(node_is_startpoint); edge_based_graph_factory.GetEdgeBasedNodeWeights(edge_based_node_weights); + connectivity_checksum = edge_based_graph_factory.GetConnectivityChecksum(); return number_of_edge_based_nodes; } @@ -862,6 +868,7 @@ void Extractor::ProcessGuidanceTurns( osrm::guidance::BearingClassesVector bearing_class_by_node_based_node; osrm::guidance::BearingClassesMap bearing_class_hash; osrm::guidance::EntryClassesMap entry_class_hash; + std::uint32_t connectivity_checksum = 0; { SuffixTable street_name_suffix_table(scripting_environment); @@ -890,7 +897,8 @@ void Extractor::ProcessGuidanceTurns( turn_data_container, bearing_class_by_node_based_node, bearing_class_hash, - entry_class_hash); + entry_class_hash, + connectivity_checksum); } util::Log() << "Writing Intersection Classification Data"; @@ -918,8 +926,8 @@ void Extractor::ProcessGuidanceTurns( config.GetPath(".osrm.tls"), turn_lane_offsets, turn_lane_masks); } - osrm::guidance::files::writeTurnData(config.GetPath(".osrm.edges").string(), - turn_data_container); + osrm::guidance::files::writeTurnData( + config.GetPath(".osrm.edges").string(), turn_data_container, connectivity_checksum); TIMER_STOP(write_guidance_data); util::Log() << "ok, after " << TIMER_SEC(write_guidance_data) << "s"; } diff --git a/src/guidance/guidance_processing.cpp b/src/guidance/guidance_processing.cpp index 7137a933ee5..760a09a5af9 100644 --- a/src/guidance/guidance_processing.cpp +++ b/src/guidance/guidance_processing.cpp @@ -5,6 +5,7 @@ #include "extractor/intersection/intersection_analysis.hpp" #include "util/assert.hpp" +#include "util/connectivity_checksum.hpp" #include "util/percent.hpp" #include @@ -31,7 +32,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, guidance::TurnDataExternalContainer &turn_data_container, BearingClassesVector &bearing_class_by_node_based_node, BearingClassesMap &bearing_class_hash, - EntryClassesMap &entry_class_hash) + EntryClassesMap &entry_class_hash, + std::uint32_t &connectivity_checksum) { util::Log() << "Generating guidance turns "; @@ -75,6 +77,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, std::vector continuous_turn_data; // populate answers from guidance std::vector delayed_turn_data; // populate answers from guidance + + util::ConnectivityChecksum checksum; }; using TurnsPipelineBufferPtr = std::shared_ptr; @@ -84,6 +88,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, const NodeID node_count = node_based_graph.GetNumberOfNodes(); NodeID current_node = 0; + connectivity_checksum = 0; + // Handle intersections in sets of 100. The pipeline below has a serial bottleneck // during the writing phase, so we want to make the parallel workers do more work // to give the serial final stage time to complete its tasks. @@ -136,6 +142,9 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, const auto &edge_geometries = edge_geometries_and_merged_edges.first; const auto &merged_edge_ids = edge_geometries_and_merged_edges.second; + buffer->checksum.process_byte(incoming_edges.size()); + buffer->checksum.process_byte(outgoing_edges.size()); + // all nodes in the graph are connected in both directions. We check all // outgoing nodes to find the incoming edge. This is a larger search overhead, // but the cost we need to pay to generate edges here is worth the additional @@ -208,6 +217,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, incoming_edge, outgoing_edge); + buffer->checksum.process_bit(is_turn_allowed); + if (!is_turn_allowed) continue; @@ -291,6 +302,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph, guidance_progress.PrintAddition(buffer->nodes_processed); + connectivity_checksum = buffer->checksum.update_checksum(connectivity_checksum); + // Guidance data std::for_each(buffer->continuous_turn_data.begin(), buffer->continuous_turn_data.end(), diff --git a/src/partition/partitioner.cpp b/src/partition/partitioner.cpp index de337f4e314..f3b888d09f6 100644 --- a/src/partition/partitioner.cpp +++ b/src/partition/partitioner.cpp @@ -223,7 +223,8 @@ int Partitioner::Run(const PartitionConfig &config) files::writeCells(config.GetPath(".osrm.cells"), storage); extractor::files::writeEdgeBasedGraph(config.GetPath(".osrm.ebg"), edge_based_graph.GetNumberOfNodes(), - graphToEdges(edge_based_graph)); + graphToEdges(edge_based_graph), + edge_based_graph.connectivity_checksum); TIMER_STOP(writing_mld_data); util::Log() << "MLD data writing took " << TIMER_SEC(writing_mld_data) << " seconds"; diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 898b77bae71..4500e2e54da 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -572,44 +572,10 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) { BOOST_ASSERT(memory_ptr != nullptr); - // read actual data into shared memory object // + // Connectivity matrix checksum + std::uint32_t turns_connectivity_checksum = 0; - // Load the HSGR file - if (boost::filesystem::exists(config.GetPath(".osrm.hsgr"))) - { - auto graph_nodes_ptr = layout.GetBlockPtr( - memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST); - auto graph_edges_ptr = layout.GetBlockPtr( - memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST); - auto checksum = layout.GetBlockPtr(memory_ptr, DataLayout::HSGR_CHECKSUM); - - util::vector_view node_list( - graph_nodes_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_NODE_LIST]); - util::vector_view edge_list( - graph_edges_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_EDGE_LIST]); - - std::vector> edge_filter; - for (auto index : util::irange(0, NUM_METRICS)) - { - auto block_id = - static_cast(storage::DataLayout::CH_EDGE_FILTER_0 + index); - auto data_ptr = layout.GetBlockPtr(memory_ptr, block_id); - auto num_entries = layout.num_entries[block_id]; - edge_filter.emplace_back(data_ptr, num_entries); - } - - contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list)); - contractor::files::readGraph( - config.GetPath(".osrm.hsgr"), *checksum, graph_view, edge_filter); - } - else - { - layout.GetBlockPtr(memory_ptr, DataLayout::HSGR_CHECKSUM); - layout.GetBlockPtr( - memory_ptr, DataLayout::CH_GRAPH_NODE_LIST); - layout.GetBlockPtr( - memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST); - } + // read actual data into shared memory object // // store the filename of the on-disk portion of the RTree { @@ -724,7 +690,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) std::move(pre_turn_bearings), std::move(post_turn_bearings)); - guidance::files::readTurnData(config.GetPath(".osrm.edges"), turn_data); + guidance::files::readTurnData( + config.GetPath(".osrm.edges"), turn_data, turns_connectivity_checksum); } // load compressed geometry @@ -921,8 +888,60 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) config.GetPath(".osrm.icd"), intersection_bearings_view, entry_classes); } - { - // Loading MLD Data + { // Load the HSGR file + if (boost::filesystem::exists(config.GetPath(".osrm.hsgr"))) + { + auto graph_nodes_ptr = + layout.GetBlockPtr( + memory_ptr, storage::DataLayout::CH_GRAPH_NODE_LIST); + auto graph_edges_ptr = + layout.GetBlockPtr( + memory_ptr, storage::DataLayout::CH_GRAPH_EDGE_LIST); + auto checksum = + layout.GetBlockPtr(memory_ptr, DataLayout::HSGR_CHECKSUM); + + util::vector_view node_list( + graph_nodes_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_NODE_LIST]); + util::vector_view edge_list( + graph_edges_ptr, layout.num_entries[storage::DataLayout::CH_GRAPH_EDGE_LIST]); + + std::vector> edge_filter; + for (auto index : util::irange(0, NUM_METRICS)) + { + auto block_id = + static_cast(storage::DataLayout::CH_EDGE_FILTER_0 + index); + auto data_ptr = layout.GetBlockPtr(memory_ptr, block_id); + auto num_entries = layout.num_entries[block_id]; + edge_filter.emplace_back(data_ptr, num_entries); + } + + std::uint32_t graph_connectivity_checksum = 0; + contractor::QueryGraphView graph_view(std::move(node_list), std::move(edge_list)); + contractor::files::readGraph(config.GetPath(".osrm.hsgr"), + *checksum, + graph_view, + edge_filter, + graph_connectivity_checksum); + if (turns_connectivity_checksum != graph_connectivity_checksum) + { + throw util::exception( + "Connectivity checksum " + std::to_string(graph_connectivity_checksum) + + " in " + config.GetPath(".osrm.hsgr").string() + + " does not equal to checksum " + std::to_string(turns_connectivity_checksum) + + " in " + config.GetPath(".osrm.edges").string()); + } + } + else + { + layout.GetBlockPtr(memory_ptr, DataLayout::HSGR_CHECKSUM); + layout.GetBlockPtr( + memory_ptr, DataLayout::CH_GRAPH_NODE_LIST); + layout.GetBlockPtr( + memory_ptr, DataLayout::CH_GRAPH_EDGE_LIST); + } + } + + { // Loading MLD Data if (boost::filesystem::exists(config.GetPath(".osrm.partition"))) { BOOST_ASSERT(layout.GetBlockSize(storage::DataLayout::MLD_LEVEL_DATA) > 0); @@ -1040,9 +1059,20 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) graph_node_to_offset_ptr, layout.num_entries[storage::DataLayout::MLD_GRAPH_NODE_TO_OFFSET]); + std::uint32_t graph_connectivity_checksum = 0; customizer::MultiLevelEdgeBasedGraphView graph_view( std::move(node_list), std::move(edge_list), std::move(node_to_offset)); - partition::files::readGraph(config.GetPath(".osrm.mldgr"), graph_view); + partition::files::readGraph( + config.GetPath(".osrm.mldgr"), graph_view, graph_connectivity_checksum); + + if (turns_connectivity_checksum != graph_connectivity_checksum) + { + throw util::exception( + "Connectivity checksum " + std::to_string(graph_connectivity_checksum) + + " in " + config.GetPath(".osrm.mldgr").string() + + " does not equal to checksum " + std::to_string(turns_connectivity_checksum) + + " in " + config.GetPath(".osrm.edges").string()); + } } } } diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index 8d594f94238..7ab17990ebe 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -524,14 +524,17 @@ Updater::NumNodesAndEdges Updater::LoadAndUpdateEdgeExpandedGraph() const { std::vector node_weights; std::vector edge_based_edge_list; - auto number_of_edge_based_nodes = - Updater::LoadAndUpdateEdgeExpandedGraph(edge_based_edge_list, node_weights); - return std::make_tuple(number_of_edge_based_nodes, std::move(edge_based_edge_list)); + std::uint32_t connectivity_checksum; + auto number_of_edge_based_nodes = Updater::LoadAndUpdateEdgeExpandedGraph( + edge_based_edge_list, node_weights, connectivity_checksum); + return std::make_tuple( + number_of_edge_based_nodes, std::move(edge_based_edge_list), connectivity_checksum); } EdgeID Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &edge_based_edge_list, - std::vector &node_weights) const + std::vector &node_weights, + std::uint32_t &connectivity_checksum) const { TIMER_START(load_edges); @@ -539,8 +542,10 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e std::vector coordinates; extractor::PackedOSMIDs osm_node_ids; - extractor::files::readEdgeBasedGraph( - config.GetPath(".osrm.ebg"), number_of_edge_based_nodes, edge_based_edge_list); + extractor::files::readEdgeBasedGraph(config.GetPath(".osrm.ebg"), + number_of_edge_based_nodes, + edge_based_edge_list, + connectivity_checksum); extractor::files::readNodes(config.GetPath(".osrm.nbg_nodes"), coordinates, osm_node_ids); const bool update_conditional_turns = diff --git a/unit_tests/util/connectivity_checksum.cpp b/unit_tests/util/connectivity_checksum.cpp new file mode 100644 index 00000000000..10b502e9fd1 --- /dev/null +++ b/unit_tests/util/connectivity_checksum.cpp @@ -0,0 +1,96 @@ +#include "util/connectivity_checksum.hpp" + +#include +#include + +BOOST_AUTO_TEST_SUITE(connectivity_checksum) + +using namespace osrm; +using namespace osrm::util; + +BOOST_AUTO_TEST_CASE(connectivity_checksum) +{ + ConnectivityChecksum checksum; + + BOOST_CHECK_EQUAL(checksum.update_checksum(42), 42); + + checksum.process_bit(1); + BOOST_CHECK_EQUAL(checksum.update_checksum(0), 0x7ebe16cd); + + for (std::size_t i = 0; i < CHAR_BIT; ++i) + checksum.process_bit(1); + BOOST_CHECK_EQUAL(checksum.update_checksum(0), 0x1aab001b); + + checksum.process_byte(1); + BOOST_CHECK_EQUAL(checksum.update_checksum(0), 0x2f7abdf7); +} + +BOOST_AUTO_TEST_CASE(connectivity_checksum_bytes_bits) +{ + ConnectivityChecksum checksum1, checksum2; + + checksum1.process_bit(1); // A + checksum1.process_bit(0); + checksum1.process_bit(1); + checksum1.process_bit(0); + + checksum1.process_bit(1); // 9 + checksum1.process_bit(0); + checksum1.process_bit(0); + checksum1.process_bit(1); + + checksum1.process_bit(1); // 5 + checksum1.process_bit(0); + checksum1.process_bit(1); + + checksum2.process_byte(0xA9); + checksum2.process_byte(0x05); + BOOST_CHECK_EQUAL(checksum1.update_checksum(0), checksum2.update_checksum(0)); +} + +BOOST_AUTO_TEST_CASE(connectivity_checksum_diff_size) +{ + ConnectivityChecksum checksum1, checksum2; + + checksum1.process_byte(1); + checksum1.process_byte(2); + checksum1.process_byte(3); + checksum1.process_byte(4); + checksum1.process_byte(5); + checksum1.process_bit(1); + + checksum2.process_byte(1); + checksum2.process_byte(2); + checksum2.process_byte(3); + checksum2.update_checksum(0); + checksum2.process_byte(4); + checksum2.process_byte(5); + checksum2.process_bit(1); + + BOOST_CHECK_EQUAL(checksum1.update_checksum(0), checksum2.update_checksum(0)); +} + +BOOST_AUTO_TEST_CASE(connectivity_checksum_parallel) +{ + ConnectivityChecksum expected, checksum1, checksum2; + + expected.process_byte(1); + expected.process_byte(2); + expected.process_byte(3); + expected.process_byte(4); + expected.process_byte(5); + expected.process_bit(1); + + checksum1.process_byte(1); + checksum1.process_byte(2); + checksum1.process_byte(3); + + checksum2.process_byte(4); + checksum2.process_byte(5); + checksum2.process_bit(1); + + BOOST_CHECK_EQUAL(expected.update_checksum(0), + checksum2.update_checksum(checksum1.update_checksum(0))); +} + +BOOST_AUTO_TEST_SUITE_END()