osrm-routed monitoring in the prometheus format

CHANGELOG.md

@@ -7,6 +7,8 @@
       - CHANGED: Upgrade Boost dependency to 1.70 [#6113](https://github.com/Project-OSRM/osrm-backend/pull/6113)
       - CHANGED: Upgrade Ubuntu CI builds to 20.04  [#6119](https://github.com/Project-OSRM/osrm-backend/pull/6119)
       - CHANGED: Make building osrm-routed optional [#6144](https://github.com/Project-OSRM/osrm-backend/pull/6144)
+    - Features
+      - ADDED: monitoring endpoint in prometheus format [#5180](https://github.com/Project-OSRM/osrm-backend/pull/5180)
 
 # 5.26.0
   - Changes from 5.25.0
@@ -929,4 +931,4 @@
         - `properties.traffic_signal_penalty`
         - `properties.use_turn_restrictions`
         - `properties.u_turn_penalty`
-        - `properties.allow_u_turn_at_via`
+        - `properties.allow_u_turn_at_via`

CMakeLists.txt

@@ -150,7 +150,7 @@ file(GLOB CustomizerGlob src/customize/*.cpp)
 file(GLOB ContractorGlob src/contractor/*.cpp)
 file(GLOB UpdaterGlob src/updater/*.cpp)
 file(GLOB StorageGlob src/storage/*.cpp)
-file(GLOB ServerGlob src/server/*.cpp src/server/**/*.cpp)
+file(GLOB ServerGlob src/server/*.cpp src/server/**/*.cpp src/monitoring/*.cpp)
 file(GLOB EngineGlob src/engine/*.cpp src/engine/**/*.cpp)
 file(GLOB ErrorcodesGlob src/osrm/errorcodes.cpp)
 
@@ -738,7 +738,7 @@ file(GLOB FlatbuffersGlob third_party/flatbuffers/include/flatbuffers/*.h)
 file(GLOB LibraryGlob include/osrm/*.hpp)
 file(GLOB ParametersGlob include/engine/api/*_parameters.hpp)
 set(ApiHeader include/engine/api/base_result.hpp)
-set(EngineHeader include/engine/status.hpp include/engine/engine_config.hpp include/engine/hint.hpp include/engine/bearing.hpp include/engine/approach.hpp include/engine/phantom_node.hpp)
+set(EngineHeader include/engine/status.hpp include/engine/engine_config.hpp include/engine/hint.hpp include/engine/bearing.hpp include/engine/approach.hpp include/engine/phantom_node.hpp include/engine/engine_info.hpp)
 set(UtilHeader include/util/coordinate.hpp include/util/json_container.hpp include/util/typedefs.hpp include/util/alias.hpp include/util/exception.hpp include/util/bearing.hpp)
 set(ExtractorHeader include/extractor/extractor.hpp include/storage/io_config.hpp include/extractor/extractor_config.hpp include/extractor/travel_mode.hpp)
 set(PartitionerHeader include/partitioner/partitioner.hpp include/partitioner/partitioner_config.hpp)

docs/monitoring.md

@@ -0,0 +1,15 @@
+## Abstract
+
+OSRM routed daemon publish health information in prometheus format.
+This option switched off by default. To enable this feature please set `-P <monitoring_port>` option on OSRM routed startup.
+
+
+## Available metrics
+
+- `osrm_routed_instance_info` metric with base instance information inside labels
+-- `algorithm` routing algorithm;
+-- `code_version` OSRM version. Same with `-v` command line option;
+-- `data_version` OSM data version if available. This value can be setted at the `osrm-extract` phase;
+-- `working_threads` actual count of http requests processing threads;
+- `osrm_http_requests_count` http calls count divided by plugins. And `invalid` count for requests were plugin wasn't set;
+- `osrm_workers_busy` http workers count that process request at this moment;

features/lib/osrm_loader.js

@@ -77,7 +77,7 @@ class OSRMDirectLoader extends OSRMBaseLoader {
     osrmUp (callback) {
         if (this.osrmIsRunning()) return callback(new Error("osrm-routed already running!"));
 
-        const command_arguments = util.format('%s -p %d -i %s -a %s', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM);
+        const command_arguments = util.format('%s -p %d -i %s -a %s -P %d', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM, this.scope.OSRM_MONITORING_PORT);
         this.child = this.scope.runBin('osrm-routed', command_arguments, this.scope.environment, (err) => {
             if (err && err.signal !== 'SIGINT') {
                 this.child = null;
@@ -110,7 +110,7 @@ class OSRMmmapLoader extends OSRMBaseLoader {
     osrmUp (callback) {
         if (this.osrmIsRunning()) return callback(new Error("osrm-routed already running!"));
 
-        const command_arguments = util.format('%s -p %d -i %s -a %s --mmap', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM);
+        const command_arguments = util.format('%s -p %d -i %s -a %s -P %d --mmap', this.inputFile, this.scope.OSRM_PORT, this.scope.OSRM_IP, this.scope.ROUTING_ALGORITHM, this.scope.OSRM_MONITORING_PORT);
         this.child = this.scope.runBin('osrm-routed', command_arguments, this.scope.environment, (err) => {
             if (err && err.signal !== 'SIGINT') {
                 this.child = null;
@@ -157,7 +157,7 @@ class OSRMDatastoreLoader extends OSRMBaseLoader {
     osrmUp (callback) {
         if (this.osrmIsRunning()) return callback();
 
-        const command_arguments = util.format('--dataset-name=%s -s -i %s -p %d -a %s', this.scope.DATASET_NAME, this.scope.OSRM_IP, this.scope.OSRM_PORT, this.scope.ROUTING_ALGORITHM);
+        const command_arguments = util.format('--dataset-name=%s -s -i %s -p %d -a %s -P %d', this.scope.DATASET_NAME, this.scope.OSRM_IP, this.scope.OSRM_PORT, this.scope.ROUTING_ALGORITHM, this.scope.OSRM_MONITORING_PORT);
         this.child = this.scope.runBin('osrm-routed', command_arguments, this.scope.environment, (err) => {
             if (err && err.signal !== 'SIGINT') {
                 this.child = null;

features/step_definitions/monitoring.js

@@ -0,0 +1,44 @@
+'use strict';
+
+module.exports = function () {
+    this.When(/^I monitor I should get$/, (table, callback) => {
+        var got = {};
+
+        this.reprocessAndLoadData((e) => {
+            if (e) return callback(e);
+            var testRow = (row, ri, cb) => {
+                var afterRequest = (err, res) => {
+                    if (err) return cb(err);
+
+                    var rows = res.body.split('\n');
+                    var metrics = {};
+                    rows.forEach((r) => {
+                        if (r.includes('{')) {
+                            var k = r.split('{')[0];
+                            metrics[k] = r.split('}')[1].split(' ')[1];
+                        } else {
+                            var kv = r.split(' ');
+                            metrics[kv[0]] = kv[1];
+                        }
+                    });
+
+                    var headers = new Set(table.raw()[0]);
+
+                    if (headers.has('key'))
+                    {
+                        got.key=row['key'];
+                        got.value=metrics[row['key']];
+                    }
+
+
+                    if (!metrics.hasOwnProperty('osrm_routed_instance_info')) {
+                        throw new Error('Have no instance information inside the monitoring!');
+                    }
+                    cb(null, got);
+                };
+                this.requestMonitoring(afterRequest);
+            };
+            this.processRowsAndDiff(table, testRow, callback);
+        });
+    });
+};

features/support/env.js

@@ -39,8 +39,10 @@ module.exports = function () {
         this.TIMEZONE_NAMES = this.PLATFORM_WINDOWS ? 'win' : 'iana';
 
         this.OSRM_PORT = process.env.OSRM_PORT && parseInt(process.env.OSRM_PORT) || 5000;
+        this.OSRM_MONITORING_PORT = process.env.OSRM_MONITORING_PORT && parseInt(process.env.OSRM_MONITORING_PORT) || 5001;
         this.OSRM_IP = process.env.OSRM_IP || '127.0.0.1';
         this.HOST = `http://${this.OSRM_IP}:${this.OSRM_PORT}`;
+        this.MONITORING_HOST = `http://${this.OSRM_IP}:${this.OSRM_MONITORING_PORT}`;
 
         this.OSRM_PROFILE = process.env.OSRM_PROFILE;
 

features/support/route.js

@@ -16,6 +16,24 @@ module.exports = function () {
         return this.sendRequest(uri, params, callback);
     };
 
+    this.requestMonitoring = (callback) => {
+        var uri = this.query = this.MONITORING_HOST,
+            limit = Timeout(this.TIMEOUT, { err: { statusCode: 408 } });
+        function runRequest (cb) {
+            request(uri, cb);
+        }
+
+        runRequest(limit((err, res, body) => {
+            if (err) {
+                if (err.statusCode === 408) throw new Error('*** osrm monitoring endpoint did not respond');
+                else if (err.code === 'ECONNREFUSED')
+                    throw new Error('*** osrm monitoring endpoint is not running');
+            } else
+                return callback(err, res, body);
+        }));
+
+    };
+
     this.requestUrl = (path, callback) => {
         var uri = this.query = [this.HOST, path].join('/'),
             limit = Timeout(this.TIMEOUT, { err: { statusCode: 408 } });

features/testbot/monitoring.feature

@@ -0,0 +1,21 @@
+@routing
+Feature: Basic Prometheus monitoring
+
+    Background:
+        Given the profile "testbot"
+        Given a grid size of 100 meters
+
+    @smallest
+    Scenario: Monitoring fog test
+        Given the node map
+            """
+            a b
+            """
+
+        And the ways
+            | nodes |
+            | ab    |
+
+        When I monitor I should get
+            | key               | value |
+            | osrm_workers_busy | 0     |

include/engine/engine.hpp

@@ -9,6 +9,7 @@
 #include "engine/api/trip_parameters.hpp"
 #include "engine/datafacade_provider.hpp"
 #include "engine/engine_config.hpp"
+#include "engine/engine_info.hpp"
 #include "engine/plugins/match.hpp"
 #include "engine/plugins/nearest.hpp"
 #include "engine/plugins/table.hpp"
@@ -39,6 +40,8 @@ class EngineInterface
     virtual Status Trip(const api::TripParameters &parameters, api::ResultT &result) const = 0;
     virtual Status Match(const api::MatchParameters &parameters, api::ResultT &result) const = 0;
     virtual Status Tile(const api::TileParameters &parameters, api::ResultT &result) const = 0;
+
+    virtual const EngineInfo &GetInfo() const = 0;
 };
 
 template <typename Algorithm> class Engine final : public EngineInterface
@@ -76,6 +79,9 @@ template <typename Algorithm> class Engine final : public EngineInterface
                                 << routing_algorithms::name<Algorithm>();
             facade_provider = std::make_unique<ImmutableProvider<Algorithm>>(config.storage_config);
         }
+        info["algorithm"] = routing_algorithms::name<Algorithm>();
+        info["code_version"] = OSRM_VERSION;
+        info["data_version"] = GetAlgorithms(api::RouteParameters()).GetFacade().GetTimestamp();
     }
 
     Engine(Engine &&) noexcept = delete;
@@ -115,11 +121,14 @@ template <typename Algorithm> class Engine final : public EngineInterface
         return tile_plugin.HandleRequest(GetAlgorithms(params), params, result);
     }
 
+    const EngineInfo &GetInfo() const override final { return info; };
+
   private:
     template <typename ParametersT> auto GetAlgorithms(const ParametersT &params) const
     {
         return RoutingAlgorithms<Algorithm>{heaps, facade_provider->Get(params)};
     }
+    EngineInfo info;
     std::unique_ptr<DataFacadeProvider<Algorithm>> facade_provider;
     mutable SearchEngineData<Algorithm> heaps;
 

include/engine/engine_info.hpp

@@ -0,0 +1,48 @@
+/*
+
+Copyright (c) 2019, Project OSRM contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list
+of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#ifndef ENGINE_INFO_HPP
+#define ENGINE_INFO_HPP
+
+#include <unordered_map>
+
+namespace osrm
+{
+namespace engine
+{
+
+/**
+ * EngineInfo to return information about engine state into the monitoring
+ * \see OSRM
+ */
+
+using EngineInfo = std::unordered_map<std::string, std::string>;
+
+} // namespace engine
+} // namespace osrm
+
+#endif

include/monitoring/monitoring.hpp

@@ -0,0 +1,106 @@
+#ifndef MONITORING_HPP
+#define MONITORING_HPP
+
+#include "monitoring/monitoring_request_handler.hpp"
+#include "server/connection.hpp"
+
+#include "util/log.hpp"
+
+#include <boost/asio.hpp>
+#include <boost/bind.hpp>
+
+#include <string>
+#include <thread>
+
+namespace osrm
+{
+namespace server
+{
+
+class Monitoring
+{
+  public:
+    static std::shared_ptr<Monitoring>
+    CreateMonitoring(std::string &ip_address, const int ip_port, const unsigned working_threads)
+    {
+        return std::make_shared<Monitoring>(ip_address, ip_port, working_threads);
+    }
+
+    explicit Monitoring(const std::string &address, const int port, const unsigned working_threads)
+        : acceptor(io_service), request_handler(working_threads),
+          new_connection(std::make_shared<Connection>(io_service, request_handler))
+    {
+        if (port == 0)
+        {
+            working = false;
+            return;
+        }
+        working = true;
+        const auto port_string = std::to_string(port);
+
+        boost::asio::ip::tcp::resolver resolver(io_service);
+        boost::asio::ip::tcp::resolver::query query(address, port_string);
+        boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query);
+
+        acceptor.open(endpoint.protocol());
+#ifdef SO_REUSEPORT
+        const int option = 1;
+        setsockopt(acceptor.native_handle(), SOL_SOCKET, SO_REUSEPORT, &option, sizeof(option));
+#endif
+        acceptor.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true));
+        acceptor.bind(endpoint);
+        acceptor.listen();
+
+        util::Log() << "Monitoring endpoint listening on: " << acceptor.local_endpoint();
+
+        acceptor.async_accept(
+            new_connection->socket(),
+            boost::bind(&Monitoring::HandleAccept, this, boost::asio::placeholders::error));
+    }
+
+    void Run()
+    {
+        if (working)
+        {
+            io_service.run();
+        }
+    }
+
+    void Stop()
+    {
+        if (working)
+        {
+            io_service.stop();
+        }
+    }
+
+    void RegisterServiceHandler(std::shared_ptr<ServiceHandlerInterface> service_handler_)
+    {
+        request_handler.RegisterServiceHandler(service_handler_);
+    }
+
+  private:
+    void HandleAccept(const boost::system::error_code &e)
+    {
+        if (!e)
+        {
+            new_connection->start();
+            new_connection = std::make_shared<Connection>(io_service, request_handler);
+            acceptor.async_accept(
+                new_connection->socket(),
+                boost::bind(&Monitoring::HandleAccept, this, boost::asio::placeholders::error));
+        }
+    }
+
+    boost::asio::io_service io_service;
+    boost::asio::ip::tcp::acceptor acceptor;
+    MonitoringRequestHandler request_handler;
+    std::shared_ptr<Connection> new_connection;
+    std::shared_ptr<std::thread> working_thread;
+    bool working;
+};
+
+} // namespace server
+} // namespace osrm
+
+#endif // MONITORING_HPP