[ISSUE #340]Integrate With OpenTelemetry for metrics in EventMesh

eventmesh-runtime/build.gradle

@@ -23,14 +23,21 @@ List metrics = [
         "io.dropwizard.metrics:metrics-json:4.1.0"
 ]
 
-
+List open_telemetry = [
+        "io.opentelemetry:opentelemetry-api:1.3.0",
+        "io.opentelemetry:opentelemetry-sdk:1.3.0",
+        "io.opentelemetry:opentelemetry-sdk-metrics:1.3.0-alpha",
+        "io.opentelemetry:opentelemetry-exporter-prometheus:1.3.0-alpha",
+        "io.prometheus:simpleclient:0.8.1",
+        "io.prometheus:simpleclient_httpserver:0.8.1"
+]
 
 List open_message = [
         "io.openmessaging:openmessaging-api:2.2.1-pubsub"
 ]
 
 
 dependencies {
-    implementation metrics, open_message, project(":eventmesh-connector-plugin:eventmesh-connector-api")
-    testImplementation metrics, open_message, project(":eventmesh-connector-plugin:eventmesh-connector-api")
+    implementation metrics, open_telemetry, open_message, project(":eventmesh-connector-plugin:eventmesh-connector-api")
+    testImplementation metrics, open_telemetry,open_message, project(":eventmesh-connector-plugin:eventmesh-connector-api")
 }

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HTTPMetricsServer.java

@@ -26,6 +26,7 @@
 import com.codahale.metrics.MetricRegistry;
 
 import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer;
+import org.apache.eventmesh.runtime.metrics.openTelemetry.OpenTelemetryExporter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,6 +44,8 @@ public class HTTPMetricsServer {
 
     public GroupMetrics groupMetrics;
 
+    public OpenTelemetryExporter openTelemetryExporter;
+
     private Logger httpLogger = LoggerFactory.getLogger("httpMonitor");
 
     private Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -56,10 +59,14 @@ public void init() throws Exception {
         topicMetrics = new TopicMetrics(this.eventMeshHTTPServer, this.metricRegistry);
         groupMetrics = new GroupMetrics(this.eventMeshHTTPServer, this.metricRegistry);
         healthMetrics = new HealthMetrics(this.eventMeshHTTPServer, this.metricRegistry);
+
+        openTelemetryExporter = new OpenTelemetryExporter(summaryMetrics);
+
         logger.info("HTTPMetricsServer inited......");
     }
 
     public void start() throws Exception {
+        openTelemetryExporter.start();
         metricsSchedule.scheduleAtFixedRate(new Runnable() {
             @Override
             public void run() {
@@ -90,6 +97,7 @@ public void run() {
 
     public void shutdown() throws Exception {
         metricsSchedule.shutdown();
+        openTelemetryExporter.shutdown();
         logger.info("HTTPMetricsServer shutdown......");
     }
 

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/openTelemetry/OpenTelemetryExporter.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.runtime.metrics.openTelemetry;
+
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.MeterProvider;
+import io.opentelemetry.api.metrics.common.Labels;
+import org.apache.eventmesh.runtime.metrics.http.SummaryMetrics;
+
+/**
+ * test
+ */
+public class OpenTelemetryExporter {
+    OpenTelemetryExporterConfiguration configuration = new OpenTelemetryExporterConfiguration();
+
+    private SummaryMetrics summaryMetrics;
+
+    private Meter meter;
+
+    public OpenTelemetryExporter(SummaryMetrics summaryMetrics) {
+        this.summaryMetrics = summaryMetrics;
+
+        // it is important to initialize the OpenTelemetry SDK as early as possible in your process.
+        MeterProvider meterProvider = configuration.initializeOpenTelemetry();
+
+        meter = meterProvider.get("OpenTelemetryExporter", "0.13.1");
+    }
+
+    public void start(){
+        //maxHTTPTPS
+        meter
+                .doubleValueObserverBuilder("eventmesh.http.request.tps.elapsed.max")
+                .setDescription("max TPS of HTTP")
+                .setUnit("HTTP")
+                .setUpdater(result -> result.observe(summaryMetrics.maxHTTPTPS(),Labels.empty()))
+                .build();
+
+        //maxHTTPCost
+        meter
+                .longValueObserverBuilder("eventmesh.http.request.elapsed.max")
+                .setDescription("max cost of HTTP")
+                .setUnit("HTTP")
+                .setUpdater(result -> result.observe(summaryMetrics.maxHTTPCost(), Labels.empty()))
+                .build();
+
+        //avgHTTPCost
+        meter
+                .doubleValueObserverBuilder("eventmesh.http.request.elapsed.avg")
+                .setDescription("avg cost of HTTP")
+                .setUnit("HTTP")
+                .setUpdater(result -> result.observe(summaryMetrics.avgHTTPCost(), Labels.empty()))
+                .build();
+    }
+
+    public void shutdown(){
+        configuration.shutdownPrometheusEndpoint();
+    }
+}

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/openTelemetry/OpenTelemetryExporterConfiguration.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.runtime.metrics.openTelemetry;
+
+import io.opentelemetry.api.metrics.MeterProvider;
+import io.opentelemetry.exporter.prometheus.PrometheusCollector;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.prometheus.client.exporter.HTTPServer;
+
+import java.io.IOException;
+
+//ues openTelemetry to export metrics data
+public class OpenTelemetryExporterConfiguration {
+
+    private  HTTPServer server;//Prometheus server
+
+    int prometheusPort = 19090;//the endpoint to export metrics
+
+    /**
+     * Initializes the Meter SDK and configures the prometheus collector with all default settings.
+     *
+     *
+     * @return A MeterProvider for use in instrumentation.
+     */
+    public MeterProvider initializeOpenTelemetry() {
+        SdkMeterProvider meterProvider = SdkMeterProvider.builder().buildAndRegisterGlobal();
+
+        PrometheusCollector.builder().setMetricProducer(meterProvider).buildAndRegister();
+
+        try {
+            server = new HTTPServer(prometheusPort,true);//Use the daemon thread to start an HTTP server to serve the default Prometheus registry.
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
+        return meterProvider;
+    }
+
+    public void shutdownPrometheusEndpoint() {
+        server.stop();
+    }
+}

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/openTelemetry/README.md

@@ -0,0 +1,41 @@
+# Open Telemetry exporter
+
+we can use Prometheus UI to see the metrics exported by openTelemetry
+
+# How to run Prometheus
+
+download Prometheus from https://prometheus.io/download/
+remember to fix the [prometheus.yml](prometheus.yml)
+
+---
+or use docker
+Start Prometheus instance with a configuration that sets up a HTTP collection job for  ```127.0.0.1:19090```
+
+See [prometheus.yml](prometheus.yml)
+
+```shell script
+docker run --network="host" --rm -it \
+    --name prometheus \
+    -v $(pwd)/prometheus.yml:/etc/prometheus/prometheus.yml \
+    prom/prometheus 
+
+```
+
+you can run the quickstart and open the Prometheus UI:
+http://localhost:9090/graph?g0.expr=max_HTTPCost&g0.tab=1&g0.stacked=0&g0.show_exemplars=0&g0.range_input=1h
+
+
+search the key word:
+
+*eventmesh_http_request_tps_elapsed_max*
+
+*eventmesh_http_request_elapsed_max*
+
+*eventmesh_http_request_elapsed_avg*
+
+## special explanation
+Prometheus runs on port 9090,Open telemetry exports data to port 19090,Prometheus will collect data from port 19090
+
+the exporter is exporting the data in 'SummaryMetrics'(package org.apache.eventmesh.runtime.metrics.http;)
+
+The export mechanism I set is to export every 3 seconds, because QuickStart only has httpcost at a short time. If the interval is set too long, it will always be 0. In practical application, I think it should be set to more than 30 seconds

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/openTelemetry/prometheus.yml

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: []
+    scheme: http
+    timeout: 10s
+    api_version: v1
+scrape_configs:
+- job_name: prometheus
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - localhost:9090
+- job_name: EventMesh_HTTP_export_test
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  static_configs:
+  - targets:
+    - 127.0.0.1:19090