diff --git a/grafana/greptimedb.json b/grafana/greptimedb.json index e296c36be2b0..142832ca0cac 100644 --- a/grafana/greptimedb.json +++ b/grafana/greptimedb.json @@ -679,7 +679,7 @@ "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))", "hide": false, "instant": false, - "legendFormat": "__auto", + "legendFormat": "sql-{{db}}-p95", "range": true, "refId": "SQL P95" }, @@ -692,7 +692,7 @@ "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))", "hide": false, "instant": false, - "legendFormat": "__auto", + "legendFormat": "sql-{{db}}-p99", "range": true, "refId": "SQL P99" }, @@ -705,7 +705,7 @@ "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))", "hide": false, "instant": false, - "legendFormat": "__auto", + "legendFormat": "promstore-read-{{db}}-p95", "range": true, "refId": "PromStore Read P95" }, @@ -718,9 +718,35 @@ "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))", "hide": false, "instant": false, - "legendFormat": "__auto", + "legendFormat": "promstore-read-{{db}}-p99", "range": true, "refId": "PromStore Read P99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS-1}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "prom-promql-{{db}}-{{method}}-p95", + "range": true, + "refId": "Prometheus PromQL P95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS-1}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "prom-promql-{{db}}-{{method}}-p99", + "range": true, + "refId": "Prometheus PromQL P99" } ], "title": "HTTP query elapsed", @@ -1090,6 +1116,106 @@ "title": "gRPC insert elapsed", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS-1}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 36, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS-1}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(db) (rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{db}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Logs ingest rate (number of lines)", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { @@ -2626,4 +2752,4 @@ "uid": "e7097237-669b-4f8d-b751-13067afbfb68", "version": 14, "weekStart": "" -} +} \ No newline at end of file diff --git a/src/servers/src/http/prometheus.rs b/src/servers/src/http/prometheus.rs index 33a702d09d75..f3a907aaf95d 100644 --- a/src/servers/src/http/prometheus.rs +++ b/src/servers/src/http/prometheus.rs @@ -193,6 +193,10 @@ pub async fn instant_query( query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema); } + let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED + .with_label_values(&[query_ctx.get_db_string().as_str(), "instant_query"]) + .start_timer(); + let result = handler.do_query(&prom_query, query_ctx).await; let (metric_name, result_type) = match retrieve_metric_name_and_result_type(&prom_query.query) { Ok((metric_name, result_type)) => (metric_name.unwrap_or_default(), result_type), @@ -242,6 +246,10 @@ pub async fn range_query( query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema); } + let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED + .with_label_values(&[query_ctx.get_db_string().as_str(), "range_query"]) + .start_timer(); + let result = handler.do_query(&prom_query, query_ctx).await; let metric_name = match retrieve_metric_name_and_result_type(&prom_query.query) { Err(err) => { @@ -316,6 +324,10 @@ pub async fn labels_query( queries = form_params.matches.0; } + let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED + .with_label_values(&[query_ctx.get_db_string().as_str(), "labels_query"]) + .start_timer(); + // Fetch all tag columns. It will be used as white-list for tag names. let mut labels = match get_all_column_names(&catalog, &schema, &handler.catalog_manager()).await { @@ -687,6 +699,10 @@ pub async fn label_values_query( let (catalog, schema) = get_catalog_schema(¶ms.db, &query_ctx); let query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema); + let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED + .with_label_values(&[query_ctx.get_db_string().as_str(), "label_values_query"]) + .start_timer(); + if label_name == METRIC_NAME_LABEL { let mut table_names = match handler .catalog_manager() @@ -968,6 +984,10 @@ pub async fn series_query( query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema); } + let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED + .with_label_values(&[query_ctx.get_db_string().as_str(), "series_query"]) + .start_timer(); + let mut series = Vec::new(); let mut merge_map = HashMap::new(); for query in queries { diff --git a/src/servers/src/metrics.rs b/src/servers/src/metrics.rs index b5924307253e..cdf927536f04 100644 --- a/src/servers/src/metrics.rs +++ b/src/servers/src/metrics.rs @@ -120,6 +120,13 @@ lazy_static! { &[METRIC_DB_LABEL] ) .unwrap(); + /// Http prometheus endpoint query duration per database. + pub static ref METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED: HistogramVec = register_histogram_vec!( + "greptime_servers_http_prometheus_promql_elapsed", + "servers http prometheus promql elapsed", + &[METRIC_DB_LABEL, METRIC_METHOD_LABEL] + ) + .unwrap(); pub static ref METRIC_HTTP_OPENTELEMETRY_METRICS_ELAPSED: HistogramVec = register_histogram_vec!( "greptime_servers_http_otlp_metrics_elapsed",