Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: track prometheus HTTP API's query latency #4458

Merged
merged 3 commits into from
Jul 29, 2024
Merged

feat: track prometheus HTTP API's query latency #4458

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
24e3b9c
feat: track prometheus HTTP API's query latency
waynexia Jul 29, 2024
9b86e03
update grafana config
waynexia Jul 29, 2024
dddae10
chore: Update src/servers/src/metrics.rs
evenyag Jul 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 131 additions & 5 deletions grafana/greptimedb.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -679,7 +679,7 @@
"expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))",
"hide": false,
"instant": false,
"legendFormat": "__auto",
"legendFormat": "sql-{{db}}-p95",
"range": true,
"refId": "SQL P95"
},
Expand All @@ -692,7 +692,7 @@
"expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))",
"hide": false,
"instant": false,
"legendFormat": "__auto",
"legendFormat": "sql-{{db}}-p99",
"range": true,
"refId": "SQL P99"
},
Expand All @@ -705,7 +705,7 @@
"expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))",
"hide": false,
"instant": false,
"legendFormat": "__auto",
"legendFormat": "promstore-read-{{db}}-p95",
"range": true,
"refId": "PromStore Read P95"
},
Expand All @@ -718,9 +718,35 @@
"expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))",
"hide": false,
"instant": false,
"legendFormat": "__auto",
"legendFormat": "promstore-read-{{db}}-p99",
"range": true,
"refId": "PromStore Read P99"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS-1}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))",
"hide": false,
"instant": false,
"legendFormat": "prom-promql-{{db}}-{{method}}-p95",
"range": true,
"refId": "Prometheus PromQL P95"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS-1}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))",
"hide": false,
"instant": false,
"legendFormat": "prom-promql-{{db}}-{{method}}-p99",
"range": true,
"refId": "Prometheus PromQL P99"
}
],
"title": "HTTP query elapsed",
Expand Down Expand Up @@ -1090,6 +1116,106 @@
"title": "gRPC insert elapsed",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS-1}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 18
},
"id": 36,
"interval": "1s",
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS-1}"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(db) (rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "{{db}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Logs ingest rate (number of lines)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
Expand Down Expand Up @@ -2626,4 +2752,4 @@
"uid": "e7097237-669b-4f8d-b751-13067afbfb68",
"version": 14,
"weekStart": ""
}
}
20 changes: 20 additions & 0 deletions src/servers/src/http/prometheus.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,10 @@ pub async fn instant_query(
query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
}

let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
.with_label_values(&[query_ctx.get_db_string().as_str(), "instant_query"])
.start_timer();

let result = handler.do_query(&prom_query, query_ctx).await;
let (metric_name, result_type) = match retrieve_metric_name_and_result_type(&prom_query.query) {
Ok((metric_name, result_type)) => (metric_name.unwrap_or_default(), result_type),
Expand Down Expand Up @@ -242,6 +246,10 @@ pub async fn range_query(
query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
}

let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
.with_label_values(&[query_ctx.get_db_string().as_str(), "range_query"])
.start_timer();

let result = handler.do_query(&prom_query, query_ctx).await;
let metric_name = match retrieve_metric_name_and_result_type(&prom_query.query) {
Err(err) => {
Expand Down Expand Up @@ -316,6 +324,10 @@ pub async fn labels_query(
queries = form_params.matches.0;
}

let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
.with_label_values(&[query_ctx.get_db_string().as_str(), "labels_query"])
.start_timer();

// Fetch all tag columns. It will be used as white-list for tag names.
let mut labels = match get_all_column_names(&catalog, &schema, &handler.catalog_manager()).await
{
Expand Down Expand Up @@ -687,6 +699,10 @@ pub async fn label_values_query(
let (catalog, schema) = get_catalog_schema(&params.db, &query_ctx);
let query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);

let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
.with_label_values(&[query_ctx.get_db_string().as_str(), "label_values_query"])
.start_timer();

if label_name == METRIC_NAME_LABEL {
let mut table_names = match handler
.catalog_manager()
Expand Down Expand Up @@ -968,6 +984,10 @@ pub async fn series_query(
query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
}

let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
.with_label_values(&[query_ctx.get_db_string().as_str(), "series_query"])
.start_timer();

let mut series = Vec::new();
let mut merge_map = HashMap::new();
for query in queries {
Expand Down
7 changes: 7 additions & 0 deletions src/servers/src/metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,13 @@ lazy_static! {
&[METRIC_DB_LABEL]
)
.unwrap();
/// Http prometheus endpoint query duration per database.
pub static ref METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED: HistogramVec = register_histogram_vec!(
"greptime_servers_http_prometheus_promql_elapsed",
"servers http prometheus promql elapsed",
&[METRIC_DB_LABEL,METRIC_METHOD_LABEL]
evenyag marked this conversation as resolved.
Show resolved Hide resolved
)
.unwrap();
pub static ref METRIC_HTTP_OPENTELEMETRY_METRICS_ELAPSED: HistogramVec =
register_histogram_vec!(
"greptime_servers_http_otlp_metrics_elapsed",
Expand Down