metrics: clickhouse query metrics

go-graphite · Nov 5, 2022 · 74a9d4e · 74a9d4e
1 parent 23a425b
commit 74a9d4e
Show file tree

Hide file tree

Showing 35 changed files with 608 additions and 149 deletions.
diff --git a/autocomplete/autocomplete.go b/autocomplete/autocomplete.go
@@ -126,10 +126,16 @@ func taggedKey(typ string, truncateSec int32, fromDate, untilDate string, tag st
 func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
 	status := http.StatusOK
-	var metricsCount int64
 	logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog)
 
-	var err error
+	var (
+		err          error
+		chReadRows   int64
+		chReadBytes  int64
+		metricsCount int64
+		readBytes    int64
+		findCache    bool
+	)
 
 	defer func() {
 		if rec := recover(); rec != nil {
@@ -144,13 +150,19 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 		}
 		d := time.Since(start).Milliseconds()
 		metrics.SendFindMetrics(metrics.FindRequestMetric, status, d, 0, h.config.Metrics.ExtendedStat, metricsCount)
+		if !findCache && chReadRows != 0 && chReadBytes != 0 {
+			errored := status != http.StatusOK && status != http.StatusNotFound
+			metrics.SendQueryRead(metrics.AutocompleteQMetric, 0, 0, d, metricsCount, readBytes, chReadRows, chReadBytes, errored)
+		}
 	}()
 
 	r.ParseMultipartForm(1024 * 1024)
 	tagPrefix := r.FormValue("tagPrefix")
 	limitStr := r.FormValue("limit")
 	limit := 10000
 
+	var body []byte
+
 	if limitStr != "" {
 		limit, err = strconv.Atoi(limitStr)
 		if err != nil {
@@ -165,8 +177,6 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 	untilDate := now.Format("2006-01-02")
 
 	var key string
-	var body []byte
-	var findCache bool
 
 	useCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue("noCache"))
 	if useCache {
@@ -215,7 +225,7 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 			queryLimit,
 		)
 
-		body, err := clickhouse.Query(
+		body, chReadRows, chReadBytes, err = clickhouse.Query(
 			scope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable),
 			h.config.ClickHouse.URL,
 			sql,
@@ -229,6 +239,7 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 			status = clickhouse.HandleError(w, err)
 			return
 		}
+		readBytes = int64(len(body))
 
 		if useCache {
 			if metrics.FinderCacheMetrics != nil {
@@ -298,10 +309,16 @@ func (h *Handler) ServeTags(w http.ResponseWriter, r *http.Request) {
 func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
 	status := http.StatusOK
-	var metricsCount int64
 	logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog)
 
-	var err error
+	var (
+		err          error
+		body         []byte
+		chReadRows   int64
+		chReadBytes  int64
+		metricsCount int64
+		findCache    bool
+	)
 
 	defer func() {
 		if rec := recover(); rec != nil {
@@ -316,6 +333,10 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
 		}
 		d := time.Since(start).Milliseconds()
 		metrics.SendFindMetrics(metrics.FindRequestMetric, status, d, 0, h.config.Metrics.ExtendedStat, metricsCount)
+		if !findCache && chReadRows > 0 && chReadBytes > 0 {
+			errored := status != http.StatusOK && status != http.StatusNotFound
+			metrics.SendQueryRead(metrics.AutocompleteQMetric, 0, 0, d, metricsCount, int64(len(body)), chReadRows, chReadBytes, errored)
+		}
 	}()
 
 	r.ParseMultipartForm(1024 * 1024)
@@ -341,8 +362,6 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
 	untilDate := start.Format("2006-01-02")
 
 	var key string
-	var body []byte
-	var findCache bool
 
 	useCache := h.config.Common.FindCache != nil && h.config.Common.FindCacheConfig.FindTimeoutSec > 0 && !parser.TruthyBool(r.FormValue("noCache"))
 	if useCache {
@@ -385,7 +404,7 @@ func (h *Handler) ServeValues(w http.ResponseWriter, r *http.Request) {
 			limit,
 		)
 
-		body, err = clickhouse.Query(
+		body, chReadRows, chReadBytes, err = clickhouse.Query(
 			scope.WithTable(r.Context(), h.config.ClickHouse.TaggedTable),
 			h.config.ClickHouse.URL,
 			sql,

diff --git a/autocomplete/autocomplete_test.go b/autocomplete/autocomplete_test.go
@@ -153,7 +153,7 @@ func TestTagsAutocomplete_ServeValuesCached(t *testing.T) {
 			assert.Equal(t, uint64(1), srv.Queries()-queries)
 
 			// wait for expire cache
-			time.Sleep(time.Second * 2)
+			time.Sleep(time.Second * 3)
 			testResponce(t, 2, h, &tt, "")
 
 			assert.Equal(t, uint64(2), srv.Queries()-queries)

diff --git a/config/config.go b/config/config.go
@@ -195,25 +195,26 @@ var knownDataTableContext = map[string]bool{
 
 // DataTable configs
 type DataTable struct {
-	Table                  string          `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
-	Reverse                bool            `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
-	MaxAge                 time.Duration   `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
-	MinAge                 time.Duration   `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
-	MaxInterval            time.Duration   `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
-	MinInterval            time.Duration   `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
-	TargetMatchAny         string          `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
-	TargetMatchAll         string          `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
-	TargetMatchAnyRegexp   *regexp.Regexp  `toml:"-" json:"-"`
-	TargetMatchAllRegexp   *regexp.Regexp  `toml:"-" json:"-"`
-	RollupConf             string          `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
-	RollupAutoTable        string          `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
-	RollupAutoInterval     *time.Duration  `toml:"rollup-auto-interval" json:"rollup-auto-interval" comment:"rollup update interval for 'rollup-conf=auto'"`
-	RollupDefaultPrecision uint32          `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
-	RollupDefaultFunction  string          `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
-	RollupUseReverted      bool            `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
-	Context                []string        `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
-	ContextMap             map[string]bool `toml:"-" json:"-"`
-	Rollup                 *rollup.Rollup  `toml:"-" json:"rollup-conf"`
+	Table                  string                `toml:"table" json:"table" comment:"data table from carbon-clickhouse"`
+	Reverse                bool                  `toml:"reverse" json:"reverse" comment:"if it stores direct or reversed metrics"`
+	MaxAge                 time.Duration         `toml:"max-age" json:"max-age" comment:"maximum age stored in the table"`
+	MinAge                 time.Duration         `toml:"min-age" json:"min-age" comment:"minimum age stored in the table"`
+	MaxInterval            time.Duration         `toml:"max-interval" json:"max-interval" comment:"maximum until-from interval allowed for the table"`
+	MinInterval            time.Duration         `toml:"min-interval" json:"min-interval" comment:"minimum until-from interval allowed for the table"`
+	TargetMatchAny         string                `toml:"target-match-any" json:"target-match-any" comment:"table allowed only if any metrics in target matches regexp"`
+	TargetMatchAll         string                `toml:"target-match-all" json:"target-match-all" comment:"table allowed only if all metrics in target matches regexp"`
+	TargetMatchAnyRegexp   *regexp.Regexp        `toml:"-" json:"-"`
+	TargetMatchAllRegexp   *regexp.Regexp        `toml:"-" json:"-"`
+	RollupConf             string                `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
+	RollupAutoTable        string                `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
+	RollupAutoInterval     *time.Duration        `toml:"rollup-auto-interval" json:"rollup-auto-interval" comment:"rollup update interval for 'rollup-conf=auto'"`
+	RollupDefaultPrecision uint32                `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
+	RollupDefaultFunction  string                `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
+	RollupUseReverted      bool                  `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
+	Context                []string              `toml:"context" json:"context" comment:"valid values are 'graphite' of 'prometheus'"`
+	ContextMap             map[string]bool       `toml:"-" json:"-"`
+	Rollup                 *rollup.Rollup        `toml:"-" json:"rollup-conf"`
+	QueryMetrics           *metrics.QueryMetrics `toml:"-" json:"-"`
 }
 
 // Debug config
@@ -674,4 +675,16 @@ func (c *Config) setupGraphiteMetrics() {
 
 		metrics.InitMetrics(&c.Metrics)
 	}
+
+	metrics.AutocompleteQMetric = metrics.InitQueryMetrics("tags", &c.Metrics)
+	metrics.FindQMetric = metrics.InitQueryMetrics("find", &c.Metrics)
+	for i := 0; i < len(c.DataTable); i++ {
+		c.DataTable[i].QueryMetrics = metrics.InitQueryMetrics(c.DataTable[i].Table, &c.Metrics)
+	}
+	if c.ClickHouse.IndexTable != "" {
+		metrics.InitQueryMetrics(c.ClickHouse.IndexTable, &c.Metrics)
+	}
+	if c.ClickHouse.TaggedTable != "" {
+		metrics.InitQueryMetrics(c.ClickHouse.TaggedTable, &c.Metrics)
+	}
 }
diff --git a/deploy/doc/config.md b/deploy/doc/config.md
@@ -2,13 +2,6 @@
 
 ## Common  `[common]`
 
-Send internal metrics to graphite relay
- - `metric-endpoint`   - graphite relay address
- - `metric-interval`   - graphite metrics send interval
- - `metric-prefix`     - graphite metrics prefix 
- - `metric-timeout`    - graphite metrics send timeout
- - `metric-batch-size` - graphite send batch size
-
 ### Finder cache
 
 Specify what storage to use for finder cache. This cache stores finder results (metrics find/tags autocomplete/render).
@@ -153,6 +146,18 @@ The configuration to get metrics from carbon-cache. See details in [graphite-web
 ## Logging `[logging]`
 It's possible to set multiple loggers. See `Config` description in [config.go](https://github.com/lomik/zapwriter/blob/master/config.go) for details.
 
+## Metrics `[metrics]`
+
+Send internal metrics to graphite relay
+ - `metric-endpoint`   - graphite relay address
+ - `statsd-endpoint`   - StatsD aggregator address
+ - `metric-interval`   - graphite metrics send interval
+ - `metric-prefix`     - graphite metrics prefix 
+ - `metric-timeout`    - graphite metrics send timeout
+ - `ranges`            - separate stat for render query (and internal clickhouse queries) until-from ranges, for example { "1d" = "24h", "7d" = "168h", "90d" = "2160h" }
+ - `request-buckets`   - request historgram buckets widths, by default [200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000]
+ - `request-labels`    - optional request historgram buckets labels
+
 [//]: # (!!!DO NOT EDIT FOLLOWING LINES!!!)
 
 # Example

diff --git a/doc/config.md b/doc/config.md
@@ -5,13 +5,6 @@
 
 ## Common  `[common]`
 
-Send internal metrics to graphite relay
- - `metric-endpoint`   - graphite relay address
- - `metric-interval`   - graphite metrics send interval
- - `metric-prefix`     - graphite metrics prefix 
- - `metric-timeout`    - graphite metrics send timeout
- - `metric-batch-size` - graphite send batch size
-
 ### Finder cache
 
 Specify what storage to use for finder cache. This cache stores finder results (metrics find/tags autocomplete/render).
@@ -156,23 +149,25 @@ The configuration to get metrics from carbon-cache. See details in [graphite-web
 ## Logging `[logging]`
 It's possible to set multiple loggers. See `Config` description in [config.go](https://github.com/lomik/zapwriter/blob/master/config.go) for details.
 
+## Metrics `[metrics]`
+
+Send internal metrics to graphite relay
+ - `metric-endpoint`   - graphite relay address
+ - `statsd-endpoint`   - StatsD aggregator address
+ - `metric-interval`   - graphite metrics send interval
+ - `metric-prefix`     - graphite metrics prefix 
+ - `metric-timeout`    - graphite metrics send timeout
+ - `ranges`            - separate stat for render query (and internal clickhouse queries) until-from ranges, for example { "1d" = "24h", "7d" = "168h", "90d" = "2160h" }
+ - `request-buckets`   - request historgram buckets widths, by default [200, 500, 1000, 2000, 3000, 5000, 7000, 10000, 15000, 20000, 25000, 30000, 40000, 50000, 60000]
+ - `request-labels`    - optional request historgram buckets labels
+
 [//]: # (!!!DO NOT EDIT FOLLOWING LINES!!!)
 
 # Example
 
 
 ```toml
 [common]
- # graphite relay address
- metric-endpoint = ""
- # graphite metrics send interval
- metric-interval = "0s"
- # graphite metrics send timeout
- metric-timeout = "0s"
- # graphite metrics prefix
- metric-prefix = ""
- # graphite send batch size
- metric-batch-size = 0
  # general listener
  listen = ":9090"
  # listener to serve /debug/pprof requests. '-pprof' argument overrides it
@@ -206,6 +201,30 @@ It's possible to set multiple loggers. See `Config` description in [config.go](h
   # maximum diration, used with short_timeout
   short-duration = "0s"
 
+[metrics]
+ # graphite relay address
+ metric-endpoint = ""
+ # statsd server address
+ statsd-endpoint = ""
+ # graphite metrics send interval
+ metric-interval = "0s"
+ # graphite metrics send timeout
+ metric-timeout = "0s"
+ # graphite metrics prefix
+ metric-prefix = ""
+ # Request historgram buckets widths
+ request-buckets = []
+ # Request historgram buckets labels
+ request-labels = []
+
+ # Additional separate stats for until-from ranges
+ [metrics.ranges]
+
+ # Additional separate stats for until-from find ranges
+ [metrics.find-ranges]
+ # Extended metrics
+ extended-stat = false
+
 [clickhouse]
  # default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params
  url = "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1"

diff --git a/find/find.go b/find/find.go
@@ -28,8 +28,8 @@ func NewCached(config *config.Config, body []byte) *Find {
 	}
 }
 
-func New(config *config.Config, ctx context.Context, query string) (*Find, error) {
-	res, err := finder.Find(config, ctx, query, 0, 0)
+func New(config *config.Config, ctx context.Context, query string, stat *finder.FinderStat) (*Find, error) {
+	res, err := finder.Find(config, ctx, query, 0, 0, stat)
 	if err != nil {
 		return nil, err
 	}

diff --git a/find/handler.go b/find/handler.go
@@ -10,6 +10,7 @@ import (
 	"github.com/go-graphite/carbonapi/pkg/parser"
 	v3pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
 	"github.com/lomik/graphite-clickhouse/config"
+	"github.com/lomik/graphite-clickhouse/finder"
 	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
 	"github.com/lomik/graphite-clickhouse/helper/utils"
 	"github.com/lomik/graphite-clickhouse/metrics"
@@ -18,22 +19,28 @@ import (
 )
 
 type Handler struct {
-	config *config.Config
+	config  *config.Config
+	qMetric *metrics.QueryMetrics
 }
 
 func NewHandler(config *config.Config) *Handler {
 	return &Handler{
-		config: config,
+		config:  config,
+		qMetric: metrics.InitQueryMetrics("find", &config.Metrics),
 	}
 }
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
 	status := http.StatusOK
-	var metricsCount int64
 	logger := scope.LoggerWithHeaders(r.Context(), r, h.config.Common.HeadersToLog).Named("metrics-find")
 	r = r.WithContext(scope.WithLogger(r.Context(), logger))
 
+	var (
+		metricsCount int64
+		stat         finder.FinderStat
+	)
+
 	defer func() {
 		if rec := recover(); rec != nil {
 			status = http.StatusInternalServerError
@@ -47,6 +54,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 		d := time.Since(start).Milliseconds()
 		metrics.SendFindMetrics(metrics.FindRequestMetric, status, d, 0, h.config.Metrics.ExtendedStat, metricsCount)
+		if stat.ChReadRows > 0 && stat.ChReadBytes > 0 {
+			errored := status != http.StatusOK && status != http.StatusNotFound
+			metrics.SendQueryRead(metrics.FindQMetric, 0, 0, d, metricsCount, stat.ReadBytes, stat.ChReadRows, stat.ChReadBytes, errored)
+		}
 	}()
 
 	r.ParseMultipartForm(1024 * 1024)
@@ -120,7 +131,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	f, err := New(h.config, r.Context(), query)
+	f, err := New(h.config, r.Context(), query, &stat)
 	if err != nil {
 		status = clickhouse.HandleError(w, err)
 		return

diff --git a/finder/base.go b/finder/base.go
@@ -38,18 +38,18 @@ func (b *BaseFinder) where(query string) *where.Where {
 	return w
 }
 
-func (b *BaseFinder) Execute(ctx context.Context, query string, from int64, until int64) (err error) {
+func (b *BaseFinder) Execute(ctx context.Context, query string, from int64, until int64, stat *FinderStat) (err error) {
 	w := b.where(query)
-
-	b.body, err = clickhouse.Query(
+	b.body, stat.ChReadRows, stat.ChReadBytes, err = clickhouse.Query(
 		scope.WithTable(ctx, b.table),
 		b.url,
 		// TODO: consider consistent query generator
 		fmt.Sprintf("SELECT Path FROM %s WHERE %s GROUP BY Path FORMAT TabSeparatedRaw", b.table, w),
 		b.opts,
 		nil,
 	)
-
+	stat.Table = b.table
+	stat.ReadBytes = int64(len(b.body))
 	return
 }