tagger: add two feature-flags that change seriesByTag behavior (#266)

.github/workflows/release.yml

@@ -32,7 +32,7 @@ jobs:
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: '2.7' # Version range or exact version of a Ruby version to use, using semvers version range syntax.
+        ruby-version: '3.3' # Version range or exact version of a Ruby version to use, using semvers version range syntax.
     - name: Install packaging dependencies
       run: |
         gem install fpm package_cloud

.github/workflows/tests.yml

@@ -93,14 +93,14 @@ jobs:
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: '2.7' # Version range or exact version of a Ruby version to use, using semvers version range syntax.
+        ruby-version: '3.3' # Version range or exact version of a Ruby version to use, using semvers version range syntax.
 
+    # gem install dotenv -v 2.8.1 # workaroaund for ruby version 2.7.8.225
     - name: Install packaging dependencies
       run: |
-        gem install dotenv -v 2.8.1 # workaroaund for ruby version 2.7.8.225
         gem install fpm package_cloud
         go install github.com/mitchellh/gox@latest
-
+        
     - name: Check packaging
       run: |
         make DEVEL=1 gox-build fpm-deb fpm-rpm

autocomplete/autocomplete.go

@@ -93,7 +93,7 @@ func (h *Handler) requestExpr(r *http.Request) (*where.Where, *where.Where, map[
 		return wr, pw, usedTags, err
 	}
 
-	wr, pw, err = finder.TaggedWhere(terms)
+	wr, pw, err = finder.TaggedWhere(terms, h.config.FeatureFlags.UseCarbonBehavior, h.config.FeatureFlags.DontMatchMissingTags)
 	if err != nil {
 		return wr, pw, usedTags, err
 	}

config/config.go

@@ -118,6 +118,12 @@ type Common struct {
 	FindCache cache.BytesCache `toml:"-" json:"-"`
 }
 
+// FeatureFlags contains feature flags that significantly change how gch responds to some requests
+type FeatureFlags struct {
+	UseCarbonBehavior    bool `toml:"use-carbon-behaviour" json:"use-carbon-behaviour" comment:"if true, prefers carbon's behaviour on how tags are treated"`
+	DontMatchMissingTags bool `toml:"dont-match-missing-tags" json:"dont-match-missing-tags" comment:"if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all"`
+}
+
 // IndexReverseRule contains rules to use direct or reversed request to index table
 type IndexReverseRule struct {
 	Suffix   string         `toml:"suffix,omitempty" json:"suffix"  comment:"rule is used when the target suffix is matched"`
@@ -341,15 +347,16 @@ type Debug struct {
 
 // Config is the daemon configuration
 type Config struct {
-	Common     Common             `toml:"common"     json:"common"`
-	Metrics    metrics.Config     `toml:"metrics"    json:"metrics"`
-	ClickHouse ClickHouse         `toml:"clickhouse" json:"clickhouse"`
-	DataTable  []DataTable        `toml:"data-table" json:"data-table" comment:"data tables, see doc/config.md for additional info"`
-	Tags       Tags               `toml:"tags"       json:"tags"       comment:"is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU" commented:"true"`
-	Carbonlink Carbonlink         `toml:"carbonlink" json:"carbonlink"`
-	Prometheus Prometheus         `toml:"prometheus" json:"prometheus"`
-	Debug      Debug              `toml:"debug"      json:"debug"      comment:"see doc/debugging.md"`
-	Logging    []zapwriter.Config `toml:"logging"    json:"logging"`
+	Common       Common             `toml:"common"        json:"common"`
+	FeatureFlags FeatureFlags       `toml:"feature-flags" json:"feature-flags"`
+	Metrics      metrics.Config     `toml:"metrics"       json:"metrics"`
+	ClickHouse   ClickHouse         `toml:"clickhouse"    json:"clickhouse"`
+	DataTable    []DataTable        `toml:"data-table"    json:"data-table" comment:"data tables, see doc/config.md for additional info"`
+	Tags         Tags               `toml:"tags"          json:"tags"       comment:"is not recommended to use, https://github.com/lomik/graphite-clickhouse/wiki/TagsRU" commented:"true"`
+	Carbonlink   Carbonlink         `toml:"carbonlink"    json:"carbonlink"`
+	Prometheus   Prometheus         `toml:"prometheus"    json:"prometheus"`
+	Debug        Debug              `toml:"debug"         json:"debug"      comment:"see doc/debugging.md"`
+	Logging      []zapwriter.Config `toml:"logging"       json:"logging"`
 }
 
 // New returns *Config with default values

deploy/doc/config.md

@@ -29,6 +29,36 @@ shortTimeoutSec = 300
 findTimeoutSec = 600
 ```
 
+## Feature flags `[feature-flags]`
+
+`use-carbon-behaviour=true`.
+
+- Tagged terms with `=` operator and empty value (e.g. `t=`) match all metrics that don't have that tag.
+
+`dont-match-missing-tags=true`.
+
+- Tagged terms with `!=`, `!=~` operators only match metrics that have that tag.
+
+### Examples
+
+Given tagged metrics:
+```
+metric.two;env=prod
+metric.one;env=stage;dc=mydc1
+metric.one;env=prod;dc=otherdc1
+```
+| Target                      | use-carbon-behaviour | Matched metrics                                   |
+|-----------------------------|----------------------|---------------------------------------------------|
+| seriesByTag('dc=')          | false                | -                                                 |
+| seriesByTag('dc=')          | true                 | metric.two;env=prod                               |
+
+| Target                   | dont-match-missing-tags | Matched metrics                                        |
+|--------------------------|-------------------------|--------------------------------------------------------|
+| seriesByTag('dc!=mydc1') | false                   | metric.two;env=prod<br>metric.one;env=prod;dc=otherdc1 |
+| seriesByTag('dc!=mydc1') | true                    | metric.one;env=prod;dc=otherdc1                        |
+| seriesByTag('dc!=~otherdc') | false                | metric.two;env=prod<br>metric.one;env=stage;dc=mydc1 |
+| seriesByTag('dc!=~otherdc') | true                 | metric.one;env=stage;dc=mydc1                     |
+
 ## ClickHouse `[clickhouse]`
 
 ### URL `url`

doc/config.md

@@ -32,6 +32,36 @@ shortTimeoutSec = 300
 findTimeoutSec = 600
 ```
 
+## Feature flags `[feature-flags]`
+
+`use-carbon-behaviour=true`.
+
+- Tagged terms with `=` operator and empty value (e.g. `t=`) match all metrics that don't have that tag.
+
+`dont-match-missing-tags=true`.
+
+- Tagged terms with `!=`, `!=~` operators only match metrics that have that tag.
+
+### Examples
+
+Given tagged metrics:
+```
+metric.two;env=prod
+metric.one;env=stage;dc=mydc1
+metric.one;env=prod;dc=otherdc1
+```
+| Target                      | use-carbon-behaviour | Matched metrics                                   |
+|-----------------------------|----------------------|---------------------------------------------------|
+| seriesByTag('dc=')          | false                | -                                                 |
+| seriesByTag('dc=')          | true                 | metric.two;env=prod                               |
+
+| Target                   | dont-match-missing-tags | Matched metrics                                        |
+|--------------------------|-------------------------|--------------------------------------------------------|
+| seriesByTag('dc!=mydc1') | false                   | metric.two;env=prod<br>metric.one;env=prod;dc=otherdc1 |
+| seriesByTag('dc!=mydc1') | true                    | metric.one;env=prod;dc=otherdc1                        |
+| seriesByTag('dc!=~otherdc') | false                | metric.two;env=prod<br>metric.one;env=stage;dc=mydc1 |
+| seriesByTag('dc!=~otherdc') | true                 | metric.one;env=stage;dc=mydc1                     |
+
 ## ClickHouse `[clickhouse]`
 
 ### URL `url`
@@ -228,6 +258,12 @@ Only one tag used as filter for index field Tag1, see graphite_tagged table [str
   # offset beetween now and until for select short cache timeout
   short-offset = 0
 
+[feature-flags]
+ # if true, prefers carbon's behaviour on how tags are treated
+ use-carbon-behaviour = false
+ # if true, seriesByTag terms containing '!=' or '!=~' operators will not match metrics that don't have the tag at all
+ dont-match-missing-tags = false
+
 [metrics]
  # graphite relay address
  metric-endpoint = ""

finder/finder.go

@@ -38,7 +38,16 @@ func newPlainFinder(ctx context.Context, config *config.Config, query string, fr
 	var f Finder
 
 	if config.ClickHouse.TaggedTable != "" && strings.HasPrefix(strings.TrimSpace(query), "seriesByTag") {
-		f = NewTagged(config.ClickHouse.URL, config.ClickHouse.TaggedTable, config.ClickHouse.TaggedUseDaily, false, opts, config.ClickHouse.TaggedCosts)
+		f = NewTagged(
+			config.ClickHouse.URL,
+			config.ClickHouse.TaggedTable,
+			config.ClickHouse.TaggedUseDaily,
+			config.FeatureFlags.UseCarbonBehavior,
+			config.FeatureFlags.DontMatchMissingTags,
+			false,
+			opts,
+			config.ClickHouse.TaggedCosts,
+		)
 
 		if len(config.Common.Blacklist) > 0 {
 			f = WrapBlacklist(f, config.Common.Blacklist)
@@ -121,7 +130,16 @@ func FindTagged(ctx context.Context, config *config.Config, terms []TaggedTerm,
 		return Result(plain), nil
 	}
 
-	fnd := NewTagged(config.ClickHouse.URL, config.ClickHouse.TaggedTable, config.ClickHouse.TaggedUseDaily, true, opts, config.ClickHouse.TaggedCosts)
+	fnd := NewTagged(
+		config.ClickHouse.URL,
+		config.ClickHouse.TaggedTable,
+		config.ClickHouse.TaggedUseDaily,
+		config.FeatureFlags.UseCarbonBehavior,
+		config.FeatureFlags.DontMatchMissingTags,
+		true,
+		opts,
+		config.ClickHouse.TaggedCosts,
+	)
 
 	err := fnd.ExecutePrepared(ctx, terms, from, until, stat)
 	if err != nil {

finder/tagged.go

@@ -72,24 +72,28 @@ func (s TaggedTermList) Less(i, j int) bool {
 }
 
 type TaggedFinder struct {
-	url            string                   // clickhouse dsn
-	table          string                   // graphite_tag table
-	absKeepEncoded bool                     // Abs returns url encoded value. For queries from prometheus
-	opts           clickhouse.Options       // clickhouse query timeout
-	taggedCosts    map[string]*config.Costs // costs for taggs (sor tune index search)
-	dailyEnabled   bool
+	url                  string                   // clickhouse dsn
+	table                string                   // graphite_tag table
+	absKeepEncoded       bool                     // Abs returns url encoded value. For queries from prometheus
+	opts                 clickhouse.Options       // clickhouse query timeout
+	taggedCosts          map[string]*config.Costs // costs for taggs (sor tune index search)
+	dailyEnabled         bool
+	useCarbonBehavior    bool
+	dontMatchMissingTags bool
 
 	body []byte // clickhouse response
 }
 
-func NewTagged(url string, table string, dailyEnabled bool, absKeepEncoded bool, opts clickhouse.Options, taggedCosts map[string]*config.Costs) *TaggedFinder {
+func NewTagged(url string, table string, dailyEnabled, useCarbonBehavior, dontMatchMissingTags, absKeepEncoded bool, opts clickhouse.Options, taggedCosts map[string]*config.Costs) *TaggedFinder {
 	return &TaggedFinder{
-		url:            url,
-		table:          table,
-		absKeepEncoded: absKeepEncoded,
-		opts:           opts,
-		taggedCosts:    taggedCosts,
-		dailyEnabled:   dailyEnabled,
+		url:                  url,
+		table:                table,
+		absKeepEncoded:       absKeepEncoded,
+		opts:                 opts,
+		taggedCosts:          taggedCosts,
+		dailyEnabled:         dailyEnabled,
+		useCarbonBehavior:    useCarbonBehavior,
+		dontMatchMissingTags: dontMatchMissingTags,
 	}
 }
 
@@ -102,12 +106,17 @@ func (term *TaggedTerm) concatMask() string {
 	return fmt.Sprintf("%s=%s", term.Key, v)
 }
 
-func TaggedTermWhere1(term *TaggedTerm) (string, error) {
+func TaggedTermWhere1(term *TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (string, error) {
 	// positive expression check only in Tag1
 	// negative check in all Tags
 	switch term.Op {
 	case TaggedTermEq:
-		if strings.Index(term.Value, "*") >= 0 {
+		if useCarbonBehaviour && term.Value == "" {
+			// special case
+			// container_name=""  ==> response should not contain container_name
+			return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.HasPrefix("x", term.Key+"=")), nil
+		}
+		if strings.Contains(term.Value, "*") {
 			return where.Like("Tag1", term.concatMask()), nil
 		}
 		var values []string
@@ -127,35 +136,52 @@ func TaggedTermWhere1(term *TaggedTerm) (string, error) {
 			// container_name!=""  ==> container_name exists and it is not empty
 			return where.HasPrefixAndNotEq("Tag1", term.Key+"="), nil
 		}
-		if strings.Index(term.Value, "*") >= 0 {
-			return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.Like("x", term.concatMask())), nil
+		var whereLikeAnyVal string
+		if dontMatchMissingTags {
+			whereLikeAnyVal = where.HasPrefix("Tag1", term.Key+"=") + " AND "
+		}
+		if strings.Contains(term.Value, "*") {
+			whereLike := where.Like("x", term.concatMask())
+			return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereLike), nil
 		}
 		var values []string
 		if err := where.GlobExpandSimple(term.Value, term.Key+"=", &values); err != nil {
 			return "", err
 		}
 		if len(values) == 1 {
-			return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.Eq("x", values[0])), nil
+			whereEq := where.Eq("x", values[0])
+			return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereEq), nil
 		} else if len(values) > 1 {
-			return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.In("x", values)), nil
+			whereIn := where.In("x", values)
+			return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereIn), nil
 		} else {
-			return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.Eq("x", term.concat())), nil
+			whereEq := where.Eq("x", term.concat())
+			return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereEq), nil
 		}
 	case TaggedTermMatch:
 		return where.Match("Tag1", term.Key, term.Value), nil
 	case TaggedTermNotMatch:
-		// return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", term.Key, term.Value), nil
-		return "NOT " + where.Match("Tag1", term.Key, term.Value), nil
+		var whereLikeAnyVal string
+		if dontMatchMissingTags {
+			whereLikeAnyVal = where.HasPrefix("Tag1", term.Key+"=") + " AND "
+		}
+		whereMatch := where.Match("x", term.Key, term.Value)
+		return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereMatch), nil
 	default:
 		return "", nil
 	}
 }
 
-func TaggedTermWhereN(term *TaggedTerm) (string, error) {
+func TaggedTermWhereN(term *TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (string, error) {
 	// arrayExists((x) -> %s, Tags)
 	switch term.Op {
 	case TaggedTermEq:
-		if strings.Index(term.Value, "*") >= 0 {
+		if useCarbonBehaviour && term.Value == "" {
+			// special case
+			// container_name=""  ==> response should not contain container_name
+			return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.HasPrefix("x", term.Key+"=")), nil
+		}
+		if strings.Contains(term.Value, "*") {
 			return fmt.Sprintf("arrayExists((x) -> %s, Tags)", where.Like("x", term.concatMask())), nil
 		}
 		var values []string
@@ -175,24 +201,37 @@ func TaggedTermWhereN(term *TaggedTerm) (string, error) {
 			// container_name!=""  ==> container_name exists and it is not empty
 			return fmt.Sprintf("arrayExists((x) -> %s, Tags)", where.HasPrefixAndNotEq("x", term.Key+"=")), nil
 		}
-		if strings.Index(term.Value, "*") >= 0 {
-			return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.Like("x", term.concatMask())), nil
+		var whereLikeAnyVal string
+		if dontMatchMissingTags {
+			whereLikeAnyVal = fmt.Sprintf("arrayExists((x) -> %s, Tags) AND ", where.HasPrefix("x", term.Key+"="))
+		}
+		if strings.Contains(term.Value, "*") {
+			whereLike := where.Like("x", term.concatMask())
+			return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereLike), nil
 		}
 		var values []string
 		if err := where.GlobExpandSimple(term.Value, term.Key+"=", &values); err != nil {
 			return "", err
 		}
 		if len(values) == 1 {
-			return "NOT arrayExists((x) -> " + where.Eq("x", values[0]) + ", Tags)", nil
+			whereEq := where.Eq("x", values[0])
+			return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereEq), nil
 		} else if len(values) > 1 {
-			return "NOT arrayExists((x) -> " + where.In("x", values) + ", Tags)", nil
+			whereIn := where.In("x", values)
+			return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereIn), nil
 		} else {
-			return "NOT arrayExists((x) -> " + where.Eq("x", term.concat()) + ", Tags)", nil
+			whereEq := where.Eq("x", term.concat())
+			return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereEq), nil
 		}
 	case TaggedTermMatch:
 		return fmt.Sprintf("arrayExists((x) -> %s, Tags)", where.Match("x", term.Key, term.Value)), nil
 	case TaggedTermNotMatch:
-		return fmt.Sprintf("NOT arrayExists((x) -> %s, Tags)", where.Match("x", term.Key, term.Value)), nil
+		var whereLikeAnyVal string
+		if dontMatchMissingTags {
+			whereLikeAnyVal = fmt.Sprintf("arrayExists((x) -> %s, Tags) AND ", where.HasPrefix("x", term.Key+"="))
+		}
+		whereMatch := where.Match("x", term.Key, term.Value)
+		return fmt.Sprintf("%sNOT arrayExists((x) -> %s, Tags)", whereLikeAnyVal, whereMatch), nil
 	default:
 		return "", nil
 	}
@@ -387,10 +426,10 @@ func ParseSeriesByTag(query string, config *config.Config) ([]TaggedTerm, error)
 	return ParseTaggedConditions(conditions, config, false)
 }
 
-func TaggedWhere(terms []TaggedTerm) (*where.Where, *where.Where, error) {
+func TaggedWhere(terms []TaggedTerm, useCarbonBehaviour, dontMatchMissingTags bool) (*where.Where, *where.Where, error) {
 	w := where.New()
 	pw := where.New()
-	x, err := TaggedTermWhere1(&terms[0])
+	x, err := TaggedTermWhere1(&terms[0], useCarbonBehaviour, dontMatchMissingTags)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -400,7 +439,7 @@ func TaggedWhere(terms []TaggedTerm) (*where.Where, *where.Where, error) {
 	w.And(x)
 
 	for i := 1; i < len(terms); i++ {
-		and, err := TaggedTermWhereN(&terms[i])
+		and, err := TaggedTermWhereN(&terms[i], useCarbonBehaviour, dontMatchMissingTags)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -426,7 +465,7 @@ func (t *TaggedFinder) Execute(ctx context.Context, config *config.Config, query
 }
 
 func (t *TaggedFinder) whereFilter(terms []TaggedTerm, from int64, until int64) (*where.Where, *where.Where, error) {
-	w, pw, err := TaggedWhere(terms)
+	w, pw, err := TaggedWhere(terms, t.useCarbonBehavior, t.dontMatchMissingTags)
 	if err != nil {
 		return nil, nil, err
 	}