Merge pull request #146 from appuio/fix/source-key-generation

Generalize source key implementation to arbitrary-length keys

pkg/invoice/invoice_golden_test.go

@@ -64,8 +64,8 @@ func (q fakeQuerier) Query(ctx context.Context, query string, ts time.Time, _ ..
 		res = append(res, &model.Sample{
 			Metric: map[model.LabelName]model.LabelValue{
 				"product":  model.LabelValue(k),
-				"category": model.LabelValue(fmt.Sprintf("%s:%s", sk.Zone, sk.Namespace)),
-				"tenant":   model.LabelValue(sk.Tenant),
+				"category": model.LabelValue(fmt.Sprintf("%s:%s", sk.Part(1), sk.Part(3))),
+				"tenant":   model.LabelValue(sk.Part(2)),
 			},
 			Value: s.Value,
 		})

pkg/report/report.go

@@ -118,10 +118,10 @@ func processSample(ctx context.Context, tx *sqlx.Tx, ts time.Time, query db.Quer
 
 	var upsertedTenant db.Tenant
 	err = upsertTenant(ctx, tx, &upsertedTenant, db.Tenant{
-		Source: skey.Tenant,
+		Source: skey.Tenant(),
 	}, ts)
 	if err != nil {
-		return fmt.Errorf("failed to upsert tenant '%s': %w", skey.Tenant, err)
+		return fmt.Errorf("failed to upsert tenant '%s': %w", skey.Tenant(), err)
 	}
 
 	var upsertedCategory db.Category

pkg/sourcekey/sourcekey.go

@@ -2,6 +2,9 @@ package sourcekey
 
 import (
 	"fmt"
+	"math"
+	"math/bits"
+	"sort"
 	"strings"
 )
 
@@ -10,119 +13,97 @@ const elementSeparator = ":"
 // SourceKey represents a source key to look up dimensions objects (currently queries and products).
 // It implements the lookup logic found in https://kb.vshn.ch/appuio-cloud/references/architecture/metering-data-flow.html#_system_idea.
 type SourceKey struct {
-	Query     string
-	Zone      string
-	Tenant    string
-	Namespace string
-
-	Class string
+	parts []string
 }
 
 // Parse parses a source key in the format of "query:zone:tenant:namespace:class" or "query:zone:tenant:namespace".
 func Parse(raw string) (SourceKey, error) {
 	parts := strings.Split(raw, elementSeparator)
-	if len(parts) == 4 {
-		return SourceKey{parts[0], parts[1], parts[2], parts[3], ""}, nil
-	} else if len(parts) == 5 {
-		return SourceKey{parts[0], parts[1], parts[2], parts[3], parts[4]}, nil
+	if parts[len(parts)-1] == "" {
+		parts = parts[0 : len(parts)-1]
+	}
+	if len(parts) >= 4 {
+		return SourceKey{parts}, nil
 	}
 
-	return SourceKey{}, fmt.Errorf("expected key with 4 to 5 elements separated by `%s` got %d", elementSeparator, len(parts))
+	return SourceKey{}, fmt.Errorf("expected key with at least 4 elements separated by `%s` got %d", elementSeparator, len(parts))
+}
+
+// Tenant returns the third element of the source key which was historically used as the tenant.
+//
+// Deprecated: We would like to get rid of this and read the tenant from a metric label.
+func (k SourceKey) Tenant() string {
+	return k.parts[2]
+}
+
+// Part returns the i-th part of the source key, or an empty string if no such part exists
+func (k SourceKey) Part(i int) string {
+	if i < len(k.parts) {
+		return k.parts[i]
+	}
+	return ""
 }
 
 // String returns the string representation "query:zone:tenant:namespace:class" of the key.
 func (k SourceKey) String() string {
-	elements := []string{k.Query, k.Zone, k.Tenant, k.Namespace}
-	if k.Class != "" {
-		elements = append(elements, k.Class)
-	}
-	return strings.Join(elements, elementSeparator)
+	return strings.Join(k.parts, elementSeparator)
 }
 
 // LookupKeys generates lookup keys for a dimension object in the database.
 // The logic is described here: https://kb.vshn.ch/appuio-cloud/references/architecture/metering-data-flow.html#_system_idea
 func (k SourceKey) LookupKeys() []string {
-	return generateSourceKeys(k.Query, k.Zone, k.Tenant, k.Namespace, k.Class)
-}
 
-func generateSourceKeys(query, zone, tenant, namespace, class string) []string {
 	keys := make([]string, 0)
-	base := []string{query, zone, tenant, namespace}
-	wildcardPositions := []int{1, 2}
-
-	if class != "" {
-		wildcardPositions = append(wildcardPositions, 3)
-		base = append(base, class)
-	}
-
-	for i := len(base); i > 0; i-- {
-		keys = append(keys, strings.Join(base[:i], elementSeparator))
-
-		for j := 1; j < len(wildcardPositions)+1; j++ {
-			perms := combinations(wildcardPositions, j)
-			for _, wcpos := range reverse(perms) {
-				elements := append([]string{}, base[:i]...)
-				for _, p := range wcpos {
-					elements[p] = "*"
+	currentKeyBase := k.parts
+
+	for len(currentKeyBase) > 1 {
+		// For the base key of a given length l, the inner l-2 elements are to be replaced with wildcards in all possible combinations.
+		// To that end, generate 2^(l-2) binary numbers, sort them by specificity, and then for each number generate a key where
+		// for each 1-digit, the element is replaced with a wildcard (and for a 0-digit, the element is kept as-is).
+		innerLength := len(currentKeyBase) - 2
+		nums := makeRange(0, int(math.Pow(2, float64(innerLength))))
+		sort.Sort(sortBySpecificity(nums))
+		for i := range nums {
+			currentKeyElements := make([]string, 0)
+			currentKeyElements = append(currentKeyElements, currentKeyBase[0])
+			for digit := 0; digit < innerLength; digit++ {
+				if nums[i]&uint(math.Pow(2, float64(innerLength-1-digit))) > 0 {
+					currentKeyElements = append(currentKeyElements, "*")
+				} else {
+					currentKeyElements = append(currentKeyElements, currentKeyBase[1+digit])
 				}
-				keys = append(keys, strings.Join(elements, elementSeparator))
 			}
+			currentKeyElements = append(currentKeyElements, currentKeyBase[len(currentKeyBase)-1])
+			keys = append(keys, strings.Join(currentKeyElements, elementSeparator))
 		}
-		if i > 2 {
-			wildcardPositions = wildcardPositions[:len(wildcardPositions)-1]
-		}
+		currentKeyBase = currentKeyBase[0 : len(currentKeyBase)-1]
 	}
-
+	keys = append(keys, currentKeyBase[0])
 	return keys
 }
 
-func reverse(s [][]int) [][]int {
-	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
-		s[i], s[j] = s[j], s[i]
+// SortBySpecificity sorts an array of uints representing binary numbers, such that numbers with fewer 1-digits come first.
+// Numbers with an equal amount of 1-digits are sorted by magnitude.
+type sortBySpecificity []uint
+
+func (a sortBySpecificity) Len() int      { return len(a) }
+func (a sortBySpecificity) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a sortBySpecificity) Less(i, j int) bool {
+	onesI := bits.OnesCount(a[i])
+	onesJ := bits.OnesCount(a[j])
+	if onesI < onesJ {
+		return true
 	}
-	return s
-}
-
-func combinations(iterable []int, r int) (rt [][]int) {
-	pool := iterable
-	n := len(pool)
-
-	if r > n {
-		return
-	}
-
-	indices := make([]int, r)
-	for i := range indices {
-		indices[i] = i
-	}
-
-	result := make([]int, r)
-	for i, el := range indices {
-		result[i] = pool[el]
+	if onesI > onesJ {
+		return false
 	}
-	s2 := make([]int, r)
-	copy(s2, result)
-	rt = append(rt, s2)
-
-	for {
-		i := r - 1
-		for ; i >= 0 && indices[i] == i+n-r; i -= 1 {
-		}
-
-		if i < 0 {
-			return
-		}
-
-		indices[i] += 1
-		for j := i + 1; j < r; j += 1 {
-			indices[j] = indices[j-1] + 1
-		}
+	return a[i] < a[j]
+}
 
-		for ; i < len(indices); i += 1 {
-			result[i] = pool[indices[i]]
-		}
-		s2 = make([]int, r)
-		copy(s2, result)
-		rt = append(rt, s2)
+func makeRange(min, max int) []uint {
+	a := make([]uint, max-min)
+	for i := range a {
+		a[i] = uint(min + i)
 	}
+	return a
 }