Virtual Resource filters

README.md

@@ -138,7 +138,7 @@ item is included in the list.
 
 **If SQLite caching is enabled** (`server.Options.SQLCache=true`),
 filtering is only supported for a subset of attributes:
-- `metadata.name`, `metadata.namespace` and `metadata.timestamp` for any resource kind
+- `id`, `metadata.name`, `metadata.namespace`, `metadata.state.name`, and `metadata.timestamp` for any resource kind
 - a short list of hardcoded attributes for a selection of specific types listed
 in [typeSpecificIndexFields](https://github.com/rancher/steve/blob/main/pkg/stores/sqlproxy/proxy_store.go#L52-L58)
 - the special string `metadata.fields[N]`, with N starting at 0, for all columns

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/rancher/apiserver v0.0.0-20240708202538-39a6f2535146
 	github.com/rancher/dynamiclistener v0.6.0
 	github.com/rancher/kubernetes-provider-detector v0.1.5
-	github.com/rancher/lasso v0.0.0-20240809125800-8da6f11865d5
+	github.com/rancher/lasso v0.0.0-20240828170735-d79536cac289
 	github.com/rancher/norman v0.0.0-20240822182819-60ccfabc4ac5
 	github.com/rancher/remotedialer v0.3.2
 	github.com/rancher/wrangler/v3 v3.0.0

go.sum

@@ -193,8 +193,8 @@ github.com/rancher/dynamiclistener v0.6.0 h1:M7x8Nq+GY0UORULANuW/AH1ocnyZaqlmTuv
 github.com/rancher/dynamiclistener v0.6.0/go.mod h1:7VNEQhAwzbYJ08S1MYb6B4vili6K7CcrG4cNZXq1j+s=
 github.com/rancher/kubernetes-provider-detector v0.1.5 h1:hWRAsWuJOemzGjz/XrbTlM7QmfO4OedvFE3QwXiH60I=
 github.com/rancher/kubernetes-provider-detector v0.1.5/go.mod h1:ypuJS7kP7rUiAn330xG46mj+Nhvym05GM8NqMVekpH0=
-github.com/rancher/lasso v0.0.0-20240809125800-8da6f11865d5 h1:qlVhaHTT7wwrI5+AGdkYHpveuoe8Ot4TdQr7LtxmVSk=
-github.com/rancher/lasso v0.0.0-20240809125800-8da6f11865d5/go.mod h1:Efx/+BbH3ivmnTPLu5cA3Gc9wT5oyGS0LBcqEuYTx+A=
+github.com/rancher/lasso v0.0.0-20240828170735-d79536cac289 h1:gbV7qLOcEgyTgep2ocl8FFhfGOUMQuvfV5OIIENHWT4=
+github.com/rancher/lasso v0.0.0-20240828170735-d79536cac289/go.mod h1:Efx/+BbH3ivmnTPLu5cA3Gc9wT5oyGS0LBcqEuYTx+A=
 github.com/rancher/norman v0.0.0-20240822182819-60ccfabc4ac5 h1:Z34NXcW0ymdpVBfd1R0vvqTXBh1gCOdPNFtB+RUahQw=
 github.com/rancher/norman v0.0.0-20240822182819-60ccfabc4ac5/go.mod h1:dyjfXBsNiroPWOdUZe7diUOUSLf6HQ/r2kEpwH/8zas=
 github.com/rancher/remotedialer v0.3.2 h1:kstZbRwPS5gPWpGg8VjEHT2poHtArs+Fc317YM8JCzU=

pkg/resources/common/formatter.go

@@ -23,18 +23,19 @@ import (
 func DefaultTemplate(clientGetter proxy.ClientGetter,
 	summaryCache *summarycache.SummaryCache,
 	asl accesscontrol.AccessSetLookup,
-	namespaceCache corecontrollers.NamespaceCache) schema.Template {
+	namespaceCache corecontrollers.NamespaceCache,
+	sqlCache bool) schema.Template {
 	return schema.Template{
 		Store:     metricsStore.NewMetricsStore(proxy.NewProxyStore(clientGetter, summaryCache, asl, namespaceCache)),
-		Formatter: formatter(summaryCache),
+		Formatter: formatter(summaryCache, sqlCache),
 	}
 }
 
 // DefaultTemplateForStore provides a default schema template which uses a provided, pre-initialized store. Primarily used when creating a Template that uses a Lasso SQL store internally.
-func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache) schema.Template {
+func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache, sqlCache bool) schema.Template {
 	return schema.Template{
 		Store:     store,
-		Formatter: formatter(summaryCache),
+		Formatter: formatter(summaryCache, sqlCache),
 	}
 }
 
@@ -71,7 +72,7 @@ func selfLink(gvr schema2.GroupVersionResource, meta metav1.Object) (prefix stri
 	return buf.String()
 }
 
-func formatter(summarycache *summarycache.SummaryCache) types.Formatter {
+func formatter(summarycache *summarycache.SummaryCache, sqlCache bool) types.Formatter {
 	return func(request *types.APIRequest, resource *types.RawResource) {
 		if resource.Schema == nil {
 			return
@@ -104,17 +105,20 @@ func formatter(summarycache *summarycache.SummaryCache) types.Formatter {
 		}
 
 		if unstr, ok := resource.APIObject.Object.(*unstructured.Unstructured); ok {
-			s, rel := summarycache.SummaryAndRelationship(unstr)
-			data.PutValue(unstr.Object, map[string]interface{}{
-				"name":          s.State,
-				"error":         s.Error,
-				"transitioning": s.Transitioning,
-				"message":       strings.Join(s.Message, ":"),
-			}, "metadata", "state")
-			data.PutValue(unstr.Object, rel, "metadata", "relationships")
-
-			summary.NormalizeConditions(unstr)
+			if !sqlCache {
+				// with the sql cache, these were already added by the indexer
+				s, rel := summarycache.SummaryAndRelationship(unstr)
+				data.PutValue(unstr.Object, map[string]interface{}{
+					"name":          s.State,
+					"error":         s.Error,
+					"transitioning": s.Transitioning,
+					"message":       strings.Join(s.Message, ":"),
+				}, "metadata", "state")
+				data.PutValue(unstr.Object, rel, "metadata", "relationships")
+
+				summary.NormalizeConditions(unstr)
 
+			}
 			includeFields(request, unstr)
 			excludeFields(request, unstr)
 			excludeValues(request, unstr)

pkg/resources/schema.go

@@ -49,7 +49,7 @@ func DefaultSchemaTemplates(cf *client.Factory,
 	discovery discovery.DiscoveryInterface,
 	namespaceCache corecontrollers.NamespaceCache) []schema.Template {
 	return []schema.Template{
-		common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache),
+		common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache, false),
 		apigroups.Template(discovery),
 		{
 			ID:        "configmap",
@@ -79,7 +79,7 @@ func DefaultSchemaTemplatesForStore(store types.Store,
 	discovery discovery.DiscoveryInterface) []schema.Template {
 
 	return []schema.Template{
-		common.DefaultTemplateForStore(store, summaryCache),
+		common.DefaultTemplateForStore(store, summaryCache, true),
 		apigroups.Template(discovery),
 		{
 			ID:        "configmap",

pkg/resources/virtual/common/common.go

@@ -0,0 +1,116 @@
+// Package common provides cache.TransformFunc's which are common to all types
+package common
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/rancher/steve/pkg/summarycache"
+	"github.com/rancher/wrangler/v3/pkg/data"
+	wranglerSummary "github.com/rancher/wrangler/v3/pkg/summary"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/cache"
+)
+
+// SummaryCache provides an interface to get a summary/relationships for an object. Implemented by the summaryCache
+// struct from pkg/summarycache
+type SummaryCache interface {
+	SummaryAndRelationship(runtime.Object) (*wranglerSummary.SummarizedObject, []summarycache.Relationship)
+}
+
+// DefaultFields produces a VirtualTransformFunc through GetTransform() that applies to all k8s types
+type DefaultFields struct {
+	Cache SummaryCache
+}
+
+// GetTransform produces the default transformation func
+func (d *DefaultFields) GetTransform() cache.TransformFunc {
+	return d.transform
+}
+
+// transform implements virtual.VirtualTransformFunc, and adds reserved fields/summary
+func (d *DefaultFields) transform(obj any) (any, error) {
+	raw, isSignal, err := getUnstructured(obj)
+	if isSignal {
+		return obj, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	raw = addIDField(raw)
+	raw, err = addSummaryFields(raw, d.Cache)
+	if err != nil {
+		return nil, fmt.Errorf("unable to add summary fields: %w", err)
+	}
+	return raw, nil
+}
+
+// addSummaryFields adds the virtual fields for object state.
+func addSummaryFields(raw *unstructured.Unstructured, cache SummaryCache) (*unstructured.Unstructured, error) {
+	s, relationships := cache.SummaryAndRelationship(raw)
+	if s != nil {
+		data.PutValue(raw.Object, map[string]interface{}{
+			"name":          s.State,
+			"error":         s.Error,
+			"transitioning": s.Transitioning,
+			"message":       strings.Join(s.Message, ":"),
+		}, "metadata", "state")
+
+	}
+	var rels []any
+	for _, relationship := range relationships {
+		rel, err := toMap(relationship)
+		if err != nil {
+			return nil, fmt.Errorf("unable to convert relationship to map: %w", err)
+		}
+		rels = append(rels, rel)
+	}
+	data.PutValue(raw.Object, rels, "metadata", "relationships")
+
+	normalizeConditions(raw)
+	return raw, nil
+}
+
+// addIDField adds the ID field based on namespace/name, and moves the current id field to _id if present
+func addIDField(raw *unstructured.Unstructured) *unstructured.Unstructured {
+	objectID := raw.GetName()
+	namespace := raw.GetNamespace()
+	if namespace != "" {
+		objectID = fmt.Sprintf("%s/%s", namespace, objectID)
+	}
+	currentIDValue, ok := raw.Object["id"]
+	if ok {
+		raw.Object["_id"] = currentIDValue
+	}
+	raw.Object["id"] = objectID
+	return raw
+}
+
+func normalizeConditions(raw *unstructured.Unstructured) {
+	var (
+		obj           data.Object
+		newConditions []any
+	)
+
+	obj = raw.Object
+	for _, condition := range obj.Slice("status", "conditions") {
+		var summary wranglerSummary.Summary
+		for _, summarizer := range wranglerSummary.ConditionSummarizers {
+			summary = summarizer(obj, []wranglerSummary.Condition{{Object: condition}}, summary)
+		}
+		condition.Set("error", summary.Error)
+		condition.Set("transitioning", summary.Transitioning)
+
+		if condition.String("lastUpdateTime") == "" {
+			condition.Set("lastUpdateTime", condition.String("lastTransitionTime"))
+		}
+		// needs to be reconverted back to a map[string]any or we can have encoding problems with unregistered types
+		var mapCondition map[string]any = condition
+		newConditions = append(newConditions, mapCondition)
+	}
+
+	if len(newConditions) > 0 {
+		obj.SetNested(newConditions, "status", "conditions")
+	}
+}