SQLite backed cache

go.mod

@@ -1,6 +1,8 @@
 module github.com/rancher/steve
 
-go 1.22
+go 1.22.0
+
+toolchain go1.22.2
 
 replace (
 	github.com/crewjam/saml => github.com/rancher/saml v0.2.0
@@ -21,6 +23,7 @@ require (
 	github.com/rancher/apiserver v0.0.0-20240503193545-2e1b0ddd9791
 	github.com/rancher/dynamiclistener v0.5.0-rc6
 	github.com/rancher/kubernetes-provider-detector v0.1.5
+	github.com/rancher/lasso v0.0.0-20240603075835-701e919d08b7
 	github.com/rancher/norman v0.0.0-20240503193601-9f5f6586bb5b
 	github.com/rancher/remotedialer v0.3.2
 	github.com/rancher/wrangler/v2 v2.2.0-rc6
@@ -30,14 +33,14 @@ require (
 	github.com/urfave/cli/v2 v2.27.1
 	golang.org/x/sync v0.7.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.29.3
+	k8s.io/api v0.30.0
 	k8s.io/apiextensions-apiserver v0.29.3
-	k8s.io/apimachinery v0.29.3
+	k8s.io/apimachinery v0.30.0
 	k8s.io/apiserver v0.29.3
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/klog v1.0.0
 	k8s.io/kube-aggregator v0.29.3
-	k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340
 )
 
 require (
@@ -47,6 +50,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
@@ -60,22 +64,25 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/rancher/lasso v0.0.0-20240424194130-d87ec407d941 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
@@ -104,6 +111,13 @@ require (
 	k8s.io/component-base v0.29.3 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	modernc.org/libc v1.49.3 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.8.0 // indirect
+	modernc.org/sqlite v1.29.10 // indirect
+	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/token v1.1.0 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 // indirect
 	sigs.k8s.io/cli-utils v0.35.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

main.go

@@ -34,7 +34,7 @@ func main() {
 func run(_ *cli.Context) error {
 	ctx := signals.SetupSignalContext()
 	debugconfig.MustSetupDebug()
-	s, err := config.ToServer(ctx)
+	s, err := config.ToServer(ctx, false)
 	if err != nil {
 		return err
 	}

pkg/controllers/schema/schemas.go

@@ -31,8 +31,10 @@ var (
 	}
 )
 
-type SchemasHandler interface {
-	OnSchemas(schemas *schema2.Collection) error
+type SchemasHandlerFunc func(schemas *schema2.Collection) error
+
+func (s SchemasHandlerFunc) OnSchemas(schemas *schema2.Collection) error {
+	return s(schemas)
 }
 
 type handler struct {
@@ -45,7 +47,7 @@ type handler struct {
 	cols    *common.DynamicColumns
 	crd     apiextcontrollerv1.CustomResourceDefinitionClient
 	ssar    authorizationv1client.SelfSubjectAccessReviewInterface
-	handler SchemasHandler
+	handler SchemasHandlerFunc
 }
 
 func Register(ctx context.Context,
@@ -54,7 +56,7 @@ func Register(ctx context.Context,
 	crd apiextcontrollerv1.CustomResourceDefinitionController,
 	apiService v1.APIServiceController,
 	ssar authorizationv1client.SelfSubjectAccessReviewInterface,
-	schemasHandler SchemasHandler,
+	schemasHandler SchemasHandlerFunc,
 	schemas *schema2.Collection) {
 
 	h := &handler{

pkg/resources/common/formatter.go

@@ -30,6 +30,14 @@ func DefaultTemplate(clientGetter proxy.ClientGetter,
 	}
 }
 
+// DefaultTemplateForStore provides a default schema template which uses a provided, pre-initialized store. Primarily used when creating a Template that uses a Lasso SQL store internally.
+func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache) schema.Template {
+	return schema.Template{
+		Store:     store,
+		Formatter: formatter(summaryCache),
+	}
+}
+
 func selfLink(gvr schema2.GroupVersionResource, meta metav1.Object) (prefix string) {
 	buf := &strings.Builder{}
 	if gvr.Group == "management.cattle.io" && gvr.Version == "v3" {

pkg/resources/schema.go

@@ -71,3 +71,33 @@ func DefaultSchemaTemplates(cf *client.Factory,
 		},
 	}
 }
+
+// DefaultSchemaTemplatesForStore returns the same default templates as DefaultSchemaTemplates, only using DefaultSchemaTemplateFoStore internally to construct the templates.
+func DefaultSchemaTemplatesForStore(store types.Store,
+	baseSchemas *types.APISchemas,
+	summaryCache *summarycache.SummaryCache,
+	discovery discovery.DiscoveryInterface) []schema.Template {
+
+	return []schema.Template{
+		common.DefaultTemplateForStore(store, summaryCache),
+		apigroups.Template(discovery),
+		{
+			ID:        "configmap",
+			Formatter: formatters.DropHelmData,
+		},
+		{
+			ID:        "secret",
+			Formatter: formatters.DropHelmData,
+		},
+		{
+			ID:        "pod",
+			Formatter: formatters.Pod,
+		},
+		{
+			ID: "management.cattle.io.cluster",
+			Customize: func(apiSchema *types.APISchema) {
+				cluster.AddApply(baseSchemas, apiSchema)
+			},
+		},
+	}
+}

pkg/server/cli/clicontext.go

@@ -23,14 +23,14 @@ type Config struct {
 }
 
 func (c *Config) MustServer(ctx context.Context) *server.Server {
-	cc, err := c.ToServer(ctx)
+	cc, err := c.ToServer(ctx, false)
 	if err != nil {
 		panic(err)
 	}
 	return cc
 }
 
-func (c *Config) ToServer(ctx context.Context) (*server.Server, error) {
+func (c *Config) ToServer(ctx context.Context, sqlCache bool) (*server.Server, error) {
 	var (
 		auth steveauth.Middleware
 	)
@@ -51,6 +51,7 @@ func (c *Config) ToServer(ctx context.Context) (*server.Server, error) {
 	return server.New(ctx, restConfig, &server.Options{
 		AuthMiddleware: auth,
 		Next:           ui.New(c.UIPath),
+		SQLCache:       sqlCache,
 	})
 }
 

pkg/server/server.go

@@ -21,6 +21,10 @@ import (
 	"github.com/rancher/steve/pkg/schema/definitions"
 	"github.com/rancher/steve/pkg/server/handler"
 	"github.com/rancher/steve/pkg/server/router"
+	metricsStore "github.com/rancher/steve/pkg/stores/metrics"
+	"github.com/rancher/steve/pkg/stores/proxy"
+	"github.com/rancher/steve/pkg/stores/sqlpartition"
+	"github.com/rancher/steve/pkg/stores/sqlproxy"
 	"github.com/rancher/steve/pkg/summarycache"
 	"k8s.io/client-go/rest"
 )
@@ -48,6 +52,7 @@ type Server struct {
 
 	aggregationSecretNamespace string
 	aggregationSecretName      string
+	SQLCache                   bool
 }
 
 type Options struct {
@@ -62,6 +67,8 @@ type Options struct {
 	AggregationSecretName      string
 	ClusterRegistry            string
 	ServerVersion              string
+	// SQLCache enables the SQLite-based lasso caching mechanism
+	SQLCache bool
 }
 
 func New(ctx context.Context, restConfig *rest.Config, opts *Options) (*Server, error) {
@@ -81,6 +88,8 @@ func New(ctx context.Context, restConfig *rest.Config, opts *Options) (*Server,
 		aggregationSecretName:      opts.AggregationSecretName,
 		ClusterRegistry:            opts.ClusterRegistry,
 		Version:                    opts.ServerVersion,
+		// SQLCache enables the SQLite-based lasso caching mechanism
+		SQLCache: opts.SQLCache,
 	}
 
 	if err := setup(ctx, server); err != nil {
@@ -147,16 +156,52 @@ func setup(ctx context.Context, server *Server) error {
 
 	summaryCache := summarycache.New(sf, ccache)
 	summaryCache.Start(ctx)
-
-	for _, template := range resources.DefaultSchemaTemplates(cf, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), server.controllers.Core.Namespace().Cache()) {
-		sf.AddTemplate(template)
-	}
-
 	cols, err := common.NewDynamicColumns(server.RESTConfig)
 	if err != nil {
 		return err
 	}
 
+	var onSchemasHandler schemacontroller.SchemasHandlerFunc
+	if server.SQLCache {
+		s, err := sqlproxy.NewProxyStore(cols, cf, summaryCache, nil)
+		if err != nil {
+			panic(err)
+		}
+
+		errStore := proxy.NewErrorStore(
+			proxy.NewUnformatterStore(
+				proxy.NewWatchRefresh(
+					sqlpartition.NewStore(
+						s,
+						asl,
+					),
+					asl,
+				),
+			),
+		)
+		store := metricsStore.NewMetricsStore(errStore)
+		// end store setup code
+
+		for _, template := range resources.DefaultSchemaTemplatesForStore(store, server.BaseSchemas, summaryCache, server.controllers.K8s.Discovery()) {
+			sf.AddTemplate(template)
+		}
+
+		onSchemasHandler = func(schemas *schema.Collection) error {
+			if err := ccache.OnSchemas(schemas); err != nil {
+				return err
+			}
+			if err := s.Reset(); err != nil {
+				return err
+			}
+			return nil
+		}
+	} else {
+		for _, template := range resources.DefaultSchemaTemplates(cf, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), server.controllers.Core.Namespace().Cache()) {
+			sf.AddTemplate(template)
+		}
+		onSchemasHandler = ccache.OnSchemas
+	}
+
 	schemas.SetupWatcher(ctx, server.BaseSchemas, asl, sf)
 
 	schemacontroller.Register(ctx,
@@ -165,7 +210,7 @@ func setup(ctx context.Context, server *Server) error {
 		server.controllers.CRD.CustomResourceDefinition(),
 		server.controllers.API.APIService(),
 		server.controllers.K8s.AuthorizationV1().SelfSubjectAccessReviews(),
-		ccache,
+		onSchemasHandler,
 		sf)
 
 	apiServer, handler, err := handler.New(server.RESTConfig, sf, server.authMiddleware, server.next, server.router)
@@ -176,6 +221,7 @@ func setup(ctx context.Context, server *Server) error {
 	server.APIServer = apiServer
 	server.Handler = handler
 	server.SchemaFactory = sf
+
 	return nil
 }
 

pkg/stores/partition/store.go

@@ -110,7 +110,7 @@ func (s *Store) Delete(apiOp *types.APIRequest, schema *types.APISchema, id stri
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return toAPI(schema, obj, warnings), nil
+	return ToAPI(schema, obj, warnings), nil
 }
 
 // ByID looks up a single object by its ID.
@@ -124,7 +124,7 @@ func (s *Store) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return toAPI(schema, obj, warnings), nil
+	return ToAPI(schema, obj, warnings), nil
 }
 
 func (s *Store) listPartition(ctx context.Context, apiOp *types.APIRequest, schema *types.APISchema, partition Partition,
@@ -226,7 +226,7 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP
 
 	for _, item := range list {
 		item := item.DeepCopy()
-		result.Objects = append(result.Objects, toAPI(schema, item, nil))
+		result.Objects = append(result.Objects, ToAPI(schema, item, nil))
 	}
 
 	result.Pages = pages
@@ -266,7 +266,7 @@ func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, data ty
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return toAPI(schema, obj, warnings), nil
+	return ToAPI(schema, obj, warnings), nil
 }
 
 // Update updates a single object in the store.
@@ -280,7 +280,7 @@ func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, data ty
 	if err != nil {
 		return types.APIObject{}, err
 	}
-	return toAPI(schema, obj, warnings), nil
+	return ToAPI(schema, obj, warnings), nil
 }
 
 // Watch returns a channel of events for a list or resource.
@@ -310,7 +310,7 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types
 				return err
 			}
 			for i := range c {
-				response <- toAPIEvent(apiOp, schema, i)
+				response <- ToAPIEvent(apiOp, schema, i)
 			}
 			return nil
 		})
@@ -326,7 +326,7 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types
 	return response, nil
 }
 
-func toAPI(schema *types.APISchema, obj runtime.Object, warnings []types.Warning) types.APIObject {
+func ToAPI(schema *types.APISchema, obj runtime.Object, warnings []types.Warning) types.APIObject {
 	if obj == nil || reflect.ValueOf(obj).IsNil() {
 		return types.APIObject{}
 	}
@@ -372,7 +372,7 @@ func moveToUnderscore(obj *unstructured.Unstructured) *unstructured.Unstructured
 	return obj
 }
 
-func toAPIEvent(apiOp *types.APIRequest, schema *types.APISchema, event watch.Event) types.APIEvent {
+func ToAPIEvent(apiOp *types.APIRequest, schema *types.APISchema, event watch.Event) types.APIEvent {
 	name := types.ChangeAPIEvent
 	switch event.Type {
 	case watch.Deleted:
@@ -393,7 +393,7 @@ func toAPIEvent(apiOp *types.APIRequest, schema *types.APISchema, event watch.Ev
 		return apiEvent
 	}
 
-	apiEvent.Object = toAPI(schema, event.Object, nil)
+	apiEvent.Object = ToAPI(schema, event.Object, nil)
 
 	m, err := meta.Accessor(event.Object)
 	if err != nil {