Updates in cache configuration to allow disabling of all caching on s…

…erver. Closes #3258

STEAMPIPE_CACHE environment variable resolves to service cache enabled as well as client cache enabled

service cache enabled is used by the plugin manager to enable/disable caching on the plugins during startup (runtime toggle is not allowed) - with a max_ttl

client cache enabled is used to enable/disable the cache on the database connection (fdw)
A TTL can also be set on the client side capped to max_ttl on the server

cmd/root.go

@@ -61,7 +61,6 @@ var rootCmd = &cobra.Command{
 		defer func() {
 			if r := recover(); r != nil {
 				error_helpers.ShowError(cmd.Context(), helpers.ToError(r))
-				debug.PrintStack()
 			}
 		}()
 
@@ -283,7 +282,7 @@ func setCloudTokenDefault(loader *steampipeconfig.WorkspaceProfileLoader) error
 		viper.SetDefault(constants.ArgCloudToken, *loader.DefaultProfile.CloudToken)
 	}
 	// 3) env var (STEAMIPE_CLOUD_TOKEN )
-	cmdconfig.SetDefaultFromEnv(constants.EnvCloudToken, constants.ArgCloudToken, "string")
+	cmdconfig.SetDefaultFromEnv(constants.EnvCloudToken, constants.ArgCloudToken, cmdconfig.String)
 
 	// 4) explicit workspace profile
 	if p := loader.ConfiguredProfile; p != nil && p.CloudToken != nil {
@@ -294,9 +293,9 @@ func setCloudTokenDefault(loader *steampipeconfig.WorkspaceProfileLoader) error
 
 func loadWorkspaceProfile() (*steampipeconfig.WorkspaceProfileLoader, error) {
 	// set viper default for workspace profile, using STEAMPIPE_WORKSPACE env var
-	cmdconfig.SetDefaultFromEnv(constants.EnvWorkspaceProfile, constants.ArgWorkspaceProfile, "string")
+	cmdconfig.SetDefaultFromEnv(constants.EnvWorkspaceProfile, constants.ArgWorkspaceProfile, cmdconfig.String)
 	// set viper default for install dir, using STEAMPIPE_INSTALL_DIR env var
-	cmdconfig.SetDefaultFromEnv(constants.EnvInstallDir, constants.ArgInstallDir, "string")
+	cmdconfig.SetDefaultFromEnv(constants.EnvInstallDir, constants.ArgInstallDir, cmdconfig.String)
 
 	// resolve the workspace profile dir
 	installDir, err := filehelpers.Tildefy(viper.GetString(constants.ArgInstallDir))

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/stevenle/topsort v0.2.0
 	github.com/turbot/go-kit v0.5.0
 	github.com/turbot/steampipe-cloud-sdk-go v0.5.0
-	github.com/turbot/steampipe-plugin-sdk/v5 v5.3.0-rc.0
+	github.com/turbot/steampipe-plugin-sdk/v5 v5.4.0-rc.1
 	github.com/xlab/treeprint v1.2.0
 	github.com/zclconf/go-cty v1.13.1
 	github.com/zclconf/go-cty-yaml v1.0.3
@@ -232,7 +232,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	golang.org/x/crypto v0.6.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
+	golang.org/x/mod v0.9.0 // indirect
 	golang.org/x/net v0.8.0 // indirect
 	rsc.io/letsencrypt v0.0.3 // indirect
 )

go.sum

@@ -1035,8 +1035,8 @@ github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:
 github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw=
 github.com/turbot/steampipe-cloud-sdk-go v0.5.0 h1:sgMpUL/gLnT5/9v6LaUDITo40npvSn+RQD5maT03wRQ=
 github.com/turbot/steampipe-cloud-sdk-go v0.5.0/go.mod h1:8M2CspUHgCGqDCJV+FNn+boBPyLRHyzDinYnoZ/kZYw=
-github.com/turbot/steampipe-plugin-sdk/v5 v5.3.0-rc.0 h1:tNKkRvzMftQ9SdPeTkbsKj+Hr515lotvyNgpu3ZAbxU=
-github.com/turbot/steampipe-plugin-sdk/v5 v5.3.0-rc.0/go.mod h1:bfvdKaLwemr13EPOMbnr7/hOQOO7m4oM7HnVMdNotgY=
+github.com/turbot/steampipe-plugin-sdk/v5 v5.4.0-rc.1 h1:X1mJovPJLyCiLDe12CN+XLoXica0elNz0LltSShn/WI=
+github.com/turbot/steampipe-plugin-sdk/v5 v5.4.0-rc.1/go.mod h1:bfvdKaLwemr13EPOMbnr7/hOQOO7m4oM7HnVMdNotgY=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
@@ -1205,8 +1205,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
-golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
+golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/cmdconfig/env_var_type.go

@@ -0,0 +1,11 @@
+package cmdconfig
+
+type EnvVarType int
+
+const (
+	String EnvVarType = iota
+	Int
+	Bool
+)
+
+//go:generate go run golang.org/x/tools/cmd/stringer -type=EnvVarType

pkg/cmdconfig/viper.go

@@ -83,15 +83,24 @@ func SetDefaultsFromConfig(configMap map[string]interface{}) {
 }
 
 // for keys which do not have a corresponding command flag, we need a separate defaulting mechanism
+// any option setting, workspace profile property or env var which does not have a command line
+// MUST have a default (unless we want the zero value to take effect)
 func setBaseDefaults() {
 	defaults := map[string]interface{}{
-		constants.ArgUpdateCheck:          true,
-		constants.ArgTelemetry:            constants.TelemetryInfo,
+		// global general options
+		constants.ArgTelemetry:   constants.TelemetryInfo,
+		constants.ArgUpdateCheck: true,
+
+		// workspace profile
+		constants.ArgAutoComplete:  true,
+		constants.ArgIntrospection: constants.IntrospectionNone,
+
+		// from global database options
 		constants.ArgDatabasePort:         constants.DatabaseDefaultPort,
-		constants.ArgMaxCacheSizeMb:       constants.DefaultMaxCacheSizeMb,
-		constants.ArgAutoComplete:         true,
 		constants.ArgDatabaseStartTimeout: constants.DBStartTimeout.Seconds(),
-		constants.ArgIntrospection:        constants.IntrospectionNone,
+		constants.ArgServiceCacheEnabled:  true,
+		constants.ArgCacheMaxTtl:          300,
+		constants.ArgMaxCacheSizeMb:       constants.DefaultMaxCacheSizeMb,
 	}
 
 	for k, v := range defaults {
@@ -100,21 +109,20 @@ func setBaseDefaults() {
 }
 
 type envMapping struct {
-	configVar string
-	// "string", "int", "bool"
-	varType string
+	configVar []string
+	varType   EnvVarType
 }
 
 // set default values of INSTALL_DIR and ModLocation from env vars
 func setDirectoryDefaultsFromEnv() {
 	envMappings := map[string]envMapping{
-		constants.EnvInstallDir:     {constants.ArgInstallDir, "string"},
-		constants.EnvWorkspaceChDir: {constants.ArgModLocation, "string"},
-		constants.EnvModLocation:    {constants.ArgModLocation, "string"},
+		constants.EnvInstallDir:     {[]string{constants.ArgInstallDir}, String},
+		constants.EnvWorkspaceChDir: {[]string{constants.ArgModLocation}, String},
+		constants.EnvModLocation:    {[]string{constants.ArgModLocation}, String},
 	}
 
-	for k, v := range envMappings {
-		SetDefaultFromEnv(k, v.configVar, v.varType)
+	for envVar, mapping := range envMappings {
+		setConfigFromEnv(envVar, mapping.configVar, mapping.varType)
 	}
 }
 
@@ -125,40 +133,52 @@ func SetDefaultsFromEnv() {
 
 	// a map of known environment variables to map to viper keys
 	envMappings := map[string]envMapping{
-		constants.EnvInstallDir:           {constants.ArgInstallDir, "string"},
-		constants.EnvWorkspaceChDir:       {constants.ArgModLocation, "string"},
-		constants.EnvModLocation:          {constants.ArgModLocation, "string"},
-		constants.EnvIntrospection:        {constants.ArgIntrospection, "string"},
-		constants.EnvTelemetry:            {constants.ArgTelemetry, "string"},
-		constants.EnvUpdateCheck:          {constants.ArgUpdateCheck, "bool"},
-		constants.EnvCloudHost:            {constants.ArgCloudHost, "string"},
-		constants.EnvCloudToken:           {constants.ArgCloudToken, "string"},
-		constants.EnvSnapshotLocation:     {constants.ArgSnapshotLocation, "string"},
-		constants.EnvWorkspaceDatabase:    {constants.ArgWorkspaceDatabase, "string"},
-		constants.EnvServicePassword:      {constants.ArgServicePassword, "string"},
-		constants.EnvCheckDisplayWidth:    {constants.ArgCheckDisplayWidth, "int"},
-		constants.EnvMaxParallel:          {constants.ArgMaxParallel, "int"},
-		constants.EnvQueryTimeout:         {constants.ArgDatabaseQueryTimeout, "int"},
-		constants.EnvDatabaseStartTimeout: {constants.ArgDatabaseStartTimeout, "int"},
-		constants.EnvCacheEnabled:         {constants.ArgCache, "bool"},
-		constants.EnvCacheTTL:             {constants.ArgCacheTtl, "int"},
+		constants.EnvInstallDir:           {[]string{constants.ArgInstallDir}, String},
+		constants.EnvWorkspaceChDir:       {[]string{constants.ArgModLocation}, String},
+		constants.EnvModLocation:          {[]string{constants.ArgModLocation}, String},
+		constants.EnvIntrospection:        {[]string{constants.ArgIntrospection}, String},
+		constants.EnvTelemetry:            {[]string{constants.ArgTelemetry}, String},
+		constants.EnvUpdateCheck:          {[]string{constants.ArgUpdateCheck}, Bool},
+		constants.EnvCloudHost:            {[]string{constants.ArgCloudHost}, String},
+		constants.EnvCloudToken:           {[]string{constants.ArgCloudToken}, String},
+		constants.EnvSnapshotLocation:     {[]string{constants.ArgSnapshotLocation}, String},
+		constants.EnvWorkspaceDatabase:    {[]string{constants.ArgWorkspaceDatabase}, String},
+		constants.EnvServicePassword:      {[]string{constants.ArgServicePassword}, String},
+		constants.EnvCheckDisplayWidth:    {[]string{constants.ArgCheckDisplayWidth}, Int},
+		constants.EnvMaxParallel:          {[]string{constants.ArgMaxParallel}, Int},
+		constants.EnvQueryTimeout:         {[]string{constants.ArgDatabaseQueryTimeout}, Int},
+		constants.EnvDatabaseStartTimeout: {[]string{constants.ArgDatabaseStartTimeout}, Int},
+		constants.EnvCacheTTL:             {[]string{constants.ArgCacheTtl}, Int},
+		constants.EnvCacheMaxTTL:          {[]string{constants.ArgCacheMaxTtl}, Int},
+
+		// we need this value to go into different locations
+		constants.EnvCacheEnabled: {[]string{
+			constants.ArgClientCacheEnabled,
+			constants.ArgServiceCacheEnabled,
+		}, Bool},
 	}
 
-	for k, v := range envMappings {
-		SetDefaultFromEnv(k, v.configVar, v.varType)
+	for envVar, v := range envMappings {
+		setConfigFromEnv(envVar, v.configVar, v.varType)
+	}
+}
+
+func setConfigFromEnv(envVar string, configs []string, varType EnvVarType) {
+	for _, configVar := range configs {
+		SetDefaultFromEnv(envVar, configVar, varType)
 	}
 }
 
-func SetDefaultFromEnv(k string, configVar string, varType string) {
+func SetDefaultFromEnv(k string, configVar string, varType EnvVarType) {
 	if val, ok := os.LookupEnv(k); ok {
 		switch varType {
-		case "string":
+		case String:
 			viper.SetDefault(configVar, val)
-		case "bool":
+		case Bool:
 			if boolVal, err := types.ToBool(val); err == nil {
 				viper.SetDefault(configVar, boolVal)
 			}
-		case "int":
+		case Int:
 			if intVal, err := types.ToInt64(val); err == nil {
 				viper.SetDefault(configVar, intVal)
 			}

pkg/constants/args.go

@@ -50,8 +50,10 @@ const (
 	ArgInput                = "input"
 	ArgDashboardInput       = "dashboard-input"
 	ArgMaxCacheSizeMb       = "max-cache-size-mb"
-	ArgCache                = "cache"
 	ArgCacheTtl             = "cache-ttl"
+	ArgClientCacheEnabled   = "client-cache-enabled"
+	ArgServiceCacheEnabled  = "service-cache-enabled"
+	ArgCacheMaxTtl          = "cache-max-ttl"
 	ArgIntrospection        = "introspection"
 	ArgShare                = "share"
 	ArgSnapshot             = "snapshot"

pkg/constants/db.go

@@ -6,7 +6,7 @@ import (
 	"github.com/turbot/steampipe/pkg/schema"
 )
 
-// dbClient constants
+// Client constants
 const (
 	// MaxParallelClientInits is the number of clients to initialize in parallel
 	// if we start initializing all clients together, it leads to bad performance on all
@@ -32,7 +32,7 @@ const (
 // constants for installing db and fdw images
 const (
 	DatabaseVersion = "14.2.0"
-	FdwVersion      = "1.6.2"
+	FdwVersion      = "1.7.0-rc.0"
 
 	// PostgresImageRef is the OCI Image ref for the database binaries
 	PostgresImageRef    = "us-docker.pkg.dev/steampipe/steampipe/db:14.2.0"
@@ -50,11 +50,13 @@ const (
 	// CommandSchema is the schema which is used to send commands to the FDW
 	CommandSchema = "steampipe_command"
 
-	CommandTableCache                = "cache"
-	CommandTableCacheOperationColumn = "operation"
-	CommandCacheOn                   = "cache_on"
-	CommandCacheOff                  = "cache_off"
-	CommandCacheClear                = "cache_clear"
+	CommandTableSettings            = "settings"
+	CommandTableSettingsKeyColumn   = "name"
+	CommandTableSettingsValueColumn = "value"
+
+	CommandTableSettingsCacheKey          = "cache"
+	CommandTableSettingsCacheTtlKey       = "cache_ttl"
+	CommandTableSettingsCacheClearTimeKey = "cache_clear_time"
 
 	CommandTableScanMetadata = "scan_metadata"
 )

pkg/constants/env.go

@@ -19,6 +19,7 @@ const (
 	EnvCheckDisplayWidth = "STEAMPIPE_CHECK_DISPLAY_WIDTH"
 	EnvCacheEnabled      = "STEAMPIPE_CACHE"
 	EnvCacheTTL          = "STEAMPIPE_CACHE_TTL"
+	EnvCacheMaxTTL       = "STEAMPIPE_CACHE_MAX_TTL"
 	EnvCacheMaxSize      = "STEAMPIPE_CACHE_MAX_SIZE_MB"
 	EnvQueryTimeout      = "STEAMPIPE_QUERY_TIMEOUT"
 

pkg/constants/metaquery_commands.go

@@ -20,6 +20,7 @@ const (
 	CmdSearchPath       = ".search_path"        // Set or show search-path
 	CmdSearchPathPrefix = ".search_path_prefix" // set search path prefix
 	CmdCache            = ".cache"              // cache control
+	CmdCacheTtl         = ".cache_ttl"          // set cache ttl
 	CmdAutoComplete     = ".autocomplete"       // enable or disable auto complete
 )
 

pkg/db/db_client/db_client_session.go

@@ -4,8 +4,11 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/spf13/viper"
+	"github.com/turbot/steampipe/pkg/constants"
 	"github.com/turbot/steampipe/pkg/db/db_common"
 	"github.com/turbot/steampipe/pkg/utils"
 )
@@ -65,10 +68,27 @@ func (c *DbClient) AcquireSession(ctx context.Context) (sessionResult *db_common
 	// make sure that we close the acquired session, in case of error
 	defer func() {
 		if sessionResult.Error != nil && databaseConnection != nil {
+			sessionResult.Session = nil
 			databaseConnection.Release()
 		}
 	}()
 
+	// if the cache is set on the workspace profile
+	// then override the default cache setting of the connection
+	if viper.IsSet(constants.ArgClientCacheEnabled) {
+		if err := db_common.SetCacheEnabled(ctx, viper.GetBool(constants.ArgClientCacheEnabled), databaseConnection.Conn()); err != nil {
+			sessionResult.Error = err
+			return sessionResult
+		}
+	}
+	if viper.IsSet(constants.ArgCacheTtl) {
+		ttl := time.Duration(viper.GetInt(constants.ArgCacheTtl)) * time.Second
+		if err := db_common.SetCacheTtl(ctx, ttl, databaseConnection.Conn()); err != nil {
+			sessionResult.Error = err
+			return sessionResult
+		}
+	}
+
 	// update required session search path if needed
 	err = c.ensureSessionSearchPath(ctx, session)
 	if err != nil {

pkg/db/db_common/acquire_session_result.go

@@ -3,6 +3,6 @@ package db_common
 import "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
 
 type AcquireSessionResult struct {
-	modconfig.ErrorAndWarnings
 	Session *DatabaseSession
+	modconfig.ErrorAndWarnings
 }