Ability to Use Postgres as a Backing Store for VSecM Safe (#1165)

Maintains feature parity with the former version.

Unit and integration tests pass.

I'm merging this. Will do follow-up PRs to address `TODO:` comments in the code, and also clean-up the code.

---

* wip

* add polling to main to test config setting for safe itself

* temporarily disable relay client

* testing persistence to postgres

* SQL statement change

* lastworking

* add entity.go

* mostly-working version

* fixed store response

* add guard clauses

* more debugging

* exception for initial persistence to db

* changes

app/safe/cmd/entity.go

@@ -0,0 +1,47 @@
+/*
+|    Protect your secrets, protect your sensitive data.
+:    Explore VMware Secrets Manager docs at https://vsecm.com/
+</
+<>/  keep your secrets... secret
+>/
+<>/' Copyright 2023-present VMware Secrets Manager contributors.
+>/'  SPDX-License-Identifier: BSD-2-Clause
+*/
+
+package main
+
+// TODO: obviously there is a need for cleanup; once things start to work
+// as expected, move the codes to where they should belong.
+
+// TODO: move me to a proper place.
+
+// TODO: by design, VSecM Safe will not use more than one backing store
+// (create an ADR for that).
+// This means, there is a chicken-and-the-egg problem for persisting the
+// internal VSecM Safe configuration.
+//
+// For postgres backing store, VSecM Safe should keep its initial config
+// in memory until the database is there; and then it should save it to
+// the database, too.
+
+// TODO: we should check for the existence of the table in postgres and
+// log an error if it's not there.
+
+// TODO: when postgres mode vsecm safe shall be read-only (except for config update)
+// until it is initialized. once initialized, it should save its config to postgres too
+// and then it should be readwrite.
+
+// TODO: we need documentation for this postgres store feature. (and also a demo recording)
+
+// TODO: it's best block requests when the db is not ready yet (in postgres mode)
+// because otherwise, the initCommand will retry in exponential backoff and
+// eventually give up.
+// or the keystone secret will not be persisted although keystone will
+// be informed that safe is ready.
+
+type SafeConfig struct {
+	Config struct {
+		BackingStore   string `json:"backingStore"`
+		DataSourceName string `json:"dataSourceName"`
+	} `json:"config"`
+}

app/safe/cmd/main.go

@@ -12,18 +12,48 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"time"
 
 	"github.com/spiffe/go-spiffe/v2/workloadapi"
 
 	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/bootstrap"
 	server "github.com/vmware-tanzu/secrets-manager/app/safe/internal/server/engine"
+	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/io"
+	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/secret/collection"
 	"github.com/vmware-tanzu/secrets-manager/core/constants/env"
 	"github.com/vmware-tanzu/secrets-manager/core/constants/key"
 	"github.com/vmware-tanzu/secrets-manager/core/crypto"
+	entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
+	cEnv "github.com/vmware-tanzu/secrets-manager/core/env"
 	log "github.com/vmware-tanzu/secrets-manager/core/log/std"
 	"github.com/vmware-tanzu/secrets-manager/core/probe"
 )
 
+func pollForConfig(ctx context.Context, id string) (*SafeConfig, error) {
+	for {
+		log.InfoLn(&id, "Polling for VSecM Safe internal configuration")
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+			vSecMSafeInternalConfig, err := collection.ReadSecret(id, "vsecm-safe")
+			if err != nil {
+				log.InfoLn(&id, "Failed to load VSecM Safe internal configuration", err.Error())
+			} else if vSecMSafeInternalConfig != nil && len(vSecMSafeInternalConfig.Values) > 0 {
+				var safeConfig SafeConfig
+				err := json.Unmarshal([]byte(vSecMSafeInternalConfig.Values[0]), &safeConfig)
+				if err != nil {
+					log.InfoLn(&id, "Failed to parse VSecM Safe internal configuration", err.Error())
+				} else {
+					return &safeConfig, nil
+				}
+			}
+			time.Sleep(5 * time.Second)
+		}
+	}
+}
+
 func main() {
 	id := crypto.Id()
 
@@ -38,6 +68,28 @@ func main() {
 	)
 	defer cancel()
 
+	if cEnv.BackingStoreForSafe() == entity.Postgres {
+		go func() {
+			log.InfoLn(&id, "Backing store is postgres.")
+			log.InfoLn(&id, "VSecM Safe will remain read-only until the internal configuration is loaded.")
+
+			safeConfig, err := pollForConfig(ctx, id)
+			if err != nil {
+				log.FatalLn(&id, "Failed to retrieve VSecM Safe internal configuration", err.Error())
+			}
+
+			log.InfoLn(&id, "VSecM Safe internal configuration loaded. Initializing database.")
+
+			err = io.InitDB(safeConfig.Config.DataSourceName)
+			if err != nil {
+				log.FatalLn(&id, "Failed to initialize database:", err)
+				return
+			}
+
+			log.InfoLn(&id, "Database connection initialized.")
+		}()
+	}
+
 	log.InfoLn(&id, "Acquiring identity...")
 
 	// Channel to notify when the bootstrap timeout has been reached.

app/safe/internal/server/route/base/extract/extract.go

@@ -19,7 +19,7 @@ import (
 	log "github.com/vmware-tanzu/secrets-manager/core/log/std"
 )
 
-// WorkloadIDAndParts extracts the workload identifier and its constituent parts
+// WorkloadIdAndParts extracts the workload identifier and its constituent parts
 // from a SPIFFE ID string, based on a predefined prefix that is removed from
 // the SPIFFE ID.
 //
@@ -32,7 +32,7 @@ import (
 //     which is essentially the first part of the SPIFFE ID after removing the
 //     prefix. The second return value is a slice of strings representing all
 //     parts of the SPIFFE ID after the prefix removal.
-func WorkloadIDAndParts(spiffeid string) (string, []string) {
+func WorkloadIdAndParts(spiffeid string) (string, []string) {
 	re := env.NameRegExpForWorkload()
 	if re == "" {
 		return "", nil

app/safe/internal/server/route/fetch/fetch.go

@@ -88,7 +88,7 @@ func Fetch(
 
 	log.DebugLn(&cid, "Fetch: preparing request")
 
-	workloadId, parts := extract.WorkloadIDAndParts(spiffeid)
+	workloadId, parts := extract.WorkloadIdAndParts(spiffeid)
 	if len(parts) == 0 {
 		handle.BadPeerSvidResponse(cid, w, spiffeid, j)
 		return

app/safe/internal/state/io/disk.go

@@ -34,6 +34,10 @@ import (
 //     channel allows the function to operate asynchronously, notifying the
 //     caller of any issues in the process of persisting the secret.
 func PersistToDisk(secret entity.SecretStored, errChan chan<- error) {
+	if env.BackingStoreForSafe() != entity.File {
+		panic("Attempted to save to disk when backing store is not file")
+	}
+
 	backupCount := env.SecretBackupCountForSafe()
 
 	// Save the secret

app/safe/internal/state/io/postgres.go

@@ -0,0 +1,90 @@
+package io
+
+import (
+	"database/sql"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+
+	_ "github.com/lib/pq"
+
+	"github.com/vmware-tanzu/secrets-manager/core/crypto"
+	entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
+	"github.com/vmware-tanzu/secrets-manager/core/env"
+	log "github.com/vmware-tanzu/secrets-manager/core/log/std"
+	"github.com/vmware-tanzu/secrets-manager/lib/backoff"
+)
+
+var db *sql.DB
+
+// InitDB initializes the database connection
+func InitDB(dataSourceName string) error {
+	var err error
+	db, err = sql.Open("postgres", dataSourceName)
+	if err != nil {
+		return err
+	}
+	return db.Ping()
+}
+
+// PersistToPostgres saves a given secret to the Postgres database
+func PersistToPostgres(secret entity.SecretStored, errChan chan<- error) {
+	cid := secret.Meta.CorrelationId
+
+	log.TraceLn(&cid, "PersistToPostgres: Persisting secret to database")
+
+	// Serialize the secret to JSON
+	jsonData, err := json.Marshal(secret)
+	if err != nil {
+		errChan <- errors.Join(err, errors.New("PersistToPostgres: Failed to marshal secret"))
+		log.ErrorLn(&cid, "PersistToPostgres: Error marshaling secret:", err.Error())
+		return
+	}
+
+	// Encrypt the JSON data
+	var encryptedData string
+	fipsMode := env.FipsCompliantModeForSafe()
+
+	if fipsMode {
+		encryptedBytes, err := crypto.EncryptBytesAes(jsonData)
+		if err != nil {
+			errChan <- errors.Join(err, errors.New("PersistToPostgres: Failed to encrypt secret with AES"))
+			log.ErrorLn(&cid, "PersistToPostgres: Error encrypting secret with AES:", err.Error())
+			return
+		}
+		encryptedData = base64.StdEncoding.EncodeToString(encryptedBytes)
+	} else {
+		encryptedBytes, err := crypto.EncryptBytesAge(jsonData)
+		if err != nil {
+			errChan <- errors.Join(err, errors.New("PersistToPostgres: Failed to encrypt secret with Age"))
+			log.ErrorLn(&cid, "PersistToPostgres: Error encrypting secret with Age:", err.Error())
+			return
+		}
+		encryptedData = base64.StdEncoding.EncodeToString(encryptedBytes)
+	}
+
+	err = backoff.RetryExponential("PersistToPostgres", func() error {
+		if db == nil {
+			if secret.Name == "vsecm-safe" {
+				// TODO: implement me.
+				log.InfoLn(&cid, "PersistToPostgres: vsecm-safe secret will be persisted after db connection is initialized")
+				return nil
+			}
+			return errors.New("PersistToPostgres: Database connection is nil")
+		}
+
+		// TODO: get table name from env var.
+		_, err := db.Exec(
+			`INSERT INTO "vsecm-secrets" (name, data) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET data = $2`,
+			secret.Name, encryptedData)
+		return err
+	})
+
+	if err != nil {
+		errChan <- errors.Join(err, errors.New("PersistToPostgres: Failed to persist secret to database"))
+		log.ErrorLn(&cid, "PersistToPostgres: Error persisting secret to database:", err.Error())
+		return
+	}
+
+	log.TraceLn(&cid, "PersistToPostgres: Secret persisted to database successfully")
+}

app/safe/internal/state/io/read.go

@@ -13,6 +13,7 @@ package io
 import (
 	"encoding/json"
 	"errors"
+	"github.com/vmware-tanzu/secrets-manager/core/env"
 
 	"github.com/vmware-tanzu/secrets-manager/core/crypto"
 	entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
@@ -36,6 +37,10 @@ import (
 //     returned. The error provides context about the nature of the failure,
 //     such as issues with decryption or data deserialization.
 func ReadFromDisk(key string) (*entity.SecretStored, error) {
+	if env.BackingStoreForSafe() != entity.File {
+		panic("Attempted to read from disk when backing store is not file")
+	}
+
 	contents, err := crypto.DecryptDataFromDisk(key)
 	if err != nil {
 		return nil, errors.Join(

app/safe/internal/state/secret/collection/populate.go

@@ -72,6 +72,9 @@ func PopulateSecrets(cid string) error {
 		if err != nil {
 			log.ErrorLn(&cid, "populateSecrets:error", err.Error())
 		}
+	case data.Postgres:
+		// TODO: implement me.
+		log.WarnLn(&cid, "populateSecrets: postgres initial secrets population is not implemented yet.")
 	case data.Kubernetes:
 		panic("implement kubernetes store")
 	case data.AwsSecretStore:

app/safe/internal/state/secret/collection/read.go

@@ -15,6 +15,7 @@ import (
 	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/stats"
 	"github.com/vmware-tanzu/secrets-manager/core/crypto"
 	entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
+	"github.com/vmware-tanzu/secrets-manager/core/env"
 	log "github.com/vmware-tanzu/secrets-manager/core/log/std"
 	data "github.com/vmware-tanzu/secrets-manager/lib/entity"
 )
@@ -199,6 +200,16 @@ func ReadSecret(cid string, key string) (*entity.SecretStored, error) {
 		return &s, nil
 	}
 
+	store := env.BackingStoreForSafe()
+
+	switch store {
+	case entity.File:
+		log.TraceLn(&cid, "will read from file store.")
+	case entity.Postgres:
+		log.WarnLn(&cid, "TODO: fetch from postgres store")
+		return nil, nil
+	}
+
 	stored, err := io.ReadFromDisk(key)
 
 	if err != nil {

app/safe/internal/state/secret/queue/deletion/disk.go

@@ -69,7 +69,7 @@ func ProcessSecretBackingStoreQueue() {
 			log.TraceLn(&cid, "ProcessSecretQueue: using in-memory store.")
 			return
 		case entity.File:
-			log.TraceLn(&cid, "ProcessSecretQueue: Will persist to disk.")
+			log.TraceLn(&cid, "ProcessSecretQueue: Will delete secret from disk.")
 		case entity.Kubernetes:
 			panic("implement kubernetes store")
 		case entity.AwsSecretStore:
@@ -78,6 +78,9 @@ func ProcessSecretBackingStoreQueue() {
 			panic("implement azure secret store")
 		case entity.GcpSecretStore:
 			panic("implement gcp secret store")
+		case entity.Postgres:
+			log.WarnLn(&cid, "Delete operation has not been implemented for postgres backing store yet.")
+			return
 		}
 
 		if secret.Name == "" {

app/safe/internal/state/secret/queue/insertion/disk.go

@@ -73,8 +73,12 @@ func ProcessSecretBackingStoreQueue() {
 			panic("implement azure secret store")
 		case entity.GcpSecretStore:
 			panic("implement gcp secret store")
+		case entity.Postgres:
+			log.TraceLn(&cid, "ProcessSecretQueue: Will persist to Postgres.")
 		}
 
+		// TODO: will definitely need cleanup.
+
 		// Get a secret to be persisted to the disk.
 		secret := <-SecretUpsertQueue
 
@@ -93,7 +97,14 @@ func ProcessSecretBackingStoreQueue() {
 		//
 		// Do not call this function elsewhere.
 		// It is meant to be called inside this `processSecretQueue` goroutine.
-		io.PersistToDisk(secret, errChan)
+		if store == entity.Postgres {
+
+			// TODO: for debugging; delete values before merging.
+			log.TraceLn(&cid, "Persisting to Postgres.", secret.Name)
+			io.PersistToPostgres(secret, errChan)
+		} else {
+			io.PersistToDisk(secret, errChan)
+		}
 
 		log.TraceLn(&cid,
 			"processSecretQueue: should have persisted the secret.")

core/constants/env/env.go

@@ -37,6 +37,7 @@ const VSecMRootKeyInputModeManual VarName = "VSECM_ROOT_KEY_INPUT_MODE_MANUAL"
 const VSecMRootKeyName VarName = "VSECM_ROOT_KEY_NAME"
 const VSecMRootKeyPath VarName = "VSECM_ROOT_KEY_PATH"
 const VSecMSafeBackingStore VarName = "VSECM_SAFE_BACKING_STORE"
+const VSecMSafePostgresDataSourceName VarName = "VSECM_SAFE_POSTGRES_DATASOURCE_NAME"
 const VSecMSafeBootstrapTimeout VarName = "VSECM_SAFE_BOOTSTRAP_TIMEOUT"
 const VSecMSafeDataPath VarName = "VSECM_SAFE_DATA_PATH"
 const VSecMSafeEndpointUrl VarName = "VSECM_SAFE_ENDPOINT_URL"
@@ -125,6 +126,7 @@ const VSecMSpiffeIdPrefixSafeDefault VarValue = "^spiffe://vsecm.com/workload/vs
 const VSecMSpiffeIdPrefixSentinelDefault VarValue = "^spiffe://vsecm.com/workload/vsecm-sentinel/ns/vsecm-system/sa/vsecm-sentinel/n/[^/]+$"
 const VSecMSpiffeIdPrefixWorkloadDefault VarValue = "^spiffe://vsecm.com/workload/[^/]+/ns/[^/]+/sa/[^/]+/n/[^/]+$"
 const VSecMNameRegExpForWorkloadDefault VarValue = "^spiffe://vsecm.com/workload/([^/]+)/ns/[^/]+/sa/[^/]+/n/[^/]+$"
+const VSecMSafePostgresDataSourceNameDefault VarValue = "user=postgres dbname=postgres sslmode=disable"
 
 const VSecMRelayServerUrlDefault VarValue = "https://vsecm-relay.vsecm-system.svc.cluster.local:443/"