saving cosmosdb keys as secrets

api/v1alpha1/cosmosdb_types.go

@@ -17,10 +17,11 @@ type CosmosDBSpec struct {
 
 	// +kubebuilder:validation:MinLength=0
 
-	Location      string             `json:"location,omitempty"`
-	ResourceGroup string             `json:"resourceGroup"`
-	Kind          CosmosDBKind       `json:"kind,omitempty"`
-	Properties    CosmosDBProperties `json:"properties,omitempty"`
+	Location               string             `json:"location,omitempty"`
+	ResourceGroup          string             `json:"resourceGroup"`
+	Kind                   CosmosDBKind       `json:"kind,omitempty"`
+	Properties             CosmosDBProperties `json:"properties,omitempty"`
+	KeyVaultToStoreSecrets string             `json:"keyVaultToStoreSecrets,omitempty"`
 }
 
 // CosmosDBKind enumerates the values for kind.

config/samples/azure_v1alpha1_cosmosdb.yaml

@@ -8,3 +8,7 @@ spec:
   resourceGroup: resourcegroup-azure-operators
   properties:
     databaseAccountOfferType: Standard
+
+  # Use the field below to optionally specify a different keyvault 
+  # to store the connectiong string secrets in
+  #keyVaultToStoreSecrets: asoSecretKeyVault

controllers/cosmosdb_controller_test.go

@@ -10,22 +10,26 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-service-operator/api/v1alpha1"
+	"github.com/stretchr/testify/assert"
 
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 func TestCosmosDBHappyPath(t *testing.T) {
 	t.Parallel()
 	defer PanicRecover(t)
 	ctx := context.Background()
+	assert := assert.New(t)
 
-	cosmosDBAccountName := GenerateTestResourceNameWithRandom("cosmosdb", 8)
-	cosmosDBNamespace := "default"
+	name := GenerateTestResourceNameWithRandom("cosmosdb", 8)
+	namespace := "default"
 
 	dbInstance := &v1alpha1.CosmosDB{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      cosmosDBAccountName,
-			Namespace: cosmosDBNamespace,
+			Name:      name,
+			Namespace: namespace,
 		},
 		Spec: v1alpha1.CosmosDBSpec{
 			Location:      "westus",
@@ -37,8 +41,21 @@ func TestCosmosDBHappyPath(t *testing.T) {
 		},
 	}
 
+	key := types.NamespacedName{Name: name, Namespace: namespace}
+	secret := &v1.Secret{}
+
 	EnsureInstance(ctx, t, tc, dbInstance)
 
+	assert.Eventually(func() bool {
+		err := tc.k8sClient.Get(ctx, key, secret)
+		return err == nil && secret.ObjectMeta.Name == name && secret.ObjectMeta.Namespace == namespace
+	}, tc.timeoutFast, tc.retry, "wait for cosmosdb to have secret")
+
 	EnsureDelete(ctx, t, tc, dbInstance)
 
+	assert.Eventually(func() bool {
+		err := tc.k8sClient.Get(ctx, key, secret)
+		return err != nil
+	}, tc.timeoutFast, tc.retry, "wait for cosmosdb to delete secret")
+
 }

controllers/suite_test.go

@@ -154,7 +154,7 @@ func setup() error {
 		secretClient,
 		scheme.Scheme,
 	)
-	cosmosDbManager = resourcemanagercosmosdb.NewAzureCosmosDBManager()
+	cosmosDbManager = resourcemanagercosmosdb.NewAzureCosmosDBManager(secretClient)
 	apiMgmtManager = resourcemanagerapimgmt.NewManager()
 	resourceGroupManager = resourcegroupsresourcemanager.NewAzureResourceGroupManager()
 	eventHubManagers = resourcemanagereventhub.AzureEventHubManagers

main.go

@@ -126,6 +126,9 @@ func main() {
 	)
 	eventhubNamespaceClient := resourcemanagereventhub.NewEventHubNamespaceClient()
 	consumerGroupClient := resourcemanagereventhub.NewConsumerGroupClient()
+	cosmosDBClient := resourcemanagercosmosdb.NewAzureCosmosDBManager(
+		secretClient,
+	)
 	storageManagers := resourcemanagerstorage.AzureStorageManagers
 	keyVaultManager := resourcemanagerkeyvault.NewAzureKeyVaultManager(mgr.GetScheme())
 	keyVaultKeyManager := &resourcemanagerkeyvault.KeyvaultKeyClient{
@@ -171,7 +174,7 @@ func main() {
 	err = (&controllers.CosmosDBReconciler{
 		Reconciler: &controllers.AsyncReconciler{
 			Client:      mgr.GetClient(),
-			AzureClient: resourcemanagercosmosdb.NewAzureCosmosDBManager(),
+			AzureClient: cosmosDBClient,
 			Telemetry: telemetry.InitializeTelemetryDefault(
 				"CosmosDB",
 				ctrl.Log.WithName("controllers").WithName("CosmosDB"),

pkg/resourcemanager/cosmosdbs/cosmosdb.go

@@ -13,12 +13,15 @@ import (
 	"github.com/Azure/azure-service-operator/pkg/errhelp"
 	"github.com/Azure/azure-service-operator/pkg/resourcemanager/config"
 	"github.com/Azure/azure-service-operator/pkg/resourcemanager/iam"
+	"github.com/Azure/azure-service-operator/pkg/secrets"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/to"
 )
 
 // AzureCosmosDBManager is the struct which contains helper functions for resource groups
-type AzureCosmosDBManager struct{}
+type AzureCosmosDBManager struct {
+	SecretClient secrets.SecretClient
+}
 
 func getCosmosDBClient() (documentdb.DatabaseAccountsClient, error) {
 	cosmosDBClient := documentdb.NewDatabaseAccountsClientWithBaseURI(config.BaseURI(), config.SubscriptionID())
@@ -164,3 +167,21 @@ func (*AzureCosmosDBManager) DeleteCosmosDB(
 	}
 	return &ar, nil
 }
+
+// ListKeys lists the read & write keys for a database account
+func (*AzureCosmosDBManager) ListKeys(
+	ctx context.Context,
+	groupName string,
+	accountName string) (*documentdb.DatabaseAccountListKeysResult, error) {
+	client, err := getCosmosDBClient()
+	if err != nil {
+		return nil, errhelp.NewAzureErrorAzureError(err)
+	}
+
+	result, err := client.ListKeys(ctx, groupName, accountName)
+	if err != nil {
+		return nil, errhelp.NewAzureErrorAzureError(err)
+	}
+
+	return &result, nil
+}

pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go

@@ -10,12 +10,13 @@ import (
 	"github.com/Azure/azure-service-operator/api/v1alpha1"
 	"github.com/Azure/azure-service-operator/pkg/errhelp"
 	"github.com/Azure/azure-service-operator/pkg/resourcemanager"
+	"github.com/Azure/azure-service-operator/pkg/secrets"
 	"github.com/Azure/go-autorest/autorest"
 )
 
 // NewAzureCosmosDBManager creates a new cosmos db client
-func NewAzureCosmosDBManager() *AzureCosmosDBManager {
-	return &AzureCosmosDBManager{}
+func NewAzureCosmosDBManager(secretClient secrets.SecretClient) *AzureCosmosDBManager {
+	return &AzureCosmosDBManager{secretClient}
 }
 
 // CosmosDBManager client functions
@@ -32,5 +33,8 @@ type CosmosDBManager interface {
 	// CheckNameExistsCosmosDB check if the account name already exists globally
 	CheckNameExistsCosmosDB(ctx context.Context, accountName string) (bool, *errhelp.AzureError)
 
+	// ListKeys lists the read & write keys for a database account
+	ListKeys(ctx context.Context, groupName string, accountName string) (*documentdb.DatabaseAccountListKeysResult, error)
+
 	resourcemanager.ARMClient
 }

pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go

@@ -18,6 +18,15 @@ import (
 
 // Ensure ensures that cosmosdb is provisioned as specified
 func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) {
+	options := &resourcemanager.Options{}
+	for _, opt := range opts {
+		opt(options)
+	}
+
+	if options.SecretClient != nil {
+		m.SecretClient = options.SecretClient
+	}
+
 	instance, err := m.convert(obj)
 	if err != nil {
 		return false, err
@@ -44,6 +53,12 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o
 			}
 
 			if instance.Status.State == "Succeeded" {
+				// provisioning is complete, update the secrets
+				if err = m.createOrUpdateAccountKeysSecret(ctx, instance); err != nil {
+					instance.Status.Message = err.Error()
+					return false, err
+				}
+
 				instance.Status.Message = resourcemanager.SuccessMsg
 				instance.Status.Provisioning = false
 				instance.Status.Provisioned = true
@@ -120,6 +135,15 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o
 
 // Delete drops cosmosdb
 func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) {
+	options := &resourcemanager.Options{}
+	for _, opt := range opts {
+		opt(options)
+	}
+
+	if options.SecretClient != nil {
+		m.SecretClient = options.SecretClient
+	}
+
 	instance, err := m.convert(obj)
 	if err != nil {
 		return false, err
@@ -160,8 +184,14 @@ func (m *AzureCosmosDBManager) Delete(ctx context.Context, obj runtime.Object, o
 		return true, nil
 	}
 
-	// try to delete the cosmosdb instance
+	// create must succeed before delete succeeds
+	if instance.Status.State == "Creating" {
+		return true, nil
+	}
+
+	// try to delete the cosmosdb instance & secrets
 	_, azerr = m.DeleteCosmosDB(ctx, groupName, accountName)
+	m.deleteAccountKeysSecret(ctx, instance)
 	if azerr != nil {
 		// this is likely to happen on first try due to not waiting for the future to complete
 		if azerr.Type == errhelp.AsyncOpIncompleteError {
@@ -217,3 +247,31 @@ func (m *AzureCosmosDBManager) convert(obj runtime.Object) (*v1alpha1.CosmosDB,
 	}
 	return db, nil
 }
+
+func (m *AzureCosmosDBManager) createOrUpdateAccountKeysSecret(ctx context.Context, instance *v1alpha1.CosmosDB) error {
+	result, err := m.ListKeys(ctx, instance.Spec.ResourceGroup, instance.ObjectMeta.Name)
+	if err != nil {
+		return err
+	}
+
+	secretKey := types.NamespacedName{
+		Name:      instance.Name,
+		Namespace: instance.Namespace,
+	}
+	secretData := map[string][]byte{
+		"primaryConnectionString":    []byte(*result.PrimaryMasterKey),
+		"secondaryConnectionString":  []byte(*result.SecondaryMasterKey),
+		"primaryReadonlyMasterKey":   []byte(*result.PrimaryReadonlyMasterKey),
+		"secondaryReadonlyMasterKey": []byte(*result.SecondaryReadonlyMasterKey),
+	}
+
+	return m.SecretClient.Upsert(ctx, secretKey, secretData)
+}
+
+func (m *AzureCosmosDBManager) deleteAccountKeysSecret(ctx context.Context, instance *v1alpha1.CosmosDB) error {
+	secretKey := types.NamespacedName{
+		Name:      instance.Name,
+		Namespace: instance.Namespace,
+	}
+	return m.SecretClient.Delete(ctx, secretKey)
+}