Merge branch 'master' into cosmosdb-async-controller

Makefile

@@ -71,7 +71,9 @@ test-existing-managers: generate fmt vet manifests
 	./pkg/resourcemanager/psql/firewallrule/... \
 	./pkg/resourcemanager/appinsights/... \
 	./pkg/resourcemanager/vnet/... \
-	./pkg/resourcemanager/apim/apimgmt...
+	./pkg/resourcemanager/apim/apimgmt... \
+	./pkg/secrets/...
+
 
 # Cleanup resource groups azure created by tests using pattern matching 't-rg-'
 test-cleanup-azure-resources: 	

README.md

@@ -32,8 +32,17 @@ This project maintains [releases of the Azure Service Operator](https://github.c
 1. [Resource Group](/docs/resourcegroup/resourcegroup.md)
 2. [EventHub](/docs/eventhub/eventhub.md)
 3. [Azure SQL](/docs/azuresql/azuresql.md)
-
-For more information on troubleshooting resources, refer to [this](/docs/viewresources.md) link
+4. [Azure Keyvault](/docs/keyvault/keyvault.md)
+5. [Azure Rediscache](/docs/rediscache/rediscache.md)
+6. [Storage Account](/docs/storage/storageaccount.md)
+7. [Blob container](/docs/storage/blobcontainer.md)
+8. [Azure Database for PostgreSQL](/docs/postgresql/postgresql.md)
+9. [Virtual Network](/docs/virtualnetwork/virtualnetwork.md)
+10.[Application Insights](/docs/appinsights/appinsights.md)
+11.[API Management](/docs/apimgmt/apimgmt.md)
+12.[Cosmos DB](/docs/cosmosdb/cosmosdb.md)
+
+For more information on deploying, troubleshooting & deleting resources, refer to [this](/docs/customresource.md) link
 
 ## Building the operators
 

api/v1alpha1/keyvault_types.go

@@ -14,6 +14,7 @@ type KeyVaultSpec struct {
 	EnableSoftDelete bool                 `json:"enableSoftDelete,omitempty"`
 	NetworkPolicies  *NetworkRuleSet      `json:"networkPolicies,omitempty"`
 	AccessPolicies   *[]AccessPolicyEntry `json:"accessPolicies,omitempty"`
+	Sku              KeyVaultSku          `json:"sku,omitempty"`
 }
 
 type NetworkRuleSet struct {
@@ -38,6 +39,13 @@ type AccessPolicyEntry struct {
 	Permissions *Permissions `json:"permissions,omitempty"`
 }
 
+// KeyVaultSku the SKU of the Key Vault
+type KeyVaultSku struct {
+	// Name - The SKU name. Required for account creation; optional for update.
+	// Possible values include: 'Premium', `Standard`
+	Name string `json:"name,omitempty"`
+}
+
 type Permissions struct {
 	Keys         *[]string `json:"keys,omitempty"`
 	Secrets      *[]string `json:"secrets,omitempty"`

config/samples/azure_v1alpha1_keyvault.yaml

@@ -9,6 +9,9 @@ spec:
   resourceGroup: resourcegroup-azure-operators
   location: westus
   enableSoftDelete: false
+  # possible values for sku.Name are "Standard" or "Premium"  
+  sku:
+    name: standard
   networkPolicies: 
     bypass: AzureServices # AzureServices or None
     defaultAction: Allow # Allow or Deny

config/samples/azure_v1alpha1_keyvault_simple.yaml

@@ -9,3 +9,6 @@ spec:
   resourceGroup: resourcegroup-azure-operators
   location: westus
   enableSoftDelete: false
+  # Optional: possible values for sku.Name are "Standard" or "Premium". Default is "Standard"  
+  #sku:
+  #  name: standard

config/samples/azure_v1alpha1_postgresql_everything.yaml

@@ -0,0 +1,36 @@
+apiVersion: azure.microsoft.com/v1alpha1
+kind: PostgreSQLServer
+metadata:
+  name: postgresqlserver-sample
+spec:
+  location: southcentralus
+  resourceGroup: resourcegroup-azure-operators
+  serverVersion: "10"
+  sslEnforcement: Enabled
+  sku:
+    name: B_Gen5_2
+    tier: Basic
+    family: Gen5
+    size: "51200"
+    capacity: 2
+  # Use the field below to optionally specify a different keyvault 
+  # to store the server admin credential secrets in
+  #keyVaultToStoreSecrets: asoSecretKeyVault
+---
+apiVersion: azure.microsoft.com/v1alpha1
+kind: PostgreSQLDatabase
+metadata:
+  name: postgresqldatabase-sample
+spec:
+  resourceGroup: resourcegroup-azure-operators
+  server: postgresqlserver-sample
+---
+apiVersion: azure.microsoft.com/v1alpha1
+kind: PostgreSQLFirewallRule
+metadata:
+  name: postgresqlfirewallrule-sample
+spec:
+  resourceGroup: resourcegroup-azure-operators
+  server: postgresqlserver-sample
+  startIpAddress: 0.0.0.0
+  endIpAddress: 0.0.0.0

controllers/azuresql_combined_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	helpers "github.com/Azure/azure-service-operator/pkg/helpers"
+	kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -148,6 +149,8 @@ func TestAzureSqlServerCombinedHappyPath(t *testing.T) {
 	})
 
 	var sqlUser *azurev1alpha1.AzureSQLUser
+	var kvSqlUser1 *azurev1alpha1.AzureSQLUser
+	var kvSqlUser2 *azurev1alpha1.AzureSQLUser
 
 	// run sub tests that require 2 servers or have to be run after rollcreds test ------------------
 	t.Run("group2", func(t *testing.T) {
@@ -158,23 +161,119 @@ func TestAzureSqlServerCombinedHappyPath(t *testing.T) {
 			// create a sql user and verify it provisions
 			username := "sql-test-user" + helpers.RandomString(10)
 			roles := []string{"db_owner"}
+			keyVaultSecretFormats := []string{"adonet"}
 
 			sqlUser = &azurev1alpha1.AzureSQLUser{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      username,
 					Namespace: "default",
 				},
 				Spec: azurev1alpha1.AzureSQLUserSpec{
-					Server:        sqlServerName,
-					DbName:        sqlDatabaseName,
-					ResourceGroup: rgName,
-					Roles:         roles,
+					Server:                sqlServerName,
+					DbName:                sqlDatabaseName,
+					ResourceGroup:         rgName,
+					Roles:                 roles,
+					KeyVaultSecretFormats: keyVaultSecretFormats,
 				},
 			}
 
 			EnsureInstance(ctx, t, tc, sqlUser)
+
+			// verify user's secret has been created
+			// this test suite defaults to Kube Secrets. They do not support keyvault-specific config but the spec is passed anyway
+			// to verify that passing them does not break the service
+			assert.Eventually(func() bool {
+				key := types.NamespacedName{Name: sqlUser.ObjectMeta.Name, Namespace: sqlUser.ObjectMeta.Namespace}
+				var secrets, _ = tc.secretClient.Get(ctx, key)
+
+				return strings.Contains(string(secrets["azureSqlDatabaseName"]), sqlDatabaseName)
+			}, tc.timeoutFast, tc.retry, "wait for secret store to show azure sql user credentials")
+
 			t.Log(sqlUser.Status)
 		})
+
+		t.Run("set up user in first db with custom keyvault", func(t *testing.T) {
+			t.Parallel()
+
+			// create a sql user and verify it provisions
+			username := "sql-test-user" + helpers.RandomString(10)
+			roles := []string{"db_owner"}
+
+			// This test will attempt to persist secrets to the KV that was instantiated as part of the test suite
+			keyVaultName := tc.keyvaultName
+
+			kvSqlUser1 = &azurev1alpha1.AzureSQLUser{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      username,
+					Namespace: "default",
+				},
+				Spec: azurev1alpha1.AzureSQLUserSpec{
+					Server:                 sqlServerName,
+					DbName:                 sqlDatabaseName,
+					ResourceGroup:          rgName,
+					Roles:                  roles,
+					KeyVaultToStoreSecrets: keyVaultName,
+				},
+			}
+
+			EnsureInstance(ctx, t, tc, kvSqlUser1)
+
+			// Check that the user's secret is in the keyvault
+			keyVaultSecretClient := kvsecrets.New(keyVaultName)
+
+			assert.Eventually(func() bool {
+				keyNamespace := "azuresqluser-" + sqlServerName + "-" + sqlDatabaseName
+				key := types.NamespacedName{Name: kvSqlUser1.ObjectMeta.Name, Namespace: keyNamespace}
+				var secrets, _ = keyVaultSecretClient.Get(ctx, key)
+
+				return strings.Contains(string(secrets["azureSqlDatabaseName"]), sqlDatabaseName)
+			}, tc.timeoutFast, tc.retry, "wait for keyvault to show azure sql user credentials")
+
+			t.Log(kvSqlUser1.Status)
+		})
+
+		t.Run("set up user in first db with custom keyvault and custom formatting", func(t *testing.T) {
+			t.Parallel()
+
+			// create a sql user and verify it provisions
+			username := "sql-test-user" + helpers.RandomString(10)
+			roles := []string{"db_owner"}
+			formats := []string{"adonet"}
+
+			// This test will attempt to persist secrets to the KV that was instantiated as part of the test suite
+			keyVaultName := tc.keyvaultName
+
+			kvSqlUser2 = &azurev1alpha1.AzureSQLUser{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      username,
+					Namespace: "default",
+				},
+				Spec: azurev1alpha1.AzureSQLUserSpec{
+					Server:                 sqlServerName,
+					DbName:                 sqlDatabaseName,
+					ResourceGroup:          rgName,
+					Roles:                  roles,
+					KeyVaultToStoreSecrets: keyVaultName,
+					KeyVaultSecretFormats:  formats,
+				},
+			}
+
+			EnsureInstance(ctx, t, tc, kvSqlUser2)
+
+			// Check that the user's secret is in the keyvault
+			keyVaultSecretClient := kvsecrets.New(keyVaultName)
+
+			assert.Eventually(func() bool {
+				keyNamespace := "azuresqluser-" + sqlServerName + "-" + sqlDatabaseName
+				keyName := kvSqlUser2.ObjectMeta.Name + "-adonet"
+				key := types.NamespacedName{Name: keyName, Namespace: keyNamespace}
+				var secrets, _ = keyVaultSecretClient.Get(ctx, key)
+
+				return len(string(secrets[keyNamespace+"-"+keyName])) > 0
+			}, tc.timeoutFast, tc.retry, "wait for keyvault to show azure sql user credentials with custom formats")
+
+			t.Log(kvSqlUser2.Status)
+		})
 	})
 
 	var sqlFailoverGroupInstance *azurev1alpha1.AzureSqlFailoverGroup
@@ -183,8 +282,40 @@ func TestAzureSqlServerCombinedHappyPath(t *testing.T) {
 	sqlFailoverGroupNamespacedName := types.NamespacedName{Name: sqlFailoverGroupName, Namespace: "default"}
 
 	t.Run("group3", func(t *testing.T) {
-		t.Run("delete db user", func(t *testing.T) {
+		t.Run("delete db users and ensure that their secrets have been cleaned up", func(t *testing.T) {
 			EnsureDelete(ctx, t, tc, sqlUser)
+			EnsureDelete(ctx, t, tc, kvSqlUser1)
+			EnsureDelete(ctx, t, tc, kvSqlUser2)
+
+			// Check that the user's secret is in the keyvault
+			keyVaultSecretClient := kvsecrets.New(tc.keyvaultName)
+
+			assert.Eventually(func() bool {
+				key := types.NamespacedName{Name: sqlUser.ObjectMeta.Name, Namespace: sqlUser.ObjectMeta.Namespace}
+				var _, err = tc.secretClient.Get(ctx, key)
+
+				// Once the secret is gone, the Kube secret client will return an error
+				return err != nil && strings.Contains(err.Error(), "not found")
+			}, tc.timeoutFast, tc.retry, "wait for the azuresqluser kube secret to be deleted")
+
+			assert.Eventually(func() bool {
+				keyNamespace := "azuresqluser-" + sqlServerName + "-" + sqlDatabaseName
+				key := types.NamespacedName{Name: kvSqlUser1.ObjectMeta.Name, Namespace: keyNamespace}
+				var _, err = keyVaultSecretClient.Get(ctx, key)
+
+				// Once the secret is gone, the KV secret client will return an err
+				return err != nil && strings.Contains(err.Error(), "secret does not exist")
+			}, tc.timeoutFast, tc.retry, "wait for the azuresqluser keyvault secret to be deleted")
+
+			assert.Eventually(func() bool {
+				keyNamespace := "azuresqluser-" + sqlServerName + "-" + sqlDatabaseName
+				keyName := kvSqlUser2.ObjectMeta.Name + "-adonet"
+				key := types.NamespacedName{Name: keyName, Namespace: keyNamespace}
+				var _, err = keyVaultSecretClient.Get(ctx, key)
+
+				// Once the secret is gone, the KV secret client will return an err
+				return err != nil && strings.Contains(err.Error(), "secret does not exist")
+			}, tc.timeoutFast, tc.retry, "wait for the azuresqluser custom formatted keyvault secret to be deleted")
 		})
 
 		t.Run("delete local firewallrule", func(t *testing.T) {

controllers/eventhub_controller_test.go

@@ -13,8 +13,6 @@ import (
 	azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1"
 	"github.com/Azure/azure-service-operator/pkg/errhelp"
 
-	helpers "github.com/Azure/azure-service-operator/pkg/helpers"
-	"github.com/Azure/azure-service-operator/pkg/resourcemanager/config"
 	kvhelper "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults"
 	kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault"
 
@@ -192,13 +190,10 @@ func TestEventHubControllerCreateAndDeleteCustomKeyVault(t *testing.T) {
 	rgLocation := tc.resourceGroupLocation
 	ehnName := tc.eventhubNamespaceName
 	eventhubName := GenerateTestResourceNameWithRandom("ev", 10)
-	keyVaultNameForSecrets := helpers.FillWithRandom(GenerateTestResourceName("ev-kv"), 24)
-	userID := config.ClientID()
+	keyVaultNameForSecrets := tc.keyvaultName
 
-	// Create KeyVault with access policies
-	_, err := kvhelper.AzureKeyVaultManager.CreateVaultWithAccessPolicies(ctx, rgName, keyVaultNameForSecrets, rgLocation, userID)
-
-	_, err = kvhelper.AzureKeyVaultManager.GetVault(ctx, rgName, keyVaultNameForSecrets)
+	// Instantiate a KV client for the Keyvault that was created during test suite setup
+	_, err := kvhelper.AzureKeyVaultManager.GetVault(ctx, rgName, keyVaultNameForSecrets)
 	assert.Equal(nil, err, "wait for keyvault to be available")
 
 	// Create the EventHub object and expect the Reconcile to be created

controllers/helpers.go

@@ -52,6 +52,7 @@ type TestContext struct {
 	namespaceLocation       string
 	storageAccountName      string
 	blobContainerName       string
+	keyvaultName            string
 	resourceGroupManager    resourcegroupsresourcemanager.ResourceGroupManager
 	redisCacheManager       resourcemanagerrediscaches.RedisCacheManager
 	eventHubManagers        resourcemanagereventhub.EventHubManagers

controllers/suite_test.go

@@ -76,6 +76,8 @@ func setup() error {
 	blobContainerName := GenerateTestResourceName("blob-prime")
 	containerAccessLevel := s.PublicAccessContainer
 
+	keyvaultName := GenerateAlphaNumTestResourceName("kv-prime")
+
 	var timeout time.Duration
 
 	testEnv = &envtest.Environment{
@@ -171,7 +173,6 @@ func setup() error {
 		scheme.Scheme,
 	)
 	redisCacheManager := resourcemanagerrediscaches.NewAzureRedisCacheManager(
-		ctrl.Log.WithName("rediscachemanager").WithName("RedisCache"),
 		secretClient,
 		scheme.Scheme,
 	)
@@ -189,7 +190,7 @@ func setup() error {
 	)
 	sqlActionManager = resourcemanagersqlaction.NewAzureSqlActionManager(secretClient, scheme.Scheme)
 
-	timeout = time.Second * 720
+	timeout = time.Second * 780
 
 	err = (&KeyVaultReconciler{
 		Reconciler: &AsyncReconciler{
@@ -607,6 +608,7 @@ func setup() error {
 		namespaceLocation:       namespaceLocation,
 		storageAccountName:      storageAccountName,
 		blobContainerName:       blobContainerName,
+		keyvaultName:            keyvaultName,
 		eventHubManagers:        eventHubManagers,
 		eventhubClient:          eventhubClient,
 		resourceGroupManager:    resourceGroupManager,
@@ -673,6 +675,12 @@ func setup() error {
 		return err
 	}
 
+	log.Println("Creating KV:", keyvaultName)
+	_, err = resourcemanagerkeyvaults.AzureKeyVaultManager.CreateVaultWithAccessPolicies(context.Background(), resourceGroupName, keyvaultName, resourcegroupLocation, resourcemanagerconfig.ClientID())
+	if err != nil {
+		return err
+	}
+
 	log.Println(fmt.Sprintf("finished common controller test setup"))
 
 	return nil

devops/azure-pipelines.yaml

@@ -150,7 +150,7 @@ steps:
   env:
     BUILD_ID: $(Build.BuildId)
   inputs:
-    azureSubscription: 'Az Internal Consumption'
+    azureSubscription: 'CSE AmWest Sub - ASO'
     scriptType: bash
     scriptLocation: inlineScript
     workingDirectory: '$(MODULE_PATH)'