Merge branch 'master' into cloud-suffix

config/samples/azure_v1alpha1_blobcontainer.yaml

@@ -4,8 +4,8 @@ metadata:
   name: blobcontainer-sample
 spec:
   location: westus
-  resourcegroup: resourcegroup-sample
-  accountname: storageaccount-sample
+  resourcegroup: resourcegroup-azure-operators
+  accountname: storageaccountsample777
   # accessLevel - Specifies whether data in the container may be accessed publicly and the level of access.
   # Possible values include: 'Container', 'Blob', 'None'
   accesslevel: Container

controllers/cosmosdb_controller_test.go

@@ -28,7 +28,7 @@ func TestCosmosDBHappyPath(t *testing.T) {
 			Namespace: cosmosDBNamespace,
 		},
 		Spec: v1alpha1.CosmosDBSpec{
-			Location:      "westus",
+			Location:      tc.resourceGroupLocation,
 			ResourceGroup: tc.resourceGroupName,
 			Kind:          v1alpha1.CosmosDBKindGlobalDocumentDB,
 			Properties: v1alpha1.CosmosDBProperties{
@@ -42,3 +42,70 @@ func TestCosmosDBHappyPath(t *testing.T) {
 	EnsureDelete(ctx, t, tc, dbInstance)
 
 }
+
+func TestCosmosDBControllerNoResourceGroup(t *testing.T) {
+	t.Parallel()
+	defer PanicRecover(t)
+	ctx := context.Background()
+
+	rgLocation := tc.resourceGroupLocation
+	//wrong resource group name
+	resourceGroupName := "gone"
+
+	cosmosDBAccountName := GenerateTestResourceNameWithRandom("cosmosdb", 8)
+	cosmosDBNamespace := "default"
+
+	dbInstance1 := &v1alpha1.CosmosDB{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cosmosDBAccountName,
+			Namespace: cosmosDBNamespace,
+		},
+		Spec: v1alpha1.CosmosDBSpec{
+			Location:      rgLocation,
+			ResourceGroup: resourceGroupName,
+			Kind:          v1alpha1.CosmosDBKindGlobalDocumentDB,
+			Properties: v1alpha1.CosmosDBProperties{
+				DatabaseAccountOfferType: v1alpha1.CosmosDBDatabaseAccountOfferTypeStandard,
+			},
+		},
+	}
+	//the expected error meessage to be shown
+	errMessage := "Waiting for resource group '" + resourceGroupName + "' to be available"
+
+	EnsureInstanceWithResult(ctx, t, tc, dbInstance1, errMessage, false)
+	EnsureDelete(ctx, t, tc, dbInstance1)
+}
+
+func TestCosmosDBControllerInvalidLocation(t *testing.T) {
+	t.Parallel()
+	defer PanicRecover(t)
+	ctx := context.Background()
+
+	resourceGroupName := tc.resourceGroupName
+	//rglocation doesnot exist
+	rgLocation := GenerateTestResourceNameWithRandom("cosmos-lo", 10)
+
+	cosmosDBAccountName := GenerateTestResourceNameWithRandom("cosmos-db", 8)
+	cosmosDBNamespace := "default"
+
+	dbInstance2 := &v1alpha1.CosmosDB{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cosmosDBAccountName,
+			Namespace: cosmosDBNamespace,
+		},
+		Spec: v1alpha1.CosmosDBSpec{
+			Location:      rgLocation,
+			ResourceGroup: resourceGroupName,
+			Kind:          v1alpha1.CosmosDBKindGlobalDocumentDB,
+			Properties: v1alpha1.CosmosDBProperties{
+				DatabaseAccountOfferType: v1alpha1.CosmosDBDatabaseAccountOfferTypeStandard,
+			},
+		},
+	}
+
+	//error meessage to be expected
+	errMessage := "The specified location '" + rgLocation + "' is invalid"
+
+	EnsureInstanceWithResult(ctx, t, tc, dbInstance2, errMessage, false)
+	EnsureDelete(ctx, t, tc, dbInstance2)
+}

controllers/helpers.go

@@ -247,6 +247,15 @@ func EnsureInstanceWithResult(ctx context.Context, t *testing.T, tc TestContext,
 		statused := ConvertToStatus(instance)
 		// if we expect this resource to end up with provisioned == true then failedProvisioning == true is unrecoverable
 		if provisioned && statused.Status.FailedProvisioning {
+			if strings.Contains(statused.Status.Message, "already exists") || strings.Contains(statused.Status.Message, "AlreadyExists") {
+				t.Log("")
+				t.Log("-------")
+				t.Log("unexpected failed provisioning encountered")
+				t.Logf("%+v\n", statused.Status)
+				t.Logf("current time %v\n", time.Now())
+				t.Log("-------")
+				t.Log("")
+			}
 			return helpers.NewStop(fmt.Errorf("Failed provisioning: %s", statused.Status.Message))
 		}
 		if !strings.Contains(statused.Status.Message, message) || statused.Status.Provisioned != provisioned {

controllers/postgresql_combined_controller_test.go

@@ -10,24 +10,18 @@ import (
 	"testing"
 
 	azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1"
-	"github.com/stretchr/testify/assert"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
 )
 
 func TestPSQLDatabaseController(t *testing.T) {
 	t.Parallel()
 	defer PanicRecover(t)
 	ctx := context.Background()
-	assert := assert.New(t)
 
 	var rgName string
 	var rgLocation string
 	var postgreSQLServerName string
 	var postgreSQLServerInstance *azurev1alpha1.PostgreSQLServer
-	var postgreSQLServerNamespacedName types.NamespacedName
-	var err error
 
 	// Add any setup steps that needs to be executed before each test
 	rgName = tc.resourceGroupName
@@ -56,20 +50,7 @@ func TestPSQLDatabaseController(t *testing.T) {
 		},
 	}
 
-	err = tc.k8sClient.Create(ctx, postgreSQLServerInstance)
-	assert.Equal(nil, err, "create postgreSQLServerInstance in k8s")
-
-	postgreSQLServerNamespacedName = types.NamespacedName{Name: postgreSQLServerName, Namespace: "default"}
-
-	assert.Eventually(func() bool {
-		_ = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance)
-		return HasFinalizer(postgreSQLServerInstance, finalizerName)
-	}, tc.timeout, tc.retry, "wait for postgreSQLserver to have finlizer")
-
-	assert.Eventually(func() bool {
-		_ = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance)
-		return postgreSQLServerInstance.Status.Provisioned
-	}, tc.timeout, tc.retry, "wait for postgreSQLserver to be provisioned")
+	EnsureInstance(ctx, t, tc, postgreSQLServerInstance)
 
 	postgreSQLDatabaseName := GenerateTestResourceNameWithRandom("psql-db", 10)
 
@@ -85,28 +66,9 @@ func TestPSQLDatabaseController(t *testing.T) {
 		},
 	}
 
-	err = tc.k8sClient.Create(ctx, postgreSQLDatabaseInstance)
-	assert.Equal(nil, err, "create postgreSQLDatabaseInstance in k8s")
-
-	postgreSQLDatabaseNamespacedName := types.NamespacedName{Name: postgreSQLDatabaseName, Namespace: "default"}
-
-	assert.Eventually(func() bool {
-		_ = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance)
-		return HasFinalizer(postgreSQLDatabaseInstance, finalizerName)
-	}, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to have finalizer")
-
-	assert.Eventually(func() bool {
-		_ = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance)
-		return postgreSQLDatabaseInstance.Status.Provisioned
-	}, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to be provisioned")
+	EnsureInstance(ctx, t, tc, postgreSQLDatabaseInstance)
 
-	err = tc.k8sClient.Delete(ctx, postgreSQLDatabaseInstance)
-	assert.Equal(nil, err, "delete postgreSQLDatabaseInstance in k8s")
-
-	assert.Eventually(func() bool {
-		err = tc.k8sClient.Get(ctx, postgreSQLDatabaseNamespacedName, postgreSQLDatabaseInstance)
-		return apierrors.IsNotFound(err)
-	}, tc.timeout, tc.retry, "wait for postgreSQLDBInstance to be gone from k8s")
+	EnsureDelete(ctx, t, tc, postgreSQLDatabaseInstance)
 
 	// Test firewall rule -------------------------------
 
@@ -126,36 +88,11 @@ func TestPSQLDatabaseController(t *testing.T) {
 		},
 	}
 
-	err = tc.k8sClient.Create(ctx, postgreSQLFirewallRuleInstance)
-	assert.Equal(nil, err, "create postgreSQLFirewallRuleInstance in k8s")
-
-	postgreSQLFirewallRuleNamespacedName := types.NamespacedName{Name: postgreSQLFirewallRuleName, Namespace: "default"}
-
-	assert.Eventually(func() bool {
-		_ = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance)
-		return HasFinalizer(postgreSQLFirewallRuleInstance, finalizerName)
-	}, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to have finalizer")
+	EnsureInstance(ctx, t, tc, postgreSQLFirewallRuleInstance)
 
-	assert.Eventually(func() bool {
-		_ = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance)
-		return postgreSQLFirewallRuleInstance.Status.Provisioned
-	}, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to be provisioned")
-
-	err = tc.k8sClient.Delete(ctx, postgreSQLFirewallRuleInstance)
-	assert.Equal(nil, err, "delete postgreSQLFirewallRuleInstance in k8s")
-
-	assert.Eventually(func() bool {
-		err = tc.k8sClient.Get(ctx, postgreSQLFirewallRuleNamespacedName, postgreSQLFirewallRuleInstance)
-		return apierrors.IsNotFound(err)
-	}, tc.timeout, tc.retry, "wait for postgreSQLFirewallRuleInstance to be gone from k8s")
+	EnsureDelete(ctx, t, tc, postgreSQLFirewallRuleInstance)
 
 	// Add any teardown steps that needs to be executed after each test
-	err = tc.k8sClient.Delete(ctx, postgreSQLServerInstance)
-	assert.Equal(nil, err, "delete postgreSQLServerInstance in k8s")
-
-	assert.Eventually(func() bool {
-		err = tc.k8sClient.Get(ctx, postgreSQLServerNamespacedName, postgreSQLServerInstance)
-		return apierrors.IsNotFound(err)
-	}, tc.timeout, tc.retry, "wait for postgreSQLServerInstance to be gone from k8s")
+	EnsureDelete(ctx, t, tc, postgreSQLServerInstance)
 
 }

pkg/resourcemanager/azuresql/azuresqldb/azuresqldb_reconcile.go

@@ -6,6 +6,7 @@ package azuresqldb
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1"
 	"github.com/Azure/azure-service-operator/pkg/errhelp"
@@ -65,18 +66,16 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt
 		instance.Status.Message = resourcemanager.SuccessMsg
 		instance.Status.ResourceId = *dbGet.ID
 		return true, nil
-	} else {
-		azerr := errhelp.NewAzureErrorAzureError(err)
-		ignore := []string{
-			errhelp.NotFoundErrorCode,
-			errhelp.ResourceNotFound,
-			errhelp.ResourceGroupNotFoundErrorCode,
-		}
-		if !helpers.ContainsString(ignore, azerr.Type) {
-			instance.Status.Message = err.Error()
-			instance.Status.Provisioning = false
-			return false, fmt.Errorf("AzureSqlDb GetDB error %v", err)
-		}
+	}
+	instance.Status.Message = fmt.Sprintf("AzureSqlDb Get error %s", err.Error())
+	azerr := errhelp.NewAzureErrorAzureError(err)
+	requeuErrors := []string{
+		errhelp.ParentNotFoundErrorCode,
+		errhelp.ResourceGroupNotFoundErrorCode,
+	}
+	if helpers.ContainsString(requeuErrors, azerr.Type) {
+		instance.Status.Provisioning = false
+		return false, nil
 	}
 
 	resp, err := db.CreateOrUpdateDB(ctx, groupName, location, server, labels, azureSQLDatabaseProperties)
@@ -100,6 +99,13 @@ func (db *AzureSqlDbManager) Ensure(ctx context.Context, obj runtime.Object, opt
 			return false, nil
 		}
 
+		// if the database is busy, requeue
+		errorString := err.Error()
+		if strings.Contains(errorString, "Try again later") {
+			instance.Status.Provisioning = false
+			return false, nil
+		}
+
 		// assertion that a 404 error implies that the Azure SQL server hasn't been provisioned yet
 		if resp != nil && resp.StatusCode == 404 {
 			instance.Status.Message = fmt.Sprintf("Waiting for SQL Server %s to provision", server)

pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup_reconcile.go

@@ -58,6 +58,15 @@ func (fg *AzureSqlFailoverGroupManager) Ensure(ctx context.Context, obj runtime.
 		return true, nil
 	}
 	instance.Status.Message = fmt.Sprintf("AzureSqlFailoverGroup Get error %s", err.Error())
+	requeuErrors := []string{
+		errhelp.ResourceGroupNotFoundErrorCode,
+		errhelp.ParentNotFoundErrorCode,
+	}
+	azerr := errhelp.NewAzureErrorAzureError(err)
+	if helpers.ContainsString(requeuErrors, azerr.Type) {
+		instance.Status.Provisioning = false
+		return false, nil
+	}
 
 	_, err = fg.CreateOrUpdateFailoverGroup(ctx, groupName, serverName, failoverGroupName, sqlFailoverGroupProperties)
 	if err != nil {

pkg/resourcemanager/azuresql/azuresqlfirewallrule/azuresqlfirewallrule_reconcile.go

@@ -37,6 +37,15 @@ func (fw *AzureSqlFirewallRuleManager) Ensure(ctx context.Context, obj runtime.O
 		return true, nil
 	}
 	instance.Status.Message = fmt.Sprintf("AzureSqlFirewallRule Get error %s", err.Error())
+	requeuErrors := []string{
+		errhelp.ResourceGroupNotFoundErrorCode,
+		errhelp.ParentNotFoundErrorCode,
+	}
+	azerr := errhelp.NewAzureErrorAzureError(err)
+	if helpers.ContainsString(requeuErrors, azerr.Type) {
+		instance.Status.Provisioning = false
+		return false, nil
+	}
 
 	_, err = fw.CreateOrUpdateSQLFirewallRule(ctx, groupName, server, ruleName, startIP, endIP)
 	if err != nil {

pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go

@@ -78,28 +78,44 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op
 
 	_, err = s.GetDB(ctx, instance.Spec.ResourceGroup, instance.Spec.Server, instance.Spec.DbName)
 	if err != nil {
-		instance.Status.Message = err.Error()
+		instance.Status.Message = errhelp.StripErrorIDs(err)
+		instance.Status.Provisioning = false
 
-		catch := []string{
+		requeuErrors := []string{
 			errhelp.ResourceNotFound,
 			errhelp.ParentNotFoundErrorCode,
 			errhelp.ResourceGroupNotFoundErrorCode,
 		}
 		azerr := errhelp.NewAzureErrorAzureError(err)
-		if helpers.ContainsString(catch, azerr.Type) {
+		if helpers.ContainsString(requeuErrors, azerr.Type) {
 			return false, nil
 		}
-		return false, err
+
+		// if the database is busy, requeue
+		errorString := err.Error()
+		if strings.Contains(errorString, "Please retry the connection later") {
+			return false, nil
+		}
+
+		// if this is an unmarshall error - igmore and continue, otherwise report error and requeue
+		if !strings.Contains(errorString, "cannot unmarshal array into Go struct field serviceError2.details") {
+			return false, err
+		}
 	}
 
 	db, err := s.ConnectToSqlDb(ctx, DriverName, instance.Spec.Server, instance.Spec.DbName, SqlServerPort, adminUser, adminPassword)
 	if err != nil {
 		instance.Status.Message = errhelp.StripErrorIDs(err)
+		instance.Status.Provisioning = false
 
 		// catch firewall issue - keep cycling until it clears up
 		if strings.Contains(err.Error(), "create a firewall rule for this IP address") {
-			instance.Status.Provisioned = false
-			instance.Status.Provisioning = false
+			return false, nil
+		}
+
+		// if the database is busy, requeue
+		errorString := err.Error()
+		if strings.Contains(errorString, "Please retry the connection later") {
 			return false, nil
 		}
 

pkg/resourcemanager/azuresql/azuresqlvnetrule/azuresqlvnetrule_reconcile.go

@@ -6,6 +6,7 @@ package azuresqlvnetrule
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/2015-05-01-preview/sql"
 	azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1"
@@ -43,6 +44,15 @@ func (vr *AzureSqlVNetRuleManager) Ensure(ctx context.Context, obj runtime.Objec
 		return false, nil
 	}
 	instance.Status.Message = fmt.Sprintf("AzureSqlVNetRule Get error %s", err.Error())
+	requeuErrors := []string{
+		errhelp.ResourceGroupNotFoundErrorCode,
+		errhelp.ParentNotFoundErrorCode,
+	}
+	azerr := errhelp.NewAzureErrorAzureError(err)
+	if helpers.ContainsString(requeuErrors, azerr.Type) {
+		instance.Status.Provisioning = false
+		return false, nil
+	}
 
 	instance.Status.Provisioning = true
 	_, err = vr.CreateOrUpdateSQLVNetRule(ctx, groupName, server, ruleName, virtualNetworkRG, virtualnetworkname, subnetName, ignoreendpoint)
@@ -66,6 +76,13 @@ func (vr *AzureSqlVNetRuleManager) Ensure(ctx context.Context, obj runtime.Objec
 			return false, nil
 		}
 
+		// this happens when we try to create the VNet rule and the server doesnt exist yet
+		errorString := err.Error()
+		if strings.Contains(errorString, "does not have the server") {
+			instance.Status.Provisioning = false
+			return false, nil
+		}
+
 		return false, err
 	}