From 824bc706f03682d021aabc50d9f8606ed5bfb961 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Thu, 9 Apr 2020 15:37:55 -0600 Subject: [PATCH 01/27] Types file for replica support --- api/v1alpha1/mysqlserver_types.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index 9fb8df5b540..0892729e51c 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -12,12 +12,16 @@ import ( // MySQLServerSpec defines the desired state of MySQLServer type MySQLServerSpec struct { - Location string `json:"location"` - ResourceGroup string `json:"resourceGroup,omitempty"` - Sku AzureDBsSQLSku `json:"sku,omitempty"` - ServerVersion ServerVersion `json:"serverVersion,omitempty"` - SSLEnforcement SslEnforcementEnum `json:"sslEnforcement,omitempty"` - KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` + Location string `json:"location"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Sku AzureDBsSQLSku `json:"sku,omitempty"` + ServerVersion ServerVersion `json:"serverVersion,omitempty"` + SSLEnforcement SslEnforcementEnum `json:"sslEnforcement,omitempty"` + MinimalTLSVersion string `json:"minimalTLSVersion,omitempty"` + InfrastructureEncryption string `json:"infrastructureEncryption,omitempty"` + CreateMode string `json:"createMode,omitempty"` + ReplicaProperties ReplicaProperties `json:"replicaProperties, omitempty"` + KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` } // +kubebuilder:object:root=true @@ -41,6 +45,10 @@ type MySQLServerList struct { Items []MySQLServer `json:"items"` } +type ReplicaProperties struct { + SourceServerId string `json:"sourceServerId,omitempty"` +} + func init() { SchemeBuilder.Register(&MySQLServer{}, &MySQLServerList{}) } From 7e4c7ae80e6f06645e5401f1c94bf21952cc633a Mon Sep 17 00:00:00 2001 From: jananivMS Date: Thu, 9 Apr 2020 15:49:53 -0600 Subject: [PATCH 02/27] yamls --- config/samples/azure_v1alpha1_mysqlserver.yaml | 4 ++++ config/samples/azure_v1alpha1_mysqlserver_replica.yaml | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 config/samples/azure_v1alpha1_mysqlserver_replica.yaml diff --git a/config/samples/azure_v1alpha1_mysqlserver.yaml b/config/samples/azure_v1alpha1_mysqlserver.yaml index 3b2ff695ab5..c73f0e3b4f1 100644 --- a/config/samples/azure_v1alpha1_mysqlserver.yaml +++ b/config/samples/azure_v1alpha1_mysqlserver.yaml @@ -7,9 +7,13 @@ spec: resourceGroup: resourcegroup-azure-operators serverVersion: "8.0" sslEnforcement: Enabled + minimalTLSVersion: TLS10 # Possible values include: 'TLS10', 'TLS11', 'TLS12', 'Disabled' + infrastructureEncryption: Enabled # Possible values include: Enabled, Disabled + createMode: Default # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) sku: name: B_Gen5_2 tier: Basic family: Gen5 size: "51200" capacity: 2 + diff --git a/config/samples/azure_v1alpha1_mysqlserver_replica.yaml b/config/samples/azure_v1alpha1_mysqlserver_replica.yaml new file mode 100644 index 00000000000..b256cec145d --- /dev/null +++ b/config/samples/azure_v1alpha1_mysqlserver_replica.yaml @@ -0,0 +1,10 @@ +apiVersion: azure.microsoft.com/v1alpha1 +kind: MySQLServer +metadata: + name: mysqlserver-replica +spec: + location: eastus2 + createMode: Replica # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) + replicaProperties: + sourceServerId: /subscriptions/{SUBID}/resourceGroups/resourcegroup-azure-operators/providers/Microsoft.DBforMySQL/servers/mysqlserver-sample + From d874813927569e809b2145eeb17760a0c81a4225 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 08:35:25 -0600 Subject: [PATCH 03/27] changes for mysql replica --- api/v1alpha1/mysqlserver_types.go | 18 ++++++ pkg/resourcemanager/mysql/server/reconcile.go | 62 ++++++++++++++----- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index 0892729e51c..8643937dc1e 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -71,6 +71,24 @@ func NewDefaultMySQLServer(name, resourceGroup, location string) *MySQLServer { }, ServerVersion: ServerVersion("8.0"), SSLEnforcement: SslEnforcementEnumEnabled, + CreateMode: "Default", + }, + } +} + +func NewReplicaMySQLServer(name, resourceGroup, location string, sourceserverid string) *MySQLServer { + return &MySQLServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: MySQLServerSpec{ + Location: location, + ResourceGroup: resourceGroup, + CreateMode: "Replica", + ReplicaProperties: ReplicaProperties{ + SourceServerId: sourceserverid, + }, }, } } diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index 94155a7c50b..e7ae2b34ec3 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -6,6 +6,7 @@ package server import ( "context" "fmt" + "strings" mysql "github.com/Azure/azure-sdk-for-go/services/mysql/mgmt/2017-12-01/mysql" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -35,6 +36,19 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return true, err } + createmode := "Default" + if len(instance.Spec.CreateMode) != 0 { + createmode = instance.Spec.CreateMode + } + + // If a replica is requested, return error if source server is not specified + if strings.EqualFold(createmode, "replica") { + if len(instance.Spec.ReplicaProperties.SourceServerId) == 0 { + instance.Status.Message = "Replica requested but source server unspecified" + return true, nil + } + } + // Check to see if secret exists and if yes retrieve the admin login and password secret, err := m.GetOrPrepareSecret(ctx, instance) if err != nil { @@ -219,23 +233,43 @@ func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretN // GetOrPrepareSecret gets tje admin credentials if they are stored or generates some if not func (m *MySQLServerClient) GetOrPrepareSecret(ctx context.Context, instance *azurev1alpha1.MySQLServer) (map[string][]byte, error) { name := instance.Name + createmode := instance.Spec.CreateMode - secret := map[string][]byte{} + // If createmode == default, then this is a new server creation, so generate username/password + // If createmode == replica, then get the credentials from the source server secret and use that - key := types.NamespacedName{Name: name, Namespace: instance.Namespace} - if stored, err := m.SecretClient.Get(ctx, key); err == nil { - return stored, nil - } + secret := map[string][]byte{} + var key types.NamespacedName + var err error + if strings.EqualFold(createmode, "default") { // new Mysql server creation - randomUsername := helpers.GenerateRandomUsername(10) - randomPassword := helpers.NewPassword() + key = types.NamespacedName{Name: name, Namespace: instance.Namespace} + if stored, err := m.SecretClient.Get(ctx, key); err == nil { + return stored, nil + } - secret["username"] = []byte(randomUsername) - secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, name)) - secret["password"] = []byte(randomPassword) - secret["postgreSqlServerName"] = []byte(name) - // TODO: The below may not be right for non Azure public cloud. - secret["fullyQualifiedServerName"] = []byte(name + ".mysql.database.azure.com") + randomUsername := helpers.GenerateRandomUsername(10) + randomPassword := helpers.NewPassword() - return secret, nil + secret["username"] = []byte(randomUsername) + secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, name)) + secret["password"] = []byte(randomPassword) + secret["mySqlServerName"] = []byte(name) + // TODO: The below may not be right for non Azure public cloud. + secret["fullyQualifiedServerName"] = []byte(name + ".mysql.database.azure.com") + return secret, nil + } + // replica creation + sourceServerId := instance.Spec.ReplicaProperties.SourceServerId + if len(sourceServerId) != 0 { + //Parse to get source server name + sourceServerIdSplit := strings.Split(sourceServerId, "/") + sourceserver := sourceServerIdSplit[len(sourceServerIdSplit)-1] + key = types.NamespacedName{Name: sourceserver, Namespace: instance.Namespace} + if stored, err := m.SecretClient.Get(ctx, key); err == nil { + return stored, nil + } + // secret for source server does not exist, return error + } + return secret, err } From 25735b17c6babb87e3605a5d6b912521498979a5 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 16:53:01 -0600 Subject: [PATCH 04/27] Mysql replica support --- api/v1alpha1/mysqlserver_types.go | 18 ++-- .../samples/azure_v1alpha1_mysqlserver.yaml | 6 +- .../azure_v1alpha1_mysqlserver_replica.yaml | 2 + pkg/errhelp/errhelp.go | 1 + pkg/errhelp/errors.go | 2 + pkg/resourcemanager/mysql/server/client.go | 65 +++++++---- pkg/resourcemanager/mysql/server/manager.go | 2 +- pkg/resourcemanager/mysql/server/reconcile.go | 101 ++++++++++-------- 8 files changed, 117 insertions(+), 80 deletions(-) diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index 8643937dc1e..ae04a831b13 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -12,16 +12,14 @@ import ( // MySQLServerSpec defines the desired state of MySQLServer type MySQLServerSpec struct { - Location string `json:"location"` - ResourceGroup string `json:"resourceGroup,omitempty"` - Sku AzureDBsSQLSku `json:"sku,omitempty"` - ServerVersion ServerVersion `json:"serverVersion,omitempty"` - SSLEnforcement SslEnforcementEnum `json:"sslEnforcement,omitempty"` - MinimalTLSVersion string `json:"minimalTLSVersion,omitempty"` - InfrastructureEncryption string `json:"infrastructureEncryption,omitempty"` - CreateMode string `json:"createMode,omitempty"` - ReplicaProperties ReplicaProperties `json:"replicaProperties, omitempty"` - KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` + Location string `json:"location"` + ResourceGroup string `json:"resourceGroup,omitempty"` + Sku AzureDBsSQLSku `json:"sku,omitempty"` + ServerVersion ServerVersion `json:"serverVersion,omitempty"` + SSLEnforcement SslEnforcementEnum `json:"sslEnforcement,omitempty"` + CreateMode string `json:"createMode,omitempty"` + ReplicaProperties ReplicaProperties `json:"replicaProperties,omitempty"` + KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` } // +kubebuilder:object:root=true diff --git a/config/samples/azure_v1alpha1_mysqlserver.yaml b/config/samples/azure_v1alpha1_mysqlserver.yaml index c73f0e3b4f1..d8f43cd3c75 100644 --- a/config/samples/azure_v1alpha1_mysqlserver.yaml +++ b/config/samples/azure_v1alpha1_mysqlserver.yaml @@ -11,9 +11,9 @@ spec: infrastructureEncryption: Enabled # Possible values include: Enabled, Disabled createMode: Default # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) sku: - name: B_Gen5_2 - tier: Basic - family: Gen5 + name: B_Gen5_2 # tier + family + cores eg. - B_Gen4_1, GP_Gen5_4 + tier: Basic # possible values - 'Basic', 'GeneralPurpose', 'MemoryOptimized' + family: Gen5 size: "51200" capacity: 2 diff --git a/config/samples/azure_v1alpha1_mysqlserver_replica.yaml b/config/samples/azure_v1alpha1_mysqlserver_replica.yaml index b256cec145d..515e40589b4 100644 --- a/config/samples/azure_v1alpha1_mysqlserver_replica.yaml +++ b/config/samples/azure_v1alpha1_mysqlserver_replica.yaml @@ -4,7 +4,9 @@ metadata: name: mysqlserver-replica spec: location: eastus2 + resourceGroup: resourcegroup-azure-operators createMode: Replica # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) replicaProperties: + # sourceServer tier should be "GeneralPurpose" or higher for replica support sourceServerId: /subscriptions/{SUBID}/resourceGroups/resourcegroup-azure-operators/providers/Microsoft.DBforMySQL/servers/mysqlserver-sample diff --git a/pkg/errhelp/errhelp.go b/pkg/errhelp/errhelp.go index 58008895018..d2614f2e823 100644 --- a/pkg/errhelp/errhelp.go +++ b/pkg/errhelp/errhelp.go @@ -49,6 +49,7 @@ func StripErrorIDs(err error) string { patterns := []string{ "RequestID=", "CorrelationId:\\s", + "Tracking ID: ", } reg := regexp.MustCompile(fmt.Sprintf(`(%s)\S+`, strings.Join(patterns, "|"))) return reg.ReplaceAllString(err.Error(), "") diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index 4318fde13e5..ec0fa6c7fe2 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -55,6 +55,8 @@ const ( ServiceBusy = "ServiceBusy" NameNotAvailable = "NameNotAvailable" PublicIPIdleTimeoutIsOutOfRange = "PublicIPIdleTimeoutIsOutOfRange" + InvalidRequestContent = "InvalidRequestContent" + InternalServerError = "InternalServerError" ) func NewAzureError(err error) error { diff --git a/pkg/resourcemanager/mysql/server/client.go b/pkg/resourcemanager/mysql/server/client.go index c6a8804989f..8e26294dc5c 100644 --- a/pkg/resourcemanager/mysql/server/client.go +++ b/pkg/resourcemanager/mysql/server/client.go @@ -5,11 +5,13 @@ package server import ( "context" + "strings" mysql "github.com/Azure/azure-sdk-for-go/services/mysql/mgmt/2017-12-01/mysql" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" "github.com/Azure/azure-service-operator/pkg/secrets" + "github.com/Azure/go-autorest/autorest/to" "k8s.io/apimachinery/pkg/runtime" ) @@ -59,34 +61,55 @@ func (m *MySQLServerClient) CheckServerNameAvailability(ctx context.Context, ser } -func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string) (future mysql.ServersCreateFuture, err error) { +func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (server mysql.Server, err error) { client := getMySQLServersClient() // Check if name is valid if this is the first create call valid, err := m.CheckServerNameAvailability(ctx, servername) - if valid == false { - return future, err + if !valid { + return server, err } - - return client.Create( - ctx, - resourcegroup, - servername, - mysql.ServerForCreate{ - Location: &location, - Tags: tags, - Properties: &mysql.ServerPropertiesForDefaultCreate{ - AdministratorLogin: &adminlogin, - AdministratorLoginPassword: &adminpassword, - Version: serverversion, - SslEnforcement: sslenforcement, - //StorageProfile: &mysql.StorageProfile{}, - CreateMode: mysql.CreateModeServerPropertiesForCreate, + var result mysql.ServersCreateFuture + if strings.EqualFold(createmode, "replica") { + result, _ = client.Create( + ctx, + resourcegroup, + servername, + mysql.ServerForCreate{ + Location: &location, + Tags: tags, + Properties: &mysql.ServerPropertiesForReplica{ + SourceServerID: to.StringPtr(sourceserver), + CreateMode: mysql.CreateModeReplica, + }, + }, + ) + + } else { + + result, _ = client.Create( + ctx, + resourcegroup, + servername, + mysql.ServerForCreate{ + Location: &location, + Tags: tags, + Properties: &mysql.ServerPropertiesForDefaultCreate{ + AdministratorLogin: &adminlogin, + AdministratorLoginPassword: &adminpassword, + Version: serverversion, + SslEnforcement: sslenforcement, + //StorageProfile: &mysql.StorageProfile{}, + CreateMode: mysql.CreateModeServerPropertiesForCreate, + }, + Sku: &skuInfo, }, - Sku: &skuInfo, - }, - ) + ) + } + + return result.Result(client) + } func (m *MySQLServerClient) DeleteServer(ctx context.Context, resourcegroup string, servername string) (status string, err error) { diff --git a/pkg/resourcemanager/mysql/server/manager.go b/pkg/resourcemanager/mysql/server/manager.go index cc4b620f8d0..350a6450476 100644 --- a/pkg/resourcemanager/mysql/server/manager.go +++ b/pkg/resourcemanager/mysql/server/manager.go @@ -11,7 +11,7 @@ import ( ) type MySQLServerManager interface { - CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string) (mysql.ServersCreateFuture, error) + CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (mysql.Server, error) DeleteServer(ctx context.Context, resourcegroup string, servername string) (string, error) GetServer(ctx context.Context, resourcegroup string, servername string) (mysql.Server, error) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index e7ae2b34ec3..e0f07a5de6f 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -41,7 +41,7 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts createmode = instance.Spec.CreateMode } - // If a replica is requested, return error if source server is not specified + // If a replica is requested, ensure that source server is specified if strings.EqualFold(createmode, "replica") { if len(instance.Spec.ReplicaProperties.SourceServerId) == 0 { instance.Status.Message = "Replica requested but source server unspecified" @@ -55,17 +55,8 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return false, err } - // Update secret - err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance) - if err != nil { - return false, err - } - // convert kube labels to expected tag format - labels := map[string]*string{} - for k, v := range instance.GetLabels() { - labels[k] = &v - } + labels := helpers.LabelsToTags(instance.GetLabels()) // Check if this server already exists and its state if it does. This is required // to overcome the issue with the lack of idempotence of the Create call @@ -77,6 +68,14 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.Provisioning = false instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *server.ID + instance.Status.State = string(server.UserVisibleState) + + // Update secret - we do this on success as we need the FQ name of the server + err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *server.FullyQualifiedDomainName) + if err != nil { + return false, err + } + return true, nil } return false, nil @@ -98,7 +97,7 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts Family: to.StringPtr(instance.Spec.Sku.Family), } - _, err = m.CreateServerIfValid( + server, err = m.CreateServerIfValid( ctx, instance.Name, instance.Spec.ResourceGroup, @@ -109,10 +108,12 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts skuInfo, adminlogin, adminpassword, + createmode, + instance.Spec.ReplicaProperties.SourceServerId, ) if err != nil { // let the user know what happened - instance.Status.Message = err.Error() + instance.Status.Message = errhelp.StripErrorIDs(err) instance.Status.Provisioning = false azerr := errhelp.NewAzureErrorAzureError(err) @@ -120,12 +121,16 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: // errors we expect might happen that we are ok with waiting for return false, nil - case errhelp.ProvisioningDisabled, errhelp.LocationNotAvailableForResourceType: + case errhelp.ProvisioningDisabled, errhelp.LocationNotAvailableForResourceType, errhelp.InvalidRequestContent, errhelp.InternalServerError: // Unrecoverable error, so stop reconcilation - instance.Status.Message = "Reconcilation hit unrecoverable error: " + err.Error() + instance.Status.Message = "Reconcilation hit unrecoverable error: " + errhelp.StripErrorIDs(err) return true, nil + case errhelp.AsyncOpIncompleteError: + // Creation in progress + instance.Status.Provisioning = true + instance.Status.Message = "Server request submitted to Azure" + return false, nil } - // reconciliation not done and we don't know what happened return false, err } @@ -211,12 +216,15 @@ func (m *MySQLServerClient) convert(obj runtime.Object) (*v1alpha1.MySQLServer, } // AddServerCredsToSecrets saves the server's admin credentials in the secret store -func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.MySQLServer) error { +func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.MySQLServer, fullservername string) error { key := types.NamespacedName{ Name: secretName, Namespace: instance.Namespace, } + // Update fullyQualifiedServerName from the created server + data["fullyQualifiedServerName"] = []byte(fullservername) + err := m.SecretClient.Upsert(ctx, key, data, @@ -240,36 +248,39 @@ func (m *MySQLServerClient) GetOrPrepareSecret(ctx context.Context, instance *az secret := map[string][]byte{} var key types.NamespacedName - var err error - if strings.EqualFold(createmode, "default") { // new Mysql server creation - - key = types.NamespacedName{Name: name, Namespace: instance.Namespace} - if stored, err := m.SecretClient.Get(ctx, key); err == nil { - return stored, nil - } + var Username string + var Password string - randomUsername := helpers.GenerateRandomUsername(10) - randomPassword := helpers.NewPassword() - - secret["username"] = []byte(randomUsername) - secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, name)) - secret["password"] = []byte(randomPassword) - secret["mySqlServerName"] = []byte(name) - // TODO: The below may not be right for non Azure public cloud. - secret["fullyQualifiedServerName"] = []byte(name + ".mysql.database.azure.com") - return secret, nil + // See if secret already exists and return if it does + key = types.NamespacedName{Name: name, Namespace: instance.Namespace} + if stored, err := m.SecretClient.Get(ctx, key); err == nil { + return stored, nil } - // replica creation - sourceServerId := instance.Spec.ReplicaProperties.SourceServerId - if len(sourceServerId) != 0 { - //Parse to get source server name - sourceServerIdSplit := strings.Split(sourceServerId, "/") - sourceserver := sourceServerIdSplit[len(sourceServerIdSplit)-1] - key = types.NamespacedName{Name: sourceserver, Namespace: instance.Namespace} - if stored, err := m.SecretClient.Get(ctx, key); err == nil { - return stored, nil + + if strings.EqualFold(createmode, "default") { // new Mysql server creation + // Generate random username password if secret does not exist already + Username = helpers.GenerateRandomUsername(10) + Password = helpers.NewPassword() + } else { // replica + sourceServerId := instance.Spec.ReplicaProperties.SourceServerId + if len(sourceServerId) != 0 { + // Parse to get source server name + sourceServerIdSplit := strings.Split(sourceServerId, "/") + sourceserver := sourceServerIdSplit[len(sourceServerIdSplit)-1] + + // Get the username and password from the source server's secret + key = types.NamespacedName{Name: sourceserver, Namespace: instance.Namespace} + if sourcesecret, err := m.SecretClient.Get(ctx, key); err == nil { + Username = string(sourcesecret["username"]) + Password = string(sourcesecret["password"]) + } } - // secret for source server does not exist, return error } - return secret, err + + // Populate secret fields + secret["username"] = []byte(Username) + secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", Username, name)) + secret["password"] = []byte(Password) + secret["mySqlServerName"] = []byte(name) + return secret, nil } From 4e1b7940cac904fecdec613abf0de75468fb28f3 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 17:21:29 -0600 Subject: [PATCH 05/27] add tests and docs --- api/v1alpha1/mysqlserver_types.go | 6 +-- controllers/mysql_combined_test.go | 9 +++++ docs/mysql/mysql.md | 60 ++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 docs/mysql/mysql.md diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index ae04a831b13..3a3c8386512 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -61,11 +61,11 @@ func NewDefaultMySQLServer(name, resourceGroup, location string) *MySQLServer { Location: location, ResourceGroup: resourceGroup, Sku: AzureDBsSQLSku{ - Name: "B_Gen5_2", - Tier: SkuTier("Basic"), + Name: "GP_Gen5_4", + Tier: SkuTier("GeneralPurpose"), Family: "Gen5", Size: "51200", - Capacity: 2, + Capacity: 4, }, ServerVersion: ServerVersion("8.0"), SSLEnforcement: SslEnforcementEnumEnabled, diff --git a/controllers/mysql_combined_test.go b/controllers/mysql_combined_test.go index 59ad4976ec9..1ee87db8a1b 100644 --- a/controllers/mysql_combined_test.go +++ b/controllers/mysql_combined_test.go @@ -7,8 +7,10 @@ package controllers import ( "context" + "fmt" "testing" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -29,6 +31,12 @@ func TestMySQLHappyPath(t *testing.T) { RequireInstance(ctx, t, tc, mySQLServerInstance) + // Create a mySQL replica + fullSourceServerId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.DBforMySQL/servers/%s", config.Subscription(), rgName, mySQLServerName) + mySQLReplicaInstance := azurev1alpha1.NewReplicaMySQLServer(mySQLServerName, rgName, rgLocation, fullSourceServerId) + + EnsureInstance(ctx, t, tc, mySQLReplicaInstance) + mySQLDBName := GenerateTestResourceNameWithRandom("mysql-db", 10) // Create the mySQLDB object and expect the Reconcile to be created @@ -65,4 +73,5 @@ func TestMySQLHappyPath(t *testing.T) { EnsureDelete(ctx, t, tc, ruleInstance) EnsureDelete(ctx, t, tc, mySQLDBInstance) EnsureDelete(ctx, t, tc, mySQLServerInstance) + EnsureDelete(ctx, t, tc, mySQLReplicaInstance) } diff --git a/docs/mysql/mysql.md b/docs/mysql/mysql.md new file mode 100644 index 00000000000..4185aac9197 --- /dev/null +++ b/docs/mysql/mysql.md @@ -0,0 +1,60 @@ +# MySQL Operator + +## Resources Supported + +The MySQL operator suite consists of the following operators. + +1. MySQL server - Deploys an `Azure Database for MySQL server` given the Location, Resource group and other properties. This operator also helps creating read replicas for MySQL server. +2. MySQL database - Deploys a database under the given `Azure Database for MySQL server` +3. MySQL firewall rule - Deploys a firewall rule to allow access to the `Azure Database for MySQL server` from the specified IP range + +### MySQL server + +Here is a [sample YAML](/config/samples/azure_v1alpha1_mysqlserver.yaml) for the MySQL server. + +The value for kind, `MySQLServer` is the Custom Resource Definition (CRD) name. +`mysqlserver-sample` is the name of the MySQL server resource that will be created. + +The values under `spec` provide the values for the location where you want to create the server at and the Resource group in which you want to create it under. It also contains other values that are required to create the server like the `serverVersion`, `sslEnforcement` and the `sku` information. + +Along with creating the MySQL server, this operator also generates the admin username and password for the MySQL server and stores it in a kube secret or keyvault (based on what is specified) with the same name as the MySQL server. + +This secret contains the following fields. + +- `fullyqualifiedservername` : Fully qualified name of the MySQL server such as mysqlserver.mysql.database.azure.com +- `mysqlservername` : MySQL server name +- `username` : Server admin +- `password` : Password for the server admin +- `fullyqualifiedusername` : Fully qualified user name that is required by some apps such as @ + +For more information on where and how secrets are stored, look [here](/docs/secrets.md) + +#### Read Replicas in Azure Database for MySQL + +The MySQL server operator can also be used to create Read Replicas given the `sourceserverid` and the `location`. + +The replica inherits all other properties including the admin username and password from the source server. + +The operator reads the admin username and password for the source server from its secret (if available) and creates a secret with the same fields as described above for the replica. + +For more information on read replicas, refer [here](https://docs.microsoft.com/en-us/azure/mysql/concepts-read-replicas) + +### MySQL Database + +Here is a [sample YAML](/config/samples/azure_v1alpha1_mysqldatabase.yaml) for MySQL database + +Update the `resourcegroup` to where you want to provision the MySQL database. `server` is the name of the MySQL server where you want to create the database in. + +### MySQL firewall rule + +The MySQL firewall rule operator allows you to add a firewall rule to the MySQL server. + +Here is a [sample YAML](/config/samples/azure_v1alpha1_mysqlfirewallrule.yaml) for MySQL firewall rule + +The `server` indicates the MySQL server on which you want to configure the new MySQL firewall rule on and `resourceGroup` is the resource group of the MySQL server. The `startIpAddress` and `endIpAddress` indicate the IP range of sources to allow access to the server. + +*Note*: When the `startIpAddress` and `endIpAddress` are 0.0.0.0, it denotes a special case that adds a firewall rule to allow all Azure services to access the server. + +## Deploy, view and delete resources + +You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. From 5f371caff3409ce4086844b5529353fbec483c40 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 20:19:14 -0600 Subject: [PATCH 06/27] test fixes --- controllers/mysql_combined_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/controllers/mysql_combined_test.go b/controllers/mysql_combined_test.go index 1ee87db8a1b..a0cc952fbbe 100644 --- a/controllers/mysql_combined_test.go +++ b/controllers/mysql_combined_test.go @@ -7,10 +7,8 @@ package controllers import ( "context" - "fmt" "testing" - "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -25,6 +23,7 @@ func TestMySQLHappyPath(t *testing.T) { rgLocation := "eastus2" rgName := tc.resourceGroupName mySQLServerName := GenerateTestResourceNameWithRandom("mysql-srv", 10) + mySQLReplicaName := GenerateTestResourceNameWithRandom("mysql-rep", 10) // Create the mySQLServer object and expect the Reconcile to be created mySQLServerInstance := azurev1alpha1.NewDefaultMySQLServer(mySQLServerName, rgName, rgLocation) @@ -32,8 +31,7 @@ func TestMySQLHappyPath(t *testing.T) { RequireInstance(ctx, t, tc, mySQLServerInstance) // Create a mySQL replica - fullSourceServerId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.DBforMySQL/servers/%s", config.Subscription(), rgName, mySQLServerName) - mySQLReplicaInstance := azurev1alpha1.NewReplicaMySQLServer(mySQLServerName, rgName, rgLocation, fullSourceServerId) + mySQLReplicaInstance := azurev1alpha1.NewReplicaMySQLServer(mySQLReplicaName, rgName, rgLocation, mySQLServerInstance.Status.ResourceId) EnsureInstance(ctx, t, tc, mySQLReplicaInstance) From 37b29e25cbd17dba84f2b4519d4bc4107d0790d9 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 20:39:30 -0600 Subject: [PATCH 07/27] use suffix based on cloud env --- .../azuresqlfailovergroup/azuresqlfailovergroup.go | 5 +++-- .../azuresqlserver/azuresqlserver_reconcile.go | 3 ++- .../azuresql/azuresqluser/azuresqluser.go | 5 +++-- .../azuresql/azuresqluser/azuresqluser_reconcile.go | 5 +++-- pkg/resourcemanager/psql/server/server.go | 6 +++--- pkg/resourcemanager/psql/server/server_reconcile.go | 12 +++++++----- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup.go b/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup.go index ba36e73e454..45e4cf92fdd 100644 --- a/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup.go +++ b/pkg/resourcemanager/azuresql/azuresqlfailovergroup/azuresqlfailovergroup.go @@ -9,6 +9,7 @@ import ( azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/secrets" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -167,9 +168,9 @@ func (f *AzureSqlFailoverGroupManager) GetOrPrepareSecret(ctx context.Context, i } secret["azureSqlPrimaryServer"] = []byte(azuresqlprimaryserver) - secret["readWriteListenerEndpoint"] = []byte(failovergroupname + ".database.windows.net") + secret["readWriteListenerEndpoint"] = []byte(failovergroupname + "." + config.Environment().SQLDatabaseDNSSuffix) secret["azureSqlSecondaryServer"] = []byte(azuresqlsecondaryserver) - secret["readOnlyListenerEndpoint"] = []byte(failovergroupname + ".secondary.database.windows.net") + secret["readOnlyListenerEndpoint"] = []byte(failovergroupname + ".secondary." + config.Environment().SQLDatabaseDNSSuffix) return secret, nil } diff --git a/pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver_reconcile.go b/pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver_reconcile.go index ed0bfaebbfb..8e76fb24ab9 100644 --- a/pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqlserver/azuresqlserver_reconcile.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager" azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/resourcemanager/pollclient" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/go-autorest/autorest/to" @@ -335,7 +336,7 @@ func NewSecret(serverName string) (map[string][]byte, error) { secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, serverName)) secret["password"] = []byte(randomPassword) secret["azureSqlServerName"] = []byte(serverName) - secret["fullyQualifiedServerName"] = []byte(serverName + ".database.windows.net") + secret["fullyQualifiedServerName"] = []byte(serverName + "." + config.Environment().SQLDatabaseDNSSuffix) return secret, nil } diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser.go index 007d93d14bc..14d71ff6890 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser.go @@ -13,6 +13,7 @@ import ( azuresql "github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/2015-05-01-preview/sql" "github.com/Azure/azure-service-operator/pkg/helpers" azuresqlshared "github.com/Azure/azure-service-operator/pkg/resourcemanager/azuresql/azuresqlshared" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -65,7 +66,7 @@ func (s *AzureSqlUserManager) GetDB(ctx context.Context, resourceGroupName strin // ConnectToSqlDb connects to the SQL db using the given credentials func (s *AzureSqlUserManager) ConnectToSqlDb(ctx context.Context, drivername string, server string, database string, port int, user string, password string) (*sql.DB, error) { - fullServerAddress := fmt.Sprintf("%s.database.windows.net", server) + fullServerAddress := fmt.Sprintf("%s."+config.Environment().SQLDatabaseDNSSuffix, server) connString := fmt.Sprintf("server=%s;user id=%s;password=%s;port=%d;database=%s;Persist Security Info=False;Pooling=False;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30", fullServerAddress, user, password, port, database) db, err := sql.Open(drivername, connString) @@ -231,7 +232,7 @@ func (s *AzureSqlUserManager) GetOrPrepareSecret(ctx context.Context, instance * "password": []byte(pw), "azureSqlServerNamespace": []byte(instance.Namespace), "azureSqlServerName": []byte(instance.Spec.Server), - "fullyQualifiedServerName": []byte(instance.Spec.Server + ".database.windows.net"), + "fullyQualifiedServerName": []byte(instance.Spec.Server + "." + config.Environment().SQLDatabaseDNSSuffix), "azureSqlDatabaseName": []byte(instance.Spec.DbName), } } diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go index 235838a30a0..28036683751 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/Azure/azure-service-operator/pkg/helpers" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -178,7 +179,7 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op case "jdbc": formattedSecrets["jdbc"] = []byte(fmt.Sprintf( - "jdbc:sqlserver://%v:1433;database=%v;user=%v@%v;password=%v;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;", + "jdbc:sqlserver://%v:1433;database=%v;user=%v@%v;password=%v;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*."+config.Environment().SQLDatabaseDNSSuffix+";loginTimeout=30;", string(DBSecret["fullyQualifiedServerName"]), instance.Spec.DbName, user, @@ -187,7 +188,7 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op )) case "jdbc-urlonly": formattedSecrets["jdbc-urlonly"] = []byte(fmt.Sprintf( - "jdbc:sqlserver://%v:1433;database=%v;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;", + "jdbc:sqlserver://%v:1433;database=%v;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*."+config.Environment().SQLDatabaseDNSSuffix+";loginTimeout=30;", string(DBSecret["fullyQualifiedServerName"]), instance.Spec.DbName, )) diff --git a/pkg/resourcemanager/psql/server/server.go b/pkg/resourcemanager/psql/server/server.go index 965fd7c8454..71fa650f10e 100644 --- a/pkg/resourcemanager/psql/server/server.go +++ b/pkg/resourcemanager/psql/server/server.go @@ -137,12 +137,14 @@ func (p *PSQLServerClient) GetServer(ctx context.Context, resourcegroup string, return client.Get(ctx, resourcegroup, servername) } -func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer) error { +func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer, fullservername string) error { key := types.NamespacedName{ Name: secretName, Namespace: instance.Namespace, } + data["fullyQualifiedServerName"] = []byte(fullservername) + err := p.SecretClient.Upsert(ctx, key, data, @@ -175,8 +177,6 @@ func (p *PSQLServerClient) GetOrPrepareSecret(ctx context.Context, instance *azu secret["fullyQualifiedUsername"] = []byte(fmt.Sprintf("%s@%s", randomUsername, name)) secret["password"] = []byte(randomPassword) secret["postgreSqlServerName"] = []byte(name) - // TODO: The below may not be right for non Azure public cloud. - secret["fullyQualifiedServerName"] = []byte(name + ".postgres.database.azure.com") return secret, nil } diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index f00497a8ed1..cb39ca8d3df 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -39,11 +39,6 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts if err != nil { return false, err } - // Update secret - err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance) - if err != nil { - return false, err - } // if an error occurs thats ok as it means that it doesn't exist yet getServer, err := p.GetServer(ctx, instance.Spec.ResourceGroup, instance.Name) @@ -52,6 +47,13 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // succeeded! so end reconcilliation successfully if getServer.UserVisibleState == "Ready" { + + // Update secret with the fully qualified server name + err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *getServer.FullyQualifiedDomainName) + if err != nil { + return false, err + } + instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *getServer.ID instance.Status.Provisioned = true From f187f6955716331753a6a7eb727626ef6ec5cf74 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 21:03:52 -0600 Subject: [PATCH 08/27] fix build error --- pkg/resourcemanager/psql/server/server_manager.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resourcemanager/psql/server/server_manager.go b/pkg/resourcemanager/psql/server/server_manager.go index 3fc3ad42f8c..73948d3af05 100644 --- a/pkg/resourcemanager/psql/server/server_manager.go +++ b/pkg/resourcemanager/psql/server/server_manager.go @@ -39,7 +39,8 @@ type PostgreSQLServerManager interface { AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, - instance *azurev1alpha1.PostgreSQLServer) error + instance *azurev1alpha1.PostgreSQLServer, + fullservername string) error GetOrPrepareSecret(ctx context.Context, instance *azurev1alpha1.PostgreSQLServer) (map[string][]byte, error) From 37a47e2c29236ba61b33344f68eb98931dedbf4b Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 14 Apr 2020 21:05:30 -0600 Subject: [PATCH 09/27] fixed edge casee --- pkg/resourcemanager/mysql/server/reconcile.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index e0f07a5de6f..7c429dfc2a0 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -64,11 +64,6 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts if err == nil { instance.Status.State = string(server.UserVisibleState) if server.UserVisibleState == mysql.ServerStateReady { - instance.Status.Provisioned = true - instance.Status.Provisioning = false - instance.Status.Message = resourcemanager.SuccessMsg - instance.Status.ResourceId = *server.ID - instance.Status.State = string(server.UserVisibleState) // Update secret - we do this on success as we need the FQ name of the server err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *server.FullyQualifiedDomainName) @@ -76,6 +71,11 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return false, err } + instance.Status.Provisioned = true + instance.Status.Provisioning = false + instance.Status.Message = resourcemanager.SuccessMsg + instance.Status.ResourceId = *server.ID + instance.Status.State = string(server.UserVisibleState) return true, nil } return false, nil From 03be5198c7c3692975fb4fd46b3ccb5157d6226c Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 09:51:26 -0600 Subject: [PATCH 10/27] fix error return --- pkg/resourcemanager/mysql/server/reconcile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index 7c429dfc2a0..100cfeec7a3 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -68,7 +68,8 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // Update secret - we do this on success as we need the FQ name of the server err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *server.FullyQualifiedDomainName) if err != nil { - return false, err + instance.Status.Message = "Could not save secrets" + return true, nil } instance.Status.Provisioned = true From aaea1d9571bc04816c53ea2b5eb2be6d25e74e20 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 09:52:30 -0600 Subject: [PATCH 11/27] fix error return --- pkg/resourcemanager/psql/server/server_reconcile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index cb39ca8d3df..bff682f536f 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -51,7 +51,8 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // Update secret with the fully qualified server name err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *getServer.FullyQualifiedDomainName) if err != nil { - return false, err + instance.Status.Message = "Could not save secrets" + return true, nil } instance.Status.Message = resourcemanager.SuccessMsg From 9bc040750e047d36bf7165de022e0cd6606ff13b Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 11:38:52 -0600 Subject: [PATCH 12/27] fixed secret update --- pkg/resourcemanager/mysql/server/reconcile.go | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index 100cfeec7a3..7b36c556346 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -55,6 +55,11 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return false, err } + err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance) + if err != nil { + return false, err + } + // convert kube labels to expected tag format labels := helpers.LabelsToTags(instance.GetLabels()) @@ -65,12 +70,8 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.State = string(server.UserVisibleState) if server.UserVisibleState == mysql.ServerStateReady { - // Update secret - we do this on success as we need the FQ name of the server - err = m.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *server.FullyQualifiedDomainName) - if err != nil { - instance.Status.Message = "Could not save secrets" - return true, nil - } + // Update secret with FQ name of the server. We ignore the error. + m.UpdateServerNameInSecret(ctx, instance.Name, secret, *server.FullyQualifiedDomainName, instance) instance.Status.Provisioned = true instance.Status.Provisioning = false @@ -217,13 +218,32 @@ func (m *MySQLServerClient) convert(obj runtime.Object) (*v1alpha1.MySQLServer, } // AddServerCredsToSecrets saves the server's admin credentials in the secret store -func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.MySQLServer, fullservername string) error { +func (m *MySQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.MySQLServer) error { + key := types.NamespacedName{ + Name: secretName, + Namespace: instance.Namespace, + } + + err := m.SecretClient.Upsert(ctx, + key, + data, + secrets.WithOwner(instance), + secrets.WithScheme(m.Scheme), + ) + if err != nil { + return err + } + + return nil +} + +// UpdateSecretWithFullServerName updates the secret with the fully qualified server name +func (m *MySQLServerClient) UpdateServerNameInSecret(ctx context.Context, secretName string, data map[string][]byte, fullservername string, instance *azurev1alpha1.MySQLServer) error { key := types.NamespacedName{ Name: secretName, Namespace: instance.Namespace, } - // Update fullyQualifiedServerName from the created server data["fullyQualifiedServerName"] = []byte(fullservername) err := m.SecretClient.Upsert(ctx, From 7e53ee4f0398e2d10843a8afa583b4712bb47b8c Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 11:44:11 -0600 Subject: [PATCH 13/27] updated secret updation --- pkg/resourcemanager/psql/server/server.go | 23 ++++++++++++++++++- .../psql/server/server_reconcile.go | 15 +++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/pkg/resourcemanager/psql/server/server.go b/pkg/resourcemanager/psql/server/server.go index 71fa650f10e..a456a9cb290 100644 --- a/pkg/resourcemanager/psql/server/server.go +++ b/pkg/resourcemanager/psql/server/server.go @@ -137,7 +137,28 @@ func (p *PSQLServerClient) GetServer(ctx context.Context, resourcegroup string, return client.Get(ctx, resourcegroup, servername) } -func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer, fullservername string) error { +func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer) error { + key := types.NamespacedName{ + Name: secretName, + Namespace: instance.Namespace, + } + + data["fullyQualifiedServerName"] = []byte(fullservername) + + err := p.SecretClient.Upsert(ctx, + key, + data, + secrets.WithOwner(instance), + secrets.WithScheme(p.Scheme), + ) + if err != nil { + return err + } + + return nil +} + +func (p *PSQLServerClient) UpdateSecretWithFullServerName(ctx context.Context, secretName string, data map[string][]byte, instance *azurev1alpha1.PostgreSQLServer, fullservername string) error { key := types.NamespacedName{ Name: secretName, Namespace: instance.Namespace, diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index bff682f536f..b879adac85a 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -40,6 +40,12 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts return false, err } + // Update secret with the fully qualified server name + err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance) + if err != nil { + return false, err + } + // if an error occurs thats ok as it means that it doesn't exist yet getServer, err := p.GetServer(ctx, instance.Spec.ResourceGroup, instance.Name) if err == nil { @@ -48,17 +54,14 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // succeeded! so end reconcilliation successfully if getServer.UserVisibleState == "Ready" { - // Update secret with the fully qualified server name - err = p.AddServerCredsToSecrets(ctx, instance.Name, secret, instance, *getServer.FullyQualifiedDomainName) - if err != nil { - instance.Status.Message = "Could not save secrets" - return true, nil - } + // Update the secret with fully qualified server name. Ignore error as we have the admin creds which is critical. + p.UpdateSecretWithFullServerName(ctx, instance.Name, secret, instance, *getServer.FullyQualifiedDomainName) instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *getServer.ID instance.Status.Provisioned = true instance.Status.Provisioning = false + instance.Status.State = string(getServer.UserVisibleState) return true, nil } From fb8ce7b2977a30f220c8e0d85ce42c9293984369 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 11:49:24 -0600 Subject: [PATCH 14/27] updated errors --- pkg/resourcemanager/psql/server/server.go | 2 -- pkg/resourcemanager/psql/server/server_manager.go | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/resourcemanager/psql/server/server.go b/pkg/resourcemanager/psql/server/server.go index a456a9cb290..21c2b1c4a73 100644 --- a/pkg/resourcemanager/psql/server/server.go +++ b/pkg/resourcemanager/psql/server/server.go @@ -143,8 +143,6 @@ func (p *PSQLServerClient) AddServerCredsToSecrets(ctx context.Context, secretNa Namespace: instance.Namespace, } - data["fullyQualifiedServerName"] = []byte(fullservername) - err := p.SecretClient.Upsert(ctx, key, data, diff --git a/pkg/resourcemanager/psql/server/server_manager.go b/pkg/resourcemanager/psql/server/server_manager.go index 73948d3af05..3fc3ad42f8c 100644 --- a/pkg/resourcemanager/psql/server/server_manager.go +++ b/pkg/resourcemanager/psql/server/server_manager.go @@ -39,8 +39,7 @@ type PostgreSQLServerManager interface { AddServerCredsToSecrets(ctx context.Context, secretName string, data map[string][]byte, - instance *azurev1alpha1.PostgreSQLServer, - fullservername string) error + instance *azurev1alpha1.PostgreSQLServer) error GetOrPrepareSecret(ctx context.Context, instance *azurev1alpha1.PostgreSQLServer) (map[string][]byte, error) From 64e8cb18b1bc4023e14a24444cedb01edaf957f6 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Wed, 15 Apr 2020 15:45:17 -0600 Subject: [PATCH 15/27] add mysql doc to readme and fix formatting --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 79c97963b85..b0b69389fb2 100644 --- a/README.md +++ b/README.md @@ -32,15 +32,16 @@ This project maintains [releases of the Azure Service Operator](https://github.c 1. [Resource Group](/docs/resourcegroup/resourcegroup.md) 2. [EventHub](/docs/eventhub/eventhub.md) 3. [Azure SQL](/docs/azuresql/azuresql.md) -4. [Azure Keyvault](/docs/keyvault/keyvault.md) -5. [Azure Rediscache](/docs/rediscache/rediscache.md) -6. [Storage Account](/docs/storage/storageaccount.md) -7. [Blob container](/docs/storage/blobcontainer.md) -8. [Azure Database for PostgreSQL](/docs/postgresql/postgresql.md) -9. [Virtual Network](/docs/virtualnetwork/virtualnetwork.md) -10.[Application Insights](/docs/appinsights/appinsights.md) -11.[API Management](/docs/apimgmt/apimgmt.md) -12.[Cosmos DB](/docs/cosmosdb/cosmosdb.md) +4. [Azure Database for PostgreSQL](/docs/postgresql/postgresql.md) +5. [Azure Database for MySQL](/docs/mysql/mysql.md) +6. [Azure Keyvault](/docs/keyvault/keyvault.md) +7. [Azure Rediscache](/docs/rediscache/rediscache.md) +8. [Storage Account](/docs/storage/storageaccount.md) +9. [Blob container](/docs/storage/blobcontainer.md) +10. [Virtual Network](/docs/virtualnetwork/virtualnetwork.md) +11. [Application Insights](/docs/appinsights/appinsights.md) +12. [API Management](/docs/apimgmt/apimgmt.md) +13. [Cosmos DB](/docs/cosmosdb/cosmosdb.md) For more information on deploying, troubleshooting & deleting resources, refer to [this](/docs/customresource.md) link From 1b96561aadb3deca8965d9c80bf1bbdf001393f5 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Fri, 17 Apr 2020 17:02:40 -0600 Subject: [PATCH 16/27] updated to address PR comments --- .../azuresqluser/azuresqluser_reconcile.go | 1 - pkg/resourcemanager/mysql/server/client.go | 7 +- pkg/resourcemanager/mysql/server/manager.go | 2 +- pkg/resourcemanager/mysql/server/reconcile.go | 82 ++++++++++++++----- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go index 649752a4d96..3540721b6e0 100644 --- a/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go +++ b/pkg/resourcemanager/azuresql/azuresqluser/azuresqluser_reconcile.go @@ -277,7 +277,6 @@ func (s *AzureSqlUserManager) Ensure(ctx context.Context, obj runtime.Object, op err = s.GrantUserRoles(ctx, user, instance.Spec.Roles, db) if err != nil { - fmt.Println(err) instance.Status.Message = "GrantUserRoles failed" return false, fmt.Errorf("GrantUserRoles failed") } diff --git a/pkg/resourcemanager/mysql/server/client.go b/pkg/resourcemanager/mysql/server/client.go index 8e26294dc5c..22f78de705d 100644 --- a/pkg/resourcemanager/mysql/server/client.go +++ b/pkg/resourcemanager/mysql/server/client.go @@ -61,14 +61,14 @@ func (m *MySQLServerClient) CheckServerNameAvailability(ctx context.Context, ser } -func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (server mysql.Server, err error) { +func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (pollingURL string, server mysql.Server, err error) { client := getMySQLServersClient() // Check if name is valid if this is the first create call valid, err := m.CheckServerNameAvailability(ctx, servername) if !valid { - return server, err + return "", server, err } var result mysql.ServersCreateFuture if strings.EqualFold(createmode, "replica") { @@ -108,7 +108,8 @@ func (m *MySQLServerClient) CreateServerIfValid(ctx context.Context, servername ) } - return result.Result(client) + res, err := result.Result(client) + return result.PollingURL(), res, err } diff --git a/pkg/resourcemanager/mysql/server/manager.go b/pkg/resourcemanager/mysql/server/manager.go index 350a6450476..f90b58e5c70 100644 --- a/pkg/resourcemanager/mysql/server/manager.go +++ b/pkg/resourcemanager/mysql/server/manager.go @@ -11,7 +11,7 @@ import ( ) type MySQLServerManager interface { - CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (mysql.Server, error) + CreateServerIfValid(ctx context.Context, servername string, resourcegroup string, location string, tags map[string]*string, serverversion mysql.ServerVersion, sslenforcement mysql.SslEnforcementEnum, skuInfo mysql.Sku, adminlogin string, adminpassword string, createmode string, sourceserver string) (pollingURL string, server mysql.Server, err error) DeleteServer(ctx context.Context, resourcegroup string, servername string) (string, error) GetServer(ctx context.Context, resourcegroup string, servername string) (mysql.Server, error) diff --git a/pkg/resourcemanager/mysql/server/reconcile.go b/pkg/resourcemanager/mysql/server/reconcile.go index 7b36c556346..c8a75490add 100644 --- a/pkg/resourcemanager/mysql/server/reconcile.go +++ b/pkg/resourcemanager/mysql/server/reconcile.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-service-operator/pkg/errhelp" "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/Azure/azure-service-operator/pkg/resourcemanager" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/pollclient" "github.com/Azure/azure-service-operator/pkg/secrets" "github.com/Azure/go-autorest/autorest/to" "k8s.io/apimachinery/pkg/runtime" @@ -65,11 +66,40 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts // Check if this server already exists and its state if it does. This is required // to overcome the issue with the lack of idempotence of the Create call + hash := "" server, err := m.GetServer(ctx, instance.Spec.ResourceGroup, instance.Name) - if err == nil { + if err != nil { + // handle failures in the async operation + if instance.Status.PollingURL != "" { + pClient := pollclient.NewPollClient() + res, err := pClient.Get(ctx, instance.Status.PollingURL) + if err != nil { + instance.Status.Provisioning = false + return false, err + } + + if res.Status == "Failed" { + instance.Status.Provisioning = false + instance.Status.RequestedAt = nil + ignore := []string{ + errhelp.SubscriptionDoesNotHaveServer, + errhelp.ServiceBusy, + } + if !helpers.ContainsString(ignore, res.Error.Code) { + instance.Status.Message = res.Error.Error() + return true, nil + } + } + } + } else { instance.Status.State = string(server.UserVisibleState) - if server.UserVisibleState == mysql.ServerStateReady { + hash = helpers.Hash256(instance.Spec) + if instance.Status.SpecHash == hash && (instance.Status.Provisioned || instance.Status.FailedProvisioning) { + instance.Status.RequestedAt = nil + return true, nil + } + if server.UserVisibleState == mysql.ServerStateReady { // Update secret with FQ name of the server. We ignore the error. m.UpdateServerNameInSecret(ctx, instance.Name, secret, *server.FullyQualifiedDomainName, instance) @@ -78,9 +108,9 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.Message = resourcemanager.SuccessMsg instance.Status.ResourceId = *server.ID instance.Status.State = string(server.UserVisibleState) + instance.Status.SpecHash = hash return true, nil } - return false, nil } // if the create has been sent with no error we need to wait before calling it again @@ -99,7 +129,7 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts Family: to.StringPtr(instance.Spec.Sku.Family), } - server, err = m.CreateServerIfValid( + pollURL, server, err := m.CreateServerIfValid( ctx, instance.Name, instance.Spec.ResourceGroup, @@ -119,20 +149,35 @@ func (m *MySQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.Provisioning = false azerr := errhelp.NewAzureErrorAzureError(err) - switch azerr.Type { - case errhelp.ResourceGroupNotFoundErrorCode, errhelp.ParentNotFoundErrorCode: - // errors we expect might happen that we are ok with waiting for + catchRequeue := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + errhelp.AsyncOpIncompleteError, + errhelp.SubscriptionDoesNotHaveServer, + errhelp.ServiceBusy, + } + catchUnrecoverable := []string{ + errhelp.ProvisioningDisabled, + errhelp.LocationNotAvailableForResourceType, + errhelp.InvalidRequestContent, + errhelp.InternalServerError, + } + + // handle the errors + if helpers.ContainsString(catchRequeue, azerr.Type) { + if azerr.Type == errhelp.AsyncOpIncompleteError { + instance.Status.Provisioning = true + instance.Status.PollingURL = pollURL + } return false, nil - case errhelp.ProvisioningDisabled, errhelp.LocationNotAvailableForResourceType, errhelp.InvalidRequestContent, errhelp.InternalServerError: + } + + if helpers.ContainsString(catchUnrecoverable, azerr.Type) { // Unrecoverable error, so stop reconcilation instance.Status.Message = "Reconcilation hit unrecoverable error: " + errhelp.StripErrorIDs(err) return true, nil - case errhelp.AsyncOpIncompleteError: - // Creation in progress - instance.Status.Provisioning = true - instance.Status.Message = "Server request submitted to Azure" - return false, nil } + // reconciliation not done and we don't know what happened return false, err } @@ -272,13 +317,12 @@ func (m *MySQLServerClient) GetOrPrepareSecret(ctx context.Context, instance *az var Username string var Password string - // See if secret already exists and return if it does - key = types.NamespacedName{Name: name, Namespace: instance.Namespace} - if stored, err := m.SecretClient.Get(ctx, key); err == nil { - return stored, nil - } - if strings.EqualFold(createmode, "default") { // new Mysql server creation + // See if secret already exists and return if it does + key = types.NamespacedName{Name: name, Namespace: instance.Namespace} + if stored, err := m.SecretClient.Get(ctx, key); err == nil { + return stored, nil + } // Generate random username password if secret does not exist already Username = helpers.GenerateRandomUsername(10) Password = helpers.NewPassword() From 631352c6fdd3b08cd510a5a342c86881fe25d6e9 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Mon, 20 Apr 2020 11:10:48 -0600 Subject: [PATCH 17/27] updated yaml --- config/samples/azure_v1alpha1_mysqlserver.yaml | 6 +++--- pkg/errhelp/errors.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/samples/azure_v1alpha1_mysqlserver.yaml b/config/samples/azure_v1alpha1_mysqlserver.yaml index d8f43cd3c75..05d543e51f2 100644 --- a/config/samples/azure_v1alpha1_mysqlserver.yaml +++ b/config/samples/azure_v1alpha1_mysqlserver.yaml @@ -11,9 +11,9 @@ spec: infrastructureEncryption: Enabled # Possible values include: Enabled, Disabled createMode: Default # Possible values include: Default, Replica, PointInTimeRestore (not implemented), GeoRestore (not implemented) sku: - name: B_Gen5_2 # tier + family + cores eg. - B_Gen4_1, GP_Gen5_4 - tier: Basic # possible values - 'Basic', 'GeneralPurpose', 'MemoryOptimized' + name: GP_Gen5_4 # tier + family + cores eg. - B_Gen4_1, GP_Gen5_4 + tier: GeneralPurpose # possible values - 'Basic', 'GeneralPurpose', 'MemoryOptimized' family: Gen5 size: "51200" - capacity: 2 + capacity: 4 diff --git a/pkg/errhelp/errors.go b/pkg/errhelp/errors.go index 548427f06e5..4ff6b8d5eb5 100644 --- a/pkg/errhelp/errors.go +++ b/pkg/errhelp/errors.go @@ -57,7 +57,7 @@ const ( PublicIPIdleTimeoutIsOutOfRange = "PublicIPIdleTimeoutIsOutOfRange" InvalidRequestContent = "InvalidRequestContent" InternalServerError = "InternalServerError" - NetworkAclsValidationFailure = "NetworkAclsValidationFailure" + NetworkAclsValidationFailure = "NetworkAclsValidationFailure" ) func NewAzureError(err error) error { From d0aff6c8a1f5fd17b57148c9cff5276599e40ece Mon Sep 17 00:00:00 2001 From: jananivMS Date: Mon, 20 Apr 2020 23:29:23 -0600 Subject: [PATCH 18/27] test cleanup --- api/v1alpha1/storageaccount_types.go | 3 +- controllers/apimgmt_controller_test.go | 26 +-- controllers/appinsights_controller_test.go | 25 +-- ...edatalakegen2filesystem_controller_test.go | 4 +- controllers/azuresqlaction_controller_test.go | 16 +- .../azuresqldatabase_controller_test.go | 45 +--- .../azuresqlfailovergroup_controller_test.go | 24 +- .../azuresqlfirewallrule_controller_test.go | 28 +-- controllers/azuresqlserver_controller_test.go | 29 +-- controllers/azuresqluser_controller_test.go | 49 +---- controllers/azurevnetrule_controller_test.go | 33 +-- controllers/blobcontainer_controller_test.go | 116 ++-------- controllers/consumergroup_controller_test.go | 84 ------- ...namespace_consumergroup_controller_test.go | 104 +++++++++ ...venthub_storageaccount_controller_test.go} | 205 ++++++++---------- .../eventhubnamespace_controller_test.go | 124 ++--------- controllers/helpers.go | 61 +++++- controllers/keyvault_controller_test.go | 61 +----- controllers/rediscache_controller_test.go | 39 +--- controllers/resourcegroup_controller_test.go | 11 +- controllers/storageaccount_controller_test.go | 43 ---- controllers/suite_test.go | 126 +++++------ .../storages/blobcontainer/blob_container.go | 4 - .../blobcontainer/blob_container_reconcile.go | 1 + .../storageaccount_reconcile.go | 3 + 25 files changed, 392 insertions(+), 872 deletions(-) delete mode 100644 controllers/consumergroup_controller_test.go create mode 100644 controllers/eventhub_namespace_consumergroup_controller_test.go rename controllers/{eventhub_controller_test.go => eventhub_storageaccount_controller_test.go} (57%) delete mode 100644 controllers/storageaccount_controller_test.go diff --git a/api/v1alpha1/storageaccount_types.go b/api/v1alpha1/storageaccount_types.go index 41ba60b368c..99696d8b3fe 100644 --- a/api/v1alpha1/storageaccount_types.go +++ b/api/v1alpha1/storageaccount_types.go @@ -51,7 +51,7 @@ type StorageAccountSkuName string // Only one of the following kinds may be specified. // If none of the following kinds is specified, the default one // is StorageV2. -// +kubebuilder:validation:Enum=BlobStorage;BlockBlobStorage;FileStorage;StorageAccount;StorageV2 +// +kubebuilder:validation:Enum=BlobStorage;BlockBlobStorage;FileStorage;Storage;StorageV2 type StorageAccountKind string // StorageAccountAccessTier enumerates the values for access tier. @@ -90,7 +90,6 @@ type StorageAccountAdditionalResources struct { } // +kubebuilder:object:root=true -// +kubebuilder:subresource:status // StorageAccountList contains a list of Storage type StorageAccountList struct { diff --git a/controllers/apimgmt_controller_test.go b/controllers/apimgmt_controller_test.go index d9781435e1f..21d9410b2f6 100644 --- a/controllers/apimgmt_controller_test.go +++ b/controllers/apimgmt_controller_test.go @@ -7,21 +7,17 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/helpers" - "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestAPIMgmtController(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) // rgName := tc.resourceGroupName rgLocation := "southcentralus" @@ -50,25 +46,7 @@ func TestAPIMgmtController(t *testing.T) { }, }, } + EnsureInstance(ctx, t, tc, apiMgmtInstance) - err := tc.k8sClient.Create(ctx, apiMgmtInstance) - assert.Equal(nil, err, "create APIMgmtAPI record in k8s") - - APIMgmtNamespacedName := types.NamespacedName{Name: apiMgmtName, Namespace: "default"} - - // Wait for the APIMgmtAPI instance to be written to k8s - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, APIMgmtNamespacedName, apiMgmtInstance) - return strings.Contains(apiMgmtInstance.Status.Message, successMsg) - }, tc.timeout, tc.retry, "awaiting APIMgmt instance creation") - - // Delete the service - err = tc.k8sClient.Delete(ctx, apiMgmtInstance) - assert.Equal(nil, err, "deleting APIMgmt in k8s") - - // Wait for the APIMgmtAPI instance to be deleted - assert.Eventually(func() bool { - err := tc.k8sClient.Get(ctx, APIMgmtNamespacedName, apiMgmtInstance) - return err != nil - }, tc.timeout, tc.retry, "awaiting APIMgmtInstance deletion") + EnsureDelete(ctx, t, tc, apiMgmtInstance) } diff --git a/controllers/appinsights_controller_test.go b/controllers/appinsights_controller_test.go index aa4c19f15f9..6d9498fd571 100644 --- a/controllers/appinsights_controller_test.go +++ b/controllers/appinsights_controller_test.go @@ -7,20 +7,16 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestAppInsightsController(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) rgName := tc.resourceGroupName rgLocation := tc.resourceGroupLocation @@ -40,24 +36,7 @@ func TestAppInsightsController(t *testing.T) { }, } - err := tc.k8sClient.Create(ctx, appInsightsInstance) - assert.Equal(nil, err, "create appinsights record in k8s") + EnsureInstance(ctx, t, tc, appInsightsInstance) - appInsightsNamespacedName := types.NamespacedName{Name: appInsightsName, Namespace: "default"} - - // Wait for the AppInsights instance to be provisioned - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, appInsightsNamespacedName, appInsightsInstance) - return strings.Contains(appInsightsInstance.Status.Message, successMsg) - }, tc.timeout, tc.retry, "awaiting appinsights instance creation") - - // Delete the service - err = tc.k8sClient.Delete(ctx, appInsightsInstance) - assert.Equal(nil, err, "deleting appinsights in k8s") - - // Wait for the AppInsights instance to be deleted - assert.Eventually(func() bool { - err := tc.k8sClient.Get(ctx, appInsightsNamespacedName, appInsightsInstance) - return err != nil - }, tc.timeout, tc.retry, "awaiting appInsightsInstance deletion") + EnsureDelete(ctx, t, tc, appInsightsInstance) } diff --git a/controllers/azuredatalakegen2filesystem_controller_test.go b/controllers/azuredatalakegen2filesystem_controller_test.go index f3b2fdc978d..0eb87b83118 100644 --- a/controllers/azuredatalakegen2filesystem_controller_test.go +++ b/controllers/azuredatalakegen2filesystem_controller_test.go @@ -18,7 +18,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -func TestADLSFilesystemControllerNoResourceGroup(t *testing.T) { +/*func TestADLSFilesystemControllerNoResourceGroup(t *testing.T) { t.Parallel() defer PanicRecover(t) assert := assert.New(t) @@ -67,7 +67,7 @@ func TestADLSFilesystemControllerNoResourceGroup(t *testing.T) { return apierrors.IsNotFound(err) }, tc.timeout, tc.retry, "wait for filesystem to be gone") -} +}*/ func TestADLSFilesystemControllerNoStorageAccount(t *testing.T) { t.Parallel() diff --git a/controllers/azuresqlaction_controller_test.go b/controllers/azuresqlaction_controller_test.go index 544a5a08e6d..efaa06c3a77 100644 --- a/controllers/azuresqlaction_controller_test.go +++ b/controllers/azuresqlaction_controller_test.go @@ -51,22 +51,12 @@ func RunSQLActionHappy(t *testing.T, server string) { }, } - err := tc.k8sClient.Create(ctx, sqlActionInstance) - assert.Equal(nil, err, "create sqlaction in k8s") - - sqlActionInstanceNamespacedName := types.NamespacedName{Name: sqlActionName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlActionInstanceNamespacedName, sqlActionInstance) - return sqlActionInstance.Status.Provisioned - }, tc.timeout, tc.retry, "wait for sql action to be submitted") - - // TODO Check SQL Database credentials + EnsureInstance(ctx, t, tc, sqlActionInstance) // makre sure credentials are not the same as previous secretAfter := &v1.Secret{} assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, types.NamespacedName{Name: server, Namespace: "default"}, secretAfter) + err := tc.k8sClient.Get(ctx, types.NamespacedName{Name: server, Namespace: "default"}, secretAfter) if err != nil { return false } @@ -75,4 +65,6 @@ func RunSQLActionHappy(t *testing.T, server string) { assert.Equal(secret.Data["username"], secretAfter.Data["username"], "username should still be the same") assert.NotEqual(string(secret.Data["password"]), string(secretAfter.Data["password"]), "password should have changed") + + EnsureDelete(ctx, t, tc, sqlActionInstance) } diff --git a/controllers/azuresqldatabase_controller_test.go b/controllers/azuresqldatabase_controller_test.go index 5150b4efeca..7601267edb3 100644 --- a/controllers/azuresqldatabase_controller_test.go +++ b/controllers/azuresqldatabase_controller_test.go @@ -7,23 +7,18 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" - "github.com/stretchr/testify/assert" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestAzureSqlDatabaseControllerNoResourceGroup(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) // Add any setup steps that needs to be executed before each test rgLocation := tc.resourceGroupLocation @@ -43,32 +38,15 @@ func TestAzureSqlDatabaseControllerNoResourceGroup(t *testing.T) { Edition: 0, }, } + EnsureInstanceWithResult(ctx, t, tc, sqlDatabaseInstance, errhelp.ResourceGroupNotFoundErrorCode, false) - err := tc.k8sClient.Create(ctx, sqlDatabaseInstance) - assert.Equal(nil, err, "create db in k8s") - - sqlDatabaseNamespacedName := types.NamespacedName{Name: sqlDatabaseName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlDatabaseNamespacedName, sqlDatabaseInstance) - return strings.Contains(sqlDatabaseInstance.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for rg not found error") - - err = tc.k8sClient.Delete(ctx, sqlDatabaseInstance) - assert.Equal(nil, err, "delete db in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlDatabaseNamespacedName, sqlDatabaseInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for resource not found error") - + EnsureDelete(ctx, t, tc, sqlDatabaseInstance) } func TestAzureSqlDatabaseControllerNoServer(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) // Add any setup steps that needs to be executed before each test rgName := tc.resourceGroupName @@ -90,23 +68,8 @@ func TestAzureSqlDatabaseControllerNoServer(t *testing.T) { }, } - err := tc.k8sClient.Create(ctx, sqlDatabaseInstance) - assert.Equal(false, apierrors.IsInvalid(err), "create db resource") - assert.Equal(nil, err, "create db in k8s") - - sqlDatabaseNamespacedName := types.NamespacedName{Name: sqlDatabaseName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlDatabaseNamespacedName, sqlDatabaseInstance) - return strings.Contains(sqlDatabaseInstance.Status.Message, errhelp.ParentNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for rg not found error") - - err = tc.k8sClient.Delete(ctx, sqlDatabaseInstance) - assert.Equal(nil, err, "delete db in k8s") + EnsureInstanceWithResult(ctx, t, tc, sqlDatabaseInstance, errhelp.ParentNotFoundErrorCode, false) - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlDatabaseNamespacedName, sqlDatabaseInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for resource not found error") + EnsureDelete(ctx, t, tc, sqlDatabaseInstance) } diff --git a/controllers/azuresqlfailovergroup_controller_test.go b/controllers/azuresqlfailovergroup_controller_test.go index b20a26a6dd2..23248104150 100644 --- a/controllers/azuresqlfailovergroup_controller_test.go +++ b/controllers/azuresqlfailovergroup_controller_test.go @@ -7,30 +7,24 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/stretchr/testify/assert" "github.com/Azure/azure-service-operator/pkg/errhelp" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestAzureSqlFailoverGroupControllerNoResourceGroup(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) var rgName string var rgLocation1 string var sqlServerOneName string var sqlServerTwoName string var sqlDatabaseName string - var err error // Add any setup steps that needs to be executed before each test rgName = tc.resourceGroupName @@ -58,21 +52,7 @@ func TestAzureSqlFailoverGroupControllerNoResourceGroup(t *testing.T) { }, } - err = tc.k8sClient.Create(ctx, sqlFailoverGroupInstance) - assert.Equal(nil, err, "create failovergroup in k8s") + EnsureInstanceWithResult(ctx, t, tc, sqlFailoverGroupInstance, errhelp.ResourceGroupNotFoundErrorCode, false) - sqlFailoverGroupNamespacedName := types.NamespacedName{Name: sqlFailoverGroupName, Namespace: "default"} - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlFailoverGroupNamespacedName, sqlFailoverGroupInstance) - return strings.Contains(sqlFailoverGroupInstance.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for rg not found error to clear") - - err = tc.k8sClient.Delete(ctx, sqlFailoverGroupInstance) - assert.Equal(nil, err, "delete failovergroup in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlFailoverGroupNamespacedName, sqlFailoverGroupInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for failovergroup to be gone from k8s") + EnsureDelete(ctx, t, tc, sqlFailoverGroupInstance) } diff --git a/controllers/azuresqlfirewallrule_controller_test.go b/controllers/azuresqlfirewallrule_controller_test.go index a1b54a99a1b..eebbe717721 100644 --- a/controllers/azuresqlfirewallrule_controller_test.go +++ b/controllers/azuresqlfirewallrule_controller_test.go @@ -7,23 +7,18 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/stretchr/testify/assert" "github.com/Azure/azure-service-operator/pkg/errhelp" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestAzureSqlFirewallRuleControllerNoResourceGroup(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) // Add any setup steps that needs to be executed before each test //rgName := tc.resourceGroupName @@ -44,27 +39,8 @@ func TestAzureSqlFirewallRuleControllerNoResourceGroup(t *testing.T) { }, } - err := tc.k8sClient.Create(ctx, sqlFirewallRuleInstance) - assert.Equal(nil, err, "create sqlfirewallrule in k8s") + EnsureInstanceWithResult(ctx, t, tc, sqlFirewallRuleInstance, errhelp.ResourceGroupNotFoundErrorCode, false) - sqlFirewallRuleNamespacedName := types.NamespacedName{Name: sqlFirewallRuleName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlFirewallRuleNamespacedName, sqlFirewallRuleInstance) - return HasFinalizer(sqlFirewallRuleInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for firewallrule to have finalizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlFirewallRuleNamespacedName, sqlFirewallRuleInstance) - return strings.Contains(sqlFirewallRuleInstance.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for firewallrule to have rg not found error") - - err = tc.k8sClient.Delete(ctx, sqlFirewallRuleInstance) - assert.Equal(nil, err, "delete sqlfirewallrule in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlFirewallRuleNamespacedName, sqlFirewallRuleInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for firewallrule to be gone from k8s") + EnsureDelete(ctx, t, tc, sqlFirewallRuleInstance) } diff --git a/controllers/azuresqlserver_controller_test.go b/controllers/azuresqlserver_controller_test.go index 9eb84b99949..ecedcbe46a0 100644 --- a/controllers/azuresqlserver_controller_test.go +++ b/controllers/azuresqlserver_controller_test.go @@ -7,16 +7,11 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/stretchr/testify/assert" - "github.com/Azure/azure-service-operator/pkg/errhelp" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestAzureSqlServerControllerNoResourceGroup(t *testing.T) { @@ -39,27 +34,7 @@ func TestAzureSqlServerControllerNoResourceGroup(t *testing.T) { }, } - err := tc.k8sClient.Create(ctx, sqlServerInstance) - assert.Equal(nil, err, "create sql server in k8s") - - sqlServerNamespacedName := types.NamespacedName{Name: sqlServerName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlServerNamespacedName, sqlServerInstance) - return HasFinalizer(sqlServerInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for finalizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlServerNamespacedName, sqlServerInstance) - return strings.Contains(sqlServerInstance.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for rg error") - - err = tc.k8sClient.Delete(ctx, sqlServerInstance) - assert.Equal(nil, err, "delete sql server in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlServerNamespacedName, sqlServerInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for server to be gone") + EnsureInstanceWithResult(ctx, t, tc, sqlServerInstance, errhelp.ResourceGroupNotFoundErrorCode, false) + EnsureDelete(ctx, t, tc, sqlServerInstance) } diff --git a/controllers/azuresqluser_controller_test.go b/controllers/azuresqluser_controller_test.go index ba58fb5ac12..2c5b0f99558 100644 --- a/controllers/azuresqluser_controller_test.go +++ b/controllers/azuresqluser_controller_test.go @@ -48,30 +48,9 @@ func TestAzureSQLUserControllerNoAdminSecret(t *testing.T) { }, } - // Create the sqlUser - err = tc.k8sClient.Create(ctx, sqlUser) - assert.Equal(nil, err, "create db user in k8s") - - sqlUserNamespacedName := types.NamespacedName{Name: username, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlUserNamespacedName, sqlUser) - return HasFinalizer(sqlUser, finalizerName) - }, tc.timeout, tc.retry, "wait for finalizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlUserNamespacedName, sqlUser) - return strings.Contains(sqlUser.Status.Message, "admin secret") - }, tc.timeout, tc.retry, "wait for missing admin secret message") - - err = tc.k8sClient.Delete(ctx, sqlUser) - assert.Equal(nil, err, "delete db user in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlUserNamespacedName, sqlUser) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for user to be gone from k8s") + EnsureInstanceWithResult(ctx, t, tc, sqlUser, "admin secret", false) + EnsureDelete(ctx, t, tc, sqlUser) } func TestAzureSQLUserControllerNoResourceGroup(t *testing.T) { @@ -120,28 +99,8 @@ func TestAzureSQLUserControllerNoResourceGroup(t *testing.T) { }, } - // Create the sqlUser - err = tc.k8sClient.Create(ctx, sqlUser) - assert.Equal(nil, err, "create db user in k8s") - - sqlUserNamespacedName := types.NamespacedName{Name: username, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlUserNamespacedName, sqlUser) - return HasFinalizer(sqlUser, finalizerName) - }, tc.timeout, tc.retry, "wait for finalizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, sqlUserNamespacedName, sqlUser) - return strings.Contains(sqlUser.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for rg fail message") - - err = tc.k8sClient.Delete(ctx, sqlUser) - assert.Equal(nil, err, "delete db user in k8s") + EnsureInstanceWithResult(ctx, t, tc, sqlUser, errhelp.ResourceGroupNotFoundErrorCode, false) - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlUserNamespacedName, sqlUser) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for user to be gone from k8s") + EnsureDelete(ctx, t, tc, sqlUser) } diff --git a/controllers/azurevnetrule_controller_test.go b/controllers/azurevnetrule_controller_test.go index e4f7509b63e..a32f4007d9e 100644 --- a/controllers/azurevnetrule_controller_test.go +++ b/controllers/azurevnetrule_controller_test.go @@ -45,36 +45,9 @@ func TestAzureSqlVNetRuleControllerNoResourceGroup(t *testing.T) { }, } - err := tc.k8sClient.Create(ctx, sqlVNetRuleInstance) - assert.Equal(nil, err, "create sqlvnetrule in k8s") - - sqlVNETRuleNamespacedName := types.NamespacedName{Name: sqlVNetRuleName, Namespace: "default"} - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlVNETRuleNamespacedName, sqlVNetRuleInstance) - if err == nil { - return HasFinalizer(sqlVNetRuleInstance, finalizerName) - } else { - return false - } - }, tc.timeout, tc.retry, "wait for sqlvnetrule to have finalizer") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlVNETRuleNamespacedName, sqlVNetRuleInstance) - if err == nil { - return strings.Contains(sqlVNetRuleInstance.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) - } else { - return false - } - }, tc.timeout, tc.retry, "wait for sqlvnetrule to have rg not found error") - - err = tc.k8sClient.Delete(ctx, sqlVNetRuleInstance) - assert.Equal(nil, err, "delete sqlvnetrule in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, sqlVNETRuleNamespacedName, sqlVNetRuleInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for sqlvnetrule to be gone from k8s") + EnsureInstanceWithResult(ctx, t, tc, sqlVNetRuleInstance, errhelp.ResourceGroupNotFoundErrorCode, false) + + EnsureDelete(ctx, t, tc, sqlVNetRuleInstance) } func RunAzureSqlVNetRuleHappyPath(t *testing.T, sqlServerName string, rgLocation string) { diff --git a/controllers/blobcontainer_controller_test.go b/controllers/blobcontainer_controller_test.go index b73a56455a5..7289729ea30 100644 --- a/controllers/blobcontainer_controller_test.go +++ b/controllers/blobcontainer_controller_test.go @@ -7,27 +7,22 @@ package controllers import ( "context" - "strings" "testing" s "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" + "github.com/Azure/go-autorest/autorest/to" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" helpers "github.com/Azure/azure-service-operator/pkg/helpers" - "github.com/stretchr/testify/assert" - - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestBlobContainerControlleNoResourceGroup(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) var rgLocation string var saName string @@ -35,71 +30,30 @@ func TestBlobContainerControlleNoResourceGroup(t *testing.T) { // Add any setup steps that needs to be executed before each test rgLocation = tc.resourceGroupLocation - saName = tc.storageAccountName + saName = GenerateAlphaNumTestResourceName("blobsa") containerAccessLevel = s.PublicAccessContainer - // Add Tests for OpenAPI validation (or additonal CRD features) specified in - // your API definition. - // Avoid adding tests for vanilla CRUD operations because they would - // test Kubernetes API server, which isn't the goal here. - blobContainerName := GenerateTestResourceNameWithRandom("bc", 10) resourceGroupName := GenerateTestResourceNameWithRandom("rg", 10) - var err error - - blobContainerInstance := &azurev1alpha1.BlobContainer{ + // Create Storage account + saInstance := &azurev1alpha1.StorageAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: blobContainerName, + Name: saName, Namespace: "default", }, - Spec: azurev1alpha1.BlobContainerSpec{ - Location: rgLocation, - ResourceGroup: resourceGroupName, - AccountName: saName, - AccessLevel: containerAccessLevel, + Spec: azurev1alpha1.StorageAccountSpec{ + Location: tc.resourceGroupLocation, + ResourceGroup: tc.resourceGroupName, + Sku: azurev1alpha1.StorageAccountSku{ + Name: "Standard_RAGRS", + }, + Kind: "StorageV2", + AccessTier: "Hot", + EnableHTTPSTrafficOnly: to.BoolPtr(true), }, } - - err = tc.k8sClient.Create(ctx, blobContainerInstance) - assert.Equal(nil, err, "create blobcontainer in k8s") - - blobContainerNamespacedName := types.NamespacedName{Name: blobContainerName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, blobContainerNamespacedName, blobContainerInstance) - return strings.Contains(blobContainerInstance.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for blob to have rg not found error") - - err = tc.k8sClient.Delete(ctx, blobContainerInstance) - assert.Equal(nil, err, "delete blobcontainer in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, blobContainerNamespacedName, blobContainerInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for blob to be not found") - -} - -func TestTestBlobContainerControllerNoStorageAccount(t *testing.T) { - t.Parallel() - defer PanicRecover(t) - ctx := context.Background() - assert := assert.New(t) - - var rgLocation string - var rgName string - var containerAccessLevel s.PublicAccess - - // Add any setup steps that needs to be executed before each test - rgLocation = tc.resourceGroupLocation - rgName = tc.resourceGroupName - containerAccessLevel = s.PublicAccessContainer - - blobContainerName := GenerateTestResourceNameWithRandom("bc", 10) - storageAccountName := helpers.FillWithRandom(GenerateAlphaNumTestResourceName("sa"), 24) - - var err error + EnsureInstance(ctx, t, tc, saInstance) blobContainerInstance := &azurev1alpha1.BlobContainer{ ObjectMeta: metav1.ObjectMeta{ @@ -108,54 +62,32 @@ func TestTestBlobContainerControllerNoStorageAccount(t *testing.T) { }, Spec: azurev1alpha1.BlobContainerSpec{ Location: rgLocation, - ResourceGroup: rgName, - AccountName: storageAccountName, + ResourceGroup: resourceGroupName, + AccountName: saName, AccessLevel: containerAccessLevel, }, } + EnsureInstanceWithResult(ctx, t, tc, blobContainerInstance, errhelp.ResourceGroupNotFoundErrorCode, false) - err = tc.k8sClient.Create(ctx, blobContainerInstance) - assert.Equal(nil, err, "create blob container in k8s") - - blobContainerNamespacedName := types.NamespacedName{Name: blobContainerName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, blobContainerNamespacedName, blobContainerInstance) - return strings.Contains(blobContainerInstance.Status.Message, errhelp.ParentNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for blob to have parent not found error") - - err = tc.k8sClient.Delete(ctx, blobContainerInstance) - assert.Equal(nil, err, "delete blob container in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, blobContainerNamespacedName, blobContainerInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for blob to be not found") - + EnsureDelete(ctx, t, tc, blobContainerInstance) } -func TestTestBlobContainerControllerHappyPath(t *testing.T) { +func TestTestBlobContainerControllerNoStorageAccount(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() var rgLocation string var rgName string - var saName string var containerAccessLevel s.PublicAccess // Add any setup steps that needs to be executed before each test rgLocation = tc.resourceGroupLocation rgName = tc.resourceGroupName - saName = tc.storageAccountName containerAccessLevel = s.PublicAccessContainer - // Add Tests for OpenAPI validation (or additonal CRD features) specified in - // your API definition. - // Avoid adding tests for vanilla CRUD operations because they would - // test Kubernetes API server, which isn't the goal here. - blobContainerName := GenerateTestResourceNameWithRandom("bc", 10) + storageAccountName := helpers.FillWithRandom(GenerateAlphaNumTestResourceName("sa"), 24) blobContainerInstance := &azurev1alpha1.BlobContainer{ ObjectMeta: metav1.ObjectMeta{ @@ -165,13 +97,11 @@ func TestTestBlobContainerControllerHappyPath(t *testing.T) { Spec: azurev1alpha1.BlobContainerSpec{ Location: rgLocation, ResourceGroup: rgName, - AccountName: saName, + AccountName: storageAccountName, AccessLevel: containerAccessLevel, }, } - - EnsureInstance(ctx, t, tc, blobContainerInstance) + EnsureInstanceWithResult(ctx, t, tc, blobContainerInstance, errhelp.ParentNotFoundErrorCode, false) EnsureDelete(ctx, t, tc, blobContainerInstance) - } diff --git a/controllers/consumergroup_controller_test.go b/controllers/consumergroup_controller_test.go deleted file mode 100644 index 8d701c9bb88..00000000000 --- a/controllers/consumergroup_controller_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -// +build all consumergroup - -package controllers - -import ( - "context" - "net/http" - "testing" - - azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/stretchr/testify/assert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" -) - -func TestConsumerGroup(t *testing.T) { - t.Parallel() - assert := assert.New(t) - - var rgName string = tc.resourceGroupName - var ehnName string = tc.eventhubNamespaceName - var ehName string = tc.eventhubName - var ctx = context.Background() - defer PanicRecover(t) - - consumerGroupName := GenerateTestResourceNameWithRandom("cg", 10) - azureConsumerGroupName := consumerGroupName + "-azure" - - var err error - - // Create the consumer group object and expect the Reconcile to be created - consumerGroupInstance := &azurev1alpha1.ConsumerGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: consumerGroupName, - Namespace: "default", - }, - Spec: azurev1alpha1.ConsumerGroupSpec{ - Namespace: ehnName, - ResourceGroup: rgName, - Eventhub: ehName, - ConsumerGroupName: azureConsumerGroupName, - }, - } - - err = tc.k8sClient.Create(ctx, consumerGroupInstance) - assert.Equal(nil, err, "create consumergroup in k8s") - - consumerGroupNamespacedName := types.NamespacedName{Name: consumerGroupName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, consumerGroupNamespacedName, consumerGroupInstance) - return HasFinalizer(consumerGroupInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for finalizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, consumerGroupNamespacedName, consumerGroupInstance) - return consumerGroupInstance.Status.Provisioned - }, tc.timeout, tc.retry, "wait for provision") - - assert.Eventually(func() bool { - cg, _ := tc.consumerGroupClient.GetConsumerGroup(ctx, rgName, ehnName, ehName, azureConsumerGroupName) - return cg.Name != nil && *cg.Name == azureConsumerGroupName && cg.Response.StatusCode == http.StatusOK - }, tc.timeout, tc.retry, "wait for consumergroup to exist in Azure") - - err = tc.k8sClient.Delete(ctx, consumerGroupInstance) - assert.Equal(nil, err, "delete consumergroup in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, consumerGroupNamespacedName, consumerGroupInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for consumergroup to be gone from k8s") - - assert.Eventually(func() bool { - cg, _ := tc.consumerGroupClient.GetConsumerGroup(ctx, rgName, ehnName, ehName, azureConsumerGroupName) - return cg.Response.StatusCode != http.StatusOK - }, tc.timeout, tc.retry, "wait for consumergroup to be gone from azure") - -} diff --git a/controllers/eventhub_namespace_consumergroup_controller_test.go b/controllers/eventhub_namespace_consumergroup_controller_test.go new file mode 100644 index 00000000000..7e50b7a5d84 --- /dev/null +++ b/controllers/eventhub_namespace_consumergroup_controller_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// +build all consumergroup eventhub eventhubnamespace + +package controllers + +import ( + "context" + "net/http" + "testing" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestConsumerGroupEventHubAndNamespaceControllerHappy(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + assert := assert.New(t) + + var rgName string = tc.resourceGroupName + var rgLocation string = tc.resourceGroupLocation + eventhubNamespaceName := GenerateTestResourceNameWithRandom("ns-dev-eh", 10) + + consumerGroupName := GenerateTestResourceNameWithRandom("cg", 10) + azureConsumerGroupName := consumerGroupName + "-azure" + + // Create the Eventhub namespace object and expect the Reconcile to be created + eventhubNamespaceInstance := &azurev1alpha1.EventhubNamespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: eventhubNamespaceName, + Namespace: "default", + }, + Spec: azurev1alpha1.EventhubNamespaceSpec{ + Location: rgLocation, + ResourceGroup: rgName, + }, + } + + EnsureInstance(ctx, t, tc, eventhubNamespaceInstance) + + eventhubName := GenerateTestResourceNameWithRandom("eh-cd", 10) + + // Create the EventHub object and expect the Reconcile to be created + eventhubInstance := &azurev1alpha1.Eventhub{ + ObjectMeta: metav1.ObjectMeta{ + Name: eventhubName, + Namespace: "default", + }, + Spec: azurev1alpha1.EventhubSpec{ + Location: rgLocation, + Namespace: eventhubNamespaceName, + ResourceGroup: rgName, + Properties: azurev1alpha1.EventhubProperties{ + MessageRetentionInDays: 7, + PartitionCount: 2, + }, + AuthorizationRule: azurev1alpha1.EventhubAuthorizationRule{ + Name: "RootManageSharedAccessKey", + Rights: []string{"Listen"}, + }, + }, + } + + // verify eventhub is created successfully + EnsureInstance(ctx, t, tc, eventhubInstance) + + // Create a consumer group instance and verify + consumerGroupInstance := &azurev1alpha1.ConsumerGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: consumerGroupName, + Namespace: "default", + }, + Spec: azurev1alpha1.ConsumerGroupSpec{ + Namespace: eventhubNamespaceName, + ResourceGroup: rgName, + Eventhub: eventhubName, + ConsumerGroupName: azureConsumerGroupName, + }, + } + EnsureInstance(ctx, t, tc, consumerGroupInstance) + + assert.Eventually(func() bool { + cg, _ := tc.consumerGroupClient.GetConsumerGroup(ctx, rgName, eventhubNamespaceName, eventhubName, azureConsumerGroupName) + return cg.Name != nil && *cg.Name == azureConsumerGroupName && cg.Response.StatusCode == http.StatusOK + }, tc.timeout, tc.retry, "wait for consumergroup to exist in Azure") + + EnsureDelete(ctx, t, tc, consumerGroupInstance) + + assert.Eventually(func() bool { + cg, _ := tc.consumerGroupClient.GetConsumerGroup(ctx, rgName, eventhubNamespaceName, eventhubName, azureConsumerGroupName) + return cg.Response.StatusCode != http.StatusOK + }, tc.timeout, tc.retry, "wait for consumergroup to be gone from azure") + + // verify eventhub is deleted + EnsureDelete(ctx, t, tc, eventhubInstance) + + // verify eventhubnamespace gets deleted + EnsureDelete(ctx, t, tc, eventhubNamespaceInstance) +} diff --git a/controllers/eventhub_controller_test.go b/controllers/eventhub_storageaccount_controller_test.go similarity index 57% rename from controllers/eventhub_controller_test.go rename to controllers/eventhub_storageaccount_controller_test.go index 3e2597eee1b..9fdbbf6bd66 100644 --- a/controllers/eventhub_controller_test.go +++ b/controllers/eventhub_storageaccount_controller_test.go @@ -1,33 +1,31 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// +build all eventhub onlyeventhub +// +build all eventhub storage blobcontainer package controllers import ( "context" - "strings" + "fmt" "testing" + s "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/Azure/azure-service-operator/pkg/errhelp" - kvhelper "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" - kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" + //"github.com/Azure/azure-service-operator/pkg/errhelp" + + //kvhelper "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" + //kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" ) func TestEventHubControllerNoNamespace(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) // Add Tests for OpenAPI validation (or additonal CRD features) specified in // your API definition. @@ -53,96 +51,36 @@ func TestEventHubControllerNoNamespace(t *testing.T) { }, } - err := tc.k8sClient.Create(ctx, eventhubInstance) - assert.Equal(nil, err, "create eventhub in k8s") - - eventhubNamespacedName := types.NamespacedName{Name: eventhubName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, eventhubNamespacedName, eventhubInstance) - return strings.Contains(eventhubInstance.Status.Message, errhelp.ParentNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for eventhub to provision") - - err = tc.k8sClient.Delete(ctx, eventhubInstance) - assert.Equal(nil, err, "delete eventhub in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, eventhubNamespacedName, eventhubInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for eventHubInstance to be gone from k8s") + EnsureInstanceWithResult(ctx, t, tc, eventhubInstance, errhelp.ParentNotFoundErrorCode, false) + EnsureDelete(ctx, t, tc, eventhubInstance) } -func TestEventHubControllerCreateAndDelete(t *testing.T) { +func TestEventHubControllerCreateAndDeleteCustomSecret(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) // Add any setup steps that needs to be executed before each test rgName := tc.resourceGroupName - ehnName := tc.eventhubNamespaceName - eventhubName := GenerateTestResourceNameWithRandom("eh-cd", 10) + rgLocation := tc.resourceGroupLocation + ehnName := GenerateTestResourceNameWithRandom("eh-ns", 10) + eventhubName := GenerateTestResourceNameWithRandom("eh-customsec", 10) + secretName := "secret-" + eventhubName - // Create the EventHub object and expect the Reconcile to be created - eventhubInstance := &azurev1alpha1.Eventhub{ + // Create EventhubNamespace instance as prereq + eventhubNamespaceInstance := &azurev1alpha1.EventhubNamespace{ ObjectMeta: metav1.ObjectMeta{ - Name: eventhubName, + Name: ehnName, Namespace: "default", }, - Spec: azurev1alpha1.EventhubSpec{ - Location: "westus", - Namespace: ehnName, + Spec: azurev1alpha1.EventhubNamespaceSpec{ + Location: rgLocation, ResourceGroup: rgName, - Properties: azurev1alpha1.EventhubProperties{ - MessageRetentionInDays: 7, - PartitionCount: 2, - }, - AuthorizationRule: azurev1alpha1.EventhubAuthorizationRule{ - Name: "RootManageSharedAccessKey", - Rights: []string{"Listen"}, - }, }, } - err := tc.k8sClient.Create(ctx, eventhubInstance) - assert.Equal(nil, err, "create eventhub in k8s") - - eventhubNamespacedName := types.NamespacedName{Name: eventhubName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, eventhubNamespacedName, eventhubInstance) - return eventhubInstance.HasFinalizer(finalizerName) - }, tc.timeout, tc.retry, "wait for eventhub to have finalizer") - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, eventhubNamespacedName, eventhubInstance) - return strings.Contains(eventhubInstance.Status.Message, successMsg) - }, tc.timeout, tc.retry, "wait for eventhub to provision") - - err = tc.k8sClient.Delete(ctx, eventhubInstance) - assert.Equal(nil, err, "delete eventhub in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, eventhubNamespacedName, eventhubInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for eventHubInstance to be gone from k8s") - -} - -func TestEventHubControllerCreateAndDeleteCustomSecret(t *testing.T) { - t.Parallel() - defer PanicRecover(t) - ctx := context.Background() - assert := assert.New(t) - var err error - - // Add any setup steps that needs to be executed before each test - rgName := tc.resourceGroupName - rgLocation := tc.resourceGroupLocation - ehnName := tc.eventhubNamespaceName - eventhubName := GenerateTestResourceNameWithRandom("eh-customsec", 10) - secretName := "secret-" + eventhubName + EnsureInstance(ctx, t, tc, eventhubNamespaceInstance) // Create the EventHub object and expect the Reconcile to be created eventhubInstance := &azurev1alpha1.Eventhub{ @@ -168,15 +106,11 @@ func TestEventHubControllerCreateAndDeleteCustomSecret(t *testing.T) { EnsureInstance(ctx, t, tc, eventhubInstance) - eventhubNamespacedName := types.NamespacedName{Name: eventhubName, Namespace: "default"} + EnsureSecretsWithValue(ctx, t, tc, eventhubInstance, tc.secretClient, secretName, eventhubInstance.Namespace, "eventhubName", eventhubName) - err = tc.k8sClient.Delete(ctx, eventhubInstance) - assert.Equal(nil, err, "delete eventhub in k8s") + EnsureDelete(ctx, t, tc, eventhubInstance) - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, eventhubNamespacedName, eventhubInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for eventHubInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, eventhubNamespaceInstance) } func TestEventHubControllerCreateAndDeleteCustomKeyVault(t *testing.T) { @@ -188,7 +122,7 @@ func TestEventHubControllerCreateAndDeleteCustomKeyVault(t *testing.T) { // Add any setup steps that needs to be executed before each test rgName := tc.resourceGroupName rgLocation := tc.resourceGroupLocation - ehnName := tc.eventhubNamespaceName + ehnName := GenerateTestResourceNameWithRandom("eh-ns", 10) eventhubName := GenerateTestResourceNameWithRandom("ev", 10) keyVaultNameForSecrets := tc.keyvaultName @@ -196,6 +130,20 @@ func TestEventHubControllerCreateAndDeleteCustomKeyVault(t *testing.T) { _, err := kvhelper.AzureKeyVaultManager.GetVault(ctx, rgName, keyVaultNameForSecrets) assert.Equal(nil, err, "wait for keyvault to be available") + // Create EventhubNamespace instance as prereq + eventhubNamespaceInstance := &azurev1alpha1.EventhubNamespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ehnName, + Namespace: "default", + }, + Spec: azurev1alpha1.EventhubNamespaceSpec{ + Location: rgLocation, + ResourceGroup: rgName, + }, + } + + EnsureInstance(ctx, t, tc, eventhubNamespaceInstance) + // Create the EventHub object and expect the Reconcile to be created eventhubInstance := &azurev1alpha1.Eventhub{ ObjectMeta: metav1.ObjectMeta{ @@ -219,35 +167,76 @@ func TestEventHubControllerCreateAndDeleteCustomKeyVault(t *testing.T) { }, } - eventhubNamespacedName := types.NamespacedName{Name: eventhubName, Namespace: "default"} EnsureInstance(ctx, t, tc, eventhubInstance) // Check that the secret is added to KeyVault keyvaultSecretClient := kvsecrets.New(keyVaultNameForSecrets) - assert.Eventually(func() bool { - _, err = keyvaultSecretClient.Get(ctx, eventhubNamespacedName) - return err == nil - }, tc.timeout, tc.retry, "wait for secret to exist in keyvault") + EnsureSecrets(ctx, t, tc, eventhubInstance, keyvaultSecretClient, eventhubName, eventhubInstance.Namespace) EnsureDelete(ctx, t, tc, eventhubInstance) + EnsureDelete(ctx, t, tc, eventhubNamespaceInstance) } - -func TestEventHubControllerCreateAndDeleteCapture(t *testing.T) { +func TestEventHubCapture_StorageAccountAndBlob_Controllers(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() assert := assert.New(t) - var err error // Add any setup steps that needs to be executed before each test rgName := tc.resourceGroupName rgLocation := tc.resourceGroupLocation - ehnName := tc.eventhubNamespaceName - saName := tc.storageAccountName - bcName := tc.blobContainerName + ehnName := GenerateTestResourceNameWithRandom("eh-ns", 10) + saName := GenerateAlphaNumTestResourceName("ehsa") + bcName := GenerateTestResourceNameWithRandom("ehblob", 10) eventHubName := GenerateTestResourceNameWithRandom("eh-capture", 10) + containerAccessLevel := s.PublicAccessContainer + + // Create Eventhub namespace + eventhubNamespaceInstance := &azurev1alpha1.EventhubNamespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ehnName, + Namespace: "default", + }, + Spec: azurev1alpha1.EventhubNamespaceSpec{ + Location: rgLocation, + ResourceGroup: rgName, + }, + } + EnsureInstance(ctx, t, tc, eventhubNamespaceInstance) + + // Create Storage account + saInstance := &azurev1alpha1.StorageAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: "default", + }, + Spec: azurev1alpha1.StorageAccountSpec{ + Location: rgLocation, + ResourceGroup: rgName, + Sku: azurev1alpha1.StorageAccountSku{ + Name: "Standard_LRS", + }, + Kind: "Storage", + }, + } + EnsureInstance(ctx, t, tc, saInstance) + + // Create blob container + blobContainerInstance := &azurev1alpha1.BlobContainer{ + ObjectMeta: metav1.ObjectMeta{ + Name: bcName, + Namespace: "default", + }, + Spec: azurev1alpha1.BlobContainerSpec{ + Location: rgLocation, + ResourceGroup: rgName, + AccountName: saName, + AccessLevel: containerAccessLevel, + }, + } + EnsureInstance(ctx, t, tc, blobContainerInstance) // Create the EventHub object and expect the Reconcile to be created eventhubInstance := &azurev1alpha1.Eventhub{ @@ -290,8 +279,6 @@ func TestEventHubControllerCreateAndDeleteCapture(t *testing.T) { EnsureInstance(ctx, t, tc, eventhubInstance) - eventHubNamespacedName := types.NamespacedName{Name: eventHubName, Namespace: "default"} - assert.Eventually(func() bool { hub, _ := tc.eventhubClient.GetHub(ctx, rgName, ehnName, eventHubName) if hub.Properties == nil || hub.CaptureDescription == nil || hub.CaptureDescription.Enabled == nil { @@ -300,12 +287,8 @@ func TestEventHubControllerCreateAndDeleteCapture(t *testing.T) { return *hub.CaptureDescription.Enabled }, tc.timeout, tc.retry, "wait for eventhub capture check") - err = tc.k8sClient.Delete(ctx, eventhubInstance) - assert.Equal(nil, err, "delete eventhub in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, eventHubNamespacedName, eventhubInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for eventHubInstance to be gone from k8s") - + EnsureDelete(ctx, t, tc, eventhubInstance) + EnsureDelete(ctx, t, tc, eventhubNamespaceInstance) + EnsureDelete(ctx, t, tc, blobContainerInstance) + EnsureDelete(ctx, t, tc, saInstance) } diff --git a/controllers/eventhubnamespace_controller_test.go b/controllers/eventhubnamespace_controller_test.go index 1a0ad692757..ffc3817d710 100644 --- a/controllers/eventhubnamespace_controller_test.go +++ b/controllers/eventhubnamespace_controller_test.go @@ -12,11 +12,8 @@ import ( azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/Azure/azure-service-operator/pkg/errhelp" config "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" - "github.com/stretchr/testify/assert" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestEventHubNamespaceControllerNoResourceGroup(t *testing.T) { @@ -26,12 +23,6 @@ func TestEventHubNamespaceControllerNoResourceGroup(t *testing.T) { var rgLocation string rgLocation = tc.resourceGroupLocation - - // Add Tests for OpenAPI validation (or additonal CRD features) specified in - // your API definition. - // Avoid adding tests for vanilla CRUD operations because they would - // test Kubernetes API server, which isn't the goal here. - // setting this rg name tells the mocks to set a proper error resourceGroupName := "gone" eventhubNamespaceName := GenerateTestResourceNameWithRandom("ns-dev-eh", 10) @@ -51,10 +42,9 @@ func TestEventHubNamespaceControllerNoResourceGroup(t *testing.T) { EnsureInstanceWithResult(ctx, t, tc, eventhubNamespaceInstance, errhelp.ResourceGroupNotFoundErrorCode, false) EnsureDelete(ctx, t, tc, eventhubNamespaceInstance) - } -func TestEventHubNamespaceControllerBasicTierNetworkRule(t *testing.T) { +func TestEventHubNamespaceControllerNetworkRules(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() @@ -70,7 +60,7 @@ func TestEventHubNamespaceControllerBasicTierNetworkRule(t *testing.T) { SubnetAddressPrefix: "110.1.0.0/16", } - // Create a VNET + // Create a VNET as prereq VNetInstance := &azurev1alpha1.VirtualNetwork{ ObjectMeta: metav1.ObjectMeta{ Name: VNetName, @@ -86,7 +76,7 @@ func TestEventHubNamespaceControllerBasicTierNetworkRule(t *testing.T) { EnsureInstance(ctx, t, tc, VNetInstance) - // Create EventhubNamespace network rule for this namespace and expect success + // Create EventhubNamespace network rule using the above VNET subnetID := "/subscriptions/" + config.SubscriptionID() + "/resourceGroups/" + rgName + "/providers/Microsoft.Network/virtualNetworks/" + VNetName + "/subnets/" + subnetName vnetRules := []azurev1alpha1.VirtualNetworkRules{ { @@ -103,8 +93,8 @@ func TestEventHubNamespaceControllerBasicTierNetworkRule(t *testing.T) { eventhubNamespaceName := GenerateTestResourceNameWithRandom("ns-dev-eh", 10) - // Create the Eventhub namespace object as prereq - eventhubNamespaceInstance := &azurev1alpha1.EventhubNamespace{ + // Add this to the Eventhubnamespace that is Basic Tier and expect to fail + eventhubNamespaceInstance1 := &azurev1alpha1.EventhubNamespace{ ObjectMeta: metav1.ObjectMeta{ Name: eventhubNamespaceName, Namespace: "default", @@ -125,99 +115,12 @@ func TestEventHubNamespaceControllerBasicTierNetworkRule(t *testing.T) { }, } - EnsureInstanceWithResult(ctx, t, tc, eventhubNamespaceInstance, errhelp.BadRequest, false) - - // Delete the namespace - EnsureDelete(ctx, t, tc, eventhubNamespaceInstance) - -} - -func TestEventHubNamespaceControllerHappy(t *testing.T) { - t.Parallel() - defer PanicRecover(t) - ctx := context.Background() - assert := assert.New(t) - var err error - - var rgName string = tc.resourceGroupName - var rgLocation string = tc.resourceGroupLocation - eventhubNamespaceName := GenerateTestResourceNameWithRandom("ns-dev-eh", 10) - - // Create the Eventhub namespace object and expect the Reconcile to be created - eventhubNamespaceInstance := &azurev1alpha1.EventhubNamespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: eventhubNamespaceName, - Namespace: "default", - }, - Spec: azurev1alpha1.EventhubNamespaceSpec{ - Location: rgLocation, - ResourceGroup: rgName, - }, - } - - EnsureInstance(ctx, t, tc, eventhubNamespaceInstance) - - eventhubNamespacedName := types.NamespacedName{Name: eventhubNamespaceName, Namespace: "default"} - - err = tc.k8sClient.Delete(ctx, eventhubNamespaceInstance) - assert.Equal(nil, err, "delete eventhubns in k8s") - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, eventhubNamespacedName, eventhubNamespaceInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for eventHubnamespaceInstance to be gone from k8s") -} - -func TestEventHubNamespaceControllerHappyWithNetworkRule(t *testing.T) { - t.Parallel() - defer PanicRecover(t) - ctx := context.Background() - - var rgName string = tc.resourceGroupName - var rgLocation string = tc.resourceGroupLocation - // Create a VNET as prereq for the test - VNetName := GenerateTestResourceNameWithRandom("vnet", 10) - subnetName := "subnet-test" - VNetSubNetInstance := azurev1alpha1.VNetSubnets{ - SubnetName: subnetName, - SubnetAddressPrefix: "110.1.0.0/16", - } - - // Create a VNET - VNetInstance := &azurev1alpha1.VirtualNetwork{ - ObjectMeta: metav1.ObjectMeta{ - Name: VNetName, - Namespace: "default", - }, - Spec: azurev1alpha1.VirtualNetworkSpec{ - Location: rgLocation, - ResourceGroup: rgName, - AddressSpace: "110.0.0.0/8", - Subnets: []azurev1alpha1.VNetSubnets{VNetSubNetInstance}, - }, - } - - EnsureInstance(ctx, t, tc, VNetInstance) + EnsureInstanceWithResult(ctx, t, tc, eventhubNamespaceInstance1, errhelp.BadRequest, false) - // Create EventhubNamespace network rule for this namespace and expect success - subnetID := "/subscriptions/" + config.SubscriptionID() + "/resourceGroups/" + rgName + "/providers/Microsoft.Network/virtualNetworks/" + VNetName + "/subnets/" + subnetName - vnetRules := []azurev1alpha1.VirtualNetworkRules{ - { - SubnetID: subnetID, - IgnoreMissingServiceEndpoint: true, - }, - } - ipmask := "1.1.1.1" - ipRules := []azurev1alpha1.IPRules{ - { - IPMask: &ipmask, - }, - } - - eventhubNamespaceName := GenerateTestResourceNameWithRandom("ns-dev-eh", 10) + eventhubNamespaceName = GenerateTestResourceNameWithRandom("ns-dev-eh", 10) - // Create the Eventhub namespace object as prereq - eventhubNamespaceInstance := &azurev1alpha1.EventhubNamespace{ + // Add the network rule to a namespace that is Standard SKU and expect to pass + eventhubNamespaceInstance2 := &azurev1alpha1.EventhubNamespace{ ObjectMeta: metav1.ObjectMeta{ Name: eventhubNamespaceName, Namespace: "default", @@ -238,9 +141,14 @@ func TestEventHubNamespaceControllerHappyWithNetworkRule(t *testing.T) { }, } - EnsureInstance(ctx, t, tc, eventhubNamespaceInstance) + EnsureInstance(ctx, t, tc, eventhubNamespaceInstance2) // Delete the namespace - EnsureDelete(ctx, t, tc, eventhubNamespaceInstance) + EnsureDelete(ctx, t, tc, eventhubNamespaceInstance2) + + // Delete the namespace + EnsureDelete(ctx, t, tc, eventhubNamespaceInstance1) + + EnsureDelete(ctx, t, tc, VNetInstance) } diff --git a/controllers/helpers.go b/controllers/helpers.go index 22153b36a2a..a542b41c9a0 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -43,16 +43,16 @@ import ( ) type TestContext struct { - k8sClient client.Client - secretClient secrets.SecretClient - resourceNamePrefix string - resourceGroupName string - resourceGroupLocation string - eventhubNamespaceName string - eventhubName string - namespaceLocation string - storageAccountName string - blobContainerName string + k8sClient client.Client + secretClient secrets.SecretClient + resourceNamePrefix string + resourceGroupName string + resourceGroupLocation string + //eventhubNamespaceName string + //eventhubName string + //namespaceLocation string + //storageAccountName string + //blobContainerName string keyvaultName string resourceGroupManager resourcegroupsresourcemanager.ResourceGroupManager redisCacheManager resourcemanagerrediscaches.RedisCacheManager @@ -299,6 +299,47 @@ func EnsureDelete(ctx context.Context, t *testing.T, tc TestContext, instance ru } +func EnsureSecrets(ctx context.Context, t *testing.T, tc TestContext, instance runtime.Object, secretclient secrets.SecretClient, secretname string, secretnamespace string) { + assert := assert.New(t) + typeOf := fmt.Sprintf("%T", instance) + + key := types.NamespacedName{Name: secretname, Namespace: secretnamespace} + + // Wait for secret + err := helpers.Retry(tc.timeoutFast, tc.retry, func() error { + + _, err := secretclient.Get(ctx, key) + if err != nil { + return fmt.Errorf("secret with name %s does not exist", key.String()) + } + return nil + }) + assert.Nil(err, "error waiting for %s to have secret", typeOf) + +} +func EnsureSecretsWithValue(ctx context.Context, t *testing.T, tc TestContext, instance runtime.Object, secretclient secrets.SecretClient, secretname string, secretnamespace string, secretkey string, secretvalue string) { + assert := assert.New(t) + typeOf := fmt.Sprintf("%T", instance) + + key := types.NamespacedName{Name: secretname, Namespace: secretnamespace} + + // Wait for secret + err := helpers.Retry(tc.timeoutFast, tc.retry, func() error { + + secrets, err := secretclient.Get(ctx, key) + if err != nil { + return err + } + if !strings.Contains(string(secrets[secretkey]), secretvalue) { + return fmt.Errorf("secret with key %s not equal to %s", secretname, secretvalue) + } + + return nil + }) + assert.Nil(err, "error waiting for %s to have correct secret", typeOf) + +} + func RequireInstance(ctx context.Context, t *testing.T, tc TestContext, instance runtime.Object) { RequireInstanceWithResult(ctx, t, tc, instance, successMsg, true) } diff --git a/controllers/keyvault_controller_test.go b/controllers/keyvault_controller_test.go index 1620b901c88..2398bf6ba87 100644 --- a/controllers/keyvault_controller_test.go +++ b/controllers/keyvault_controller_test.go @@ -254,36 +254,9 @@ func TestKeyvaultControllerInvalidName(t *testing.T) { }, } - // Create the Keyvault object and expect the Reconcile to be created - err := tc.k8sClient.Create(ctx, keyVaultInstance) - assert.Equal(nil, err, "create keyvault in k8s") - - // Prep query for get - keyVaultNamespacedName := types.NamespacedName{Name: keyVaultName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, keyVaultNamespacedName, keyVaultInstance) - return HasFinalizer(keyVaultInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for keyvault to have finalizer") - - // Verify you get the invalid name error - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, keyVaultNamespacedName, keyVaultInstance) - return strings.Contains(keyVaultInstance.Status.Message, errhelp.AccountNameInvalid) - }, tc.timeout, tc.retry, "wait for invalid account name error") - - // delete key vault - err = tc.k8sClient.Delete(ctx, keyVaultInstance) - assert.Equal(nil, err, "delete keyvault in k8s") - - // verify key vault is gone from kubernetes - - assert.Eventually(func() bool { - err := tc.k8sClient.Get(ctx, keyVaultNamespacedName, keyVaultInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for keyVaultInstance to be gone from k8s") + EnsureInstanceWithResult(ctx, t, tc, keyVaultInstance, errhelp.AccountNameInvalid, false) + EnsureDelete(ctx, t, tc, keyVaultInstance) } func TestKeyvaultControllerNoResourceGroup(t *testing.T) { @@ -308,35 +281,9 @@ func TestKeyvaultControllerNoResourceGroup(t *testing.T) { }, } - // Create the Keyvault object and expect the Reconcile to be created - err := tc.k8sClient.Create(ctx, keyVaultInstance) - assert.Equal(nil, err, "create keyvault in k8s") - - // Prep query for get - keyVaultNamespacedName := types.NamespacedName{Name: keyVaultName, Namespace: "default"} - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, keyVaultNamespacedName, keyVaultInstance) - return HasFinalizer(keyVaultInstance, finalizerName) - }, tc.timeout, tc.retry, "wait for keyvault to have finalizer") - - // Verify you get the resource group not found error + EnsureInstanceWithResult(ctx, t, tc, keyVaultInstance, errhelp.ResourceGroupNotFoundErrorCode, false) - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, keyVaultNamespacedName, keyVaultInstance) - return strings.Contains(keyVaultInstance.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) - }, tc.timeout, tc.retry, "wait for ResourceGroupNotFound error") - - // delete key vault - err = tc.k8sClient.Delete(ctx, keyVaultInstance) - assert.Equal(nil, err, "delete keyvault in k8s") - - // verify key vault is gone from kubernetes - - assert.Eventually(func() bool { - err := tc.k8sClient.Get(ctx, keyVaultNamespacedName, keyVaultInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for keyVaultInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, keyVaultInstance) } diff --git a/controllers/rediscache_controller_test.go b/controllers/rediscache_controller_test.go index 3946a682a66..1be7f6b5e0c 100644 --- a/controllers/rediscache_controller_test.go +++ b/controllers/rediscache_controller_test.go @@ -61,42 +61,11 @@ func TestRedisCacheControllerHappyPath(t *testing.T) { } // create rc - //EnsureInstance(ctx, t, tc, redisCacheInstance) + EnsureInstance(ctx, t, tc, redisCacheInstance) - err = tc.k8sClient.Create(ctx, redisCacheInstance) - assert.Equal(nil, err, "create redis cache in k8s") - - names := types.NamespacedName{Name: redisCacheInstance.GetName(), Namespace: redisCacheInstance.GetNamespace()} - - // Wait for first sql server to resolve - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, names, redisCacheInstance) - return HasFinalizer(redisCacheInstance, finalizerName) - }, tc.timeoutFast, tc.retry, fmt.Sprintf("wait for %s to have finalizer", "rediscache")) - - assert.Eventually(func() bool { - _ = tc.k8sClient.Get(ctx, names, redisCacheInstance) - return strings.Contains(redisCacheInstance.Status.Message, successMsg) && redisCacheInstance.Status.Provisioned == true - }, longRunningTimeout, tc.retry, fmt.Sprintf("wait for %s to provision", "rediscache")) - - //verify secret exists in k8s for rc - secret := &v1.Secret{} - assert.Eventually(func() bool { - log.Println("get secret") - err = tc.k8sClient.Get(ctx, types.NamespacedName{Name: redisCacheInstance.Name, Namespace: redisCacheInstance.Namespace}, secret) - if err == nil { - return true - } - return false - }, 45*time.Second, tc.retry, "wait for rc to have secret") + // verify secret exists in secretclient + EnsureSecrets(ctx, t, tc, redisCacheInstance, tc.SecretClient, redisCacheInstance.Name, redisCacheInstance.Namespace) // delete rc - err = tc.k8sClient.Delete(ctx, redisCacheInstance) - assert.Equal(nil, err, fmt.Sprintf("delete %s in k8s", "rediscache")) - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, names, redisCacheInstance) - return apierrors.IsNotFound(err) - }, longRunningTimeout, tc.retry, fmt.Sprintf("wait for %s to be gone from k8s", "rediscache")) - + EnsureDelete(ctx, t, tc, redisCacheInstance) } diff --git a/controllers/resourcegroup_controller_test.go b/controllers/resourcegroup_controller_test.go index 17bd5f84664..436b1f96a9c 100644 --- a/controllers/resourcegroup_controller_test.go +++ b/controllers/resourcegroup_controller_test.go @@ -45,22 +45,13 @@ func TestResourceGroupControllerHappyPath(t *testing.T) { resourceGroupNamespacedName := types.NamespacedName{Name: resourceGroupName, Namespace: "default"} // verify rg exists in azure - assert.Eventually(func() bool { _, err := tc.resourceGroupManager.CheckExistence(ctx, resourceGroupName) return err == nil }, tc.timeout, tc.retry, "wait for resourceGroupInstance to exist in azure") // delete rg - err = tc.k8sClient.Delete(ctx, resourceGroupInstance) - assert.Equal(nil, err, "delete rg in k8s") - - // verify rg is being deleted - - assert.Eventually(func() bool { - err = tc.k8sClient.Get(ctx, resourceGroupNamespacedName, resourceGroupInstance) - return apierrors.IsNotFound(err) - }, tc.timeout, tc.retry, "wait for resourceGroupInstance to be gone from k8s") + EnsureDelete(ctx, t, tc, resourceGroupInstance) assert.Eventually(func() bool { result, _ := tc.resourceGroupManager.CheckExistence(ctx, resourceGroupName) diff --git a/controllers/storageaccount_controller_test.go b/controllers/storageaccount_controller_test.go deleted file mode 100644 index 6b82b83af33..00000000000 --- a/controllers/storageaccount_controller_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -// +build all storage - -package controllers - -import ( - "context" - "testing" - - azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/Azure/go-autorest/autorest/to" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestStorageControllerHappyPathWithoutNetworkRule(t *testing.T) { - t.Parallel() - defer PanicRecover(t) - ctx := context.Background() - StorageAccountName := GenerateAlphaNumTestResourceName("sadev") - // Create the ResourceGroup object and expect the Reconcile to be created - saInstance := &azurev1alpha1.StorageAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: StorageAccountName, - Namespace: "default", - }, - Spec: azurev1alpha1.StorageAccountSpec{ - Location: tc.resourceGroupLocation, - ResourceGroup: tc.resourceGroupName, - Sku: azurev1alpha1.StorageAccountSku{ - Name: "Standard_RAGRS", - }, - Kind: "StorageV2", - AccessTier: "Hot", - EnableHTTPSTrafficOnly: to.BoolPtr(true), - }, - } - // create rg - EnsureInstance(ctx, t, tc, saInstance) - // delete rg - EnsureDelete(ctx, t, tc, saInstance) -} diff --git a/controllers/suite_test.go b/controllers/suite_test.go index f3fb0123e48..25926076599 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -15,7 +15,6 @@ import ( "testing" "time" - s "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" k8sSecrets "github.com/Azure/azure-service-operator/pkg/secrets/kube" "k8s.io/client-go/rest" @@ -74,13 +73,13 @@ func setup() error { resourceGroupName := GenerateTestResourceName("rg-prime") resourcegroupLocation := resourcemanagerconfig.DefaultLocation() - eventhubNamespaceName := GenerateTestResourceName("evns-prime") - eventhubName := GenerateTestResourceName("ev-prime") - namespaceLocation := resourcemanagerconfig.DefaultLocation() + //eventhubNamespaceName := GenerateTestResourceName("evns-prime") + //eventhubName := GenerateTestResourceName("ev-prime") + //namespaceLocation := resourcemanagerconfig.DefaultLocation() - storageAccountName := GenerateAlphaNumTestResourceName("saprime") - blobContainerName := GenerateTestResourceName("blob-prime") - containerAccessLevel := s.PublicAccessContainer + //storageAccountName := GenerateAlphaNumTestResourceName("saprime") + //blobContainerName := GenerateTestResourceName("blob-prime") + //containerAccessLevel := s.PublicAccessContainer keyvaultName := GenerateAlphaNumTestResourceName("kv-prime") @@ -684,25 +683,26 @@ func setup() error { _, _ = resourceGroupManager.CreateGroup(context.Background(), resourceGroupName, resourcegroupLocation) } - log.Println("Creating EHNS:", eventhubNamespaceName) - eventHubNSManager := eventHubManagers.EventHubNamespace + /* + log.Println("Creating EHNS:", eventhubNamespaceName) + eventHubNSManager := eventHubManagers.EventHubNamespace - // Create the Eventhub namespace resource - _, err = eventHubNSManager.CreateNamespaceAndWait(context.Background(), resourceGroupName, eventhubNamespaceName, namespaceLocation) - if err != nil { - return err - } + // Create the Eventhub namespace resource + _, err = eventHubNSManager.CreateNamespaceAndWait(context.Background(), resourceGroupName, eventhubNamespaceName, namespaceLocation) + if err != nil { + return err + }*/ tc = TestContext{ - k8sClient: k8sClient, - secretClient: secretClient, - resourceGroupName: resourceGroupName, - resourceGroupLocation: resourcegroupLocation, - eventhubNamespaceName: eventhubNamespaceName, - eventhubName: eventhubName, - namespaceLocation: namespaceLocation, - storageAccountName: storageAccountName, - blobContainerName: blobContainerName, + k8sClient: k8sClient, + secretClient: secretClient, + resourceGroupName: resourceGroupName, + resourceGroupLocation: resourcegroupLocation, + //eventhubNamespaceName: eventhubNamespaceName, + //eventhubName: eventhubName, + //namespaceLocation: namespaceLocation, + //storageAccountName: storageAccountName, + //blobContainerName: blobContainerName, keyvaultName: keyvaultName, eventHubManagers: eventHubManagers, eventhubClient: eventhubClient, @@ -720,60 +720,60 @@ func setup() error { retry: time.Second * 3, consumerGroupClient: consumerGroupClient, } + /* + var pstate *string + finish := time.Now().Add(tc.timeout) + for { + if finish.Before(time.Now()) { + return fmt.Errorf("time out waiting for eventhub namespace") + } - var pstate *string - finish := time.Now().Add(tc.timeout) - for { - if finish.Before(time.Now()) { - return fmt.Errorf("time out waiting for eventhub namespace") + namespace, _ := eventHubManagers.EventHubNamespace.GetNamespace(context.Background(), resourceGroupName, eventhubNamespaceName) + pstate = namespace.ProvisioningState + if pstate != nil && *pstate == "Succeeded" { + break + } + time.Sleep(tc.retry) } - namespace, _ := eventHubManagers.EventHubNamespace.GetNamespace(context.Background(), resourceGroupName, eventhubNamespaceName) - pstate = namespace.ProvisioningState - if pstate != nil && *pstate == "Succeeded" { - break + log.Println("Creating EH:", eventhubName) + // Create the Eventhub resource + _, err = eventHubManagers.EventHub.CreateHub(context.Background(), resourceGroupName, eventhubNamespaceName, eventhubName, int32(7), int32(2), nil) + if err != nil { + return err } - time.Sleep(tc.retry) - } - log.Println("Creating EH:", eventhubName) - // Create the Eventhub resource - _, err = eventHubManagers.EventHub.CreateHub(context.Background(), resourceGroupName, eventhubNamespaceName, eventhubName, int32(7), int32(2), nil) - if err != nil { - return err - } + log.Println("Creating SA:", storageAccountName) + // Create the Storage Account and Container + _, _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageAccountSku{ + Name: "Standard_LRS", + }, "Storage", map[string]*string{}, "", nil, nil, nil) - log.Println("Creating SA:", storageAccountName) - // Create the Storage Account and Container - _, _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageAccountSku{ - Name: "Standard_LRS", - }, "Storage", map[string]*string{}, "", nil, nil, nil) + // Storage account needs to be in "Suceeded" state + // for container create to succeed + finish = time.Now().Add(tc.timeout) + for { - // Storage account needs to be in "Suceeded" state - // for container create to succeed - finish = time.Now().Add(tc.timeout) - for { + if finish.Before(time.Now()) { + return fmt.Errorf("time out waiting for storage account") + } - if finish.Before(time.Now()) { - return fmt.Errorf("time out waiting for storage account") + result, _ := storageAccountManager.GetStorage(context.Background(), resourceGroupName, storageAccountName) + if result.ProvisioningState == s.Succeeded { + break + } + time.Sleep(tc.retry) } - result, _ := storageAccountManager.GetStorage(context.Background(), resourceGroupName, storageAccountName) - if result.ProvisioningState == s.Succeeded { - break - } - time.Sleep(tc.retry) - } - - _, err = storageManagers.BlobContainer.CreateBlobContainer(context.Background(), resourceGroupName, storageAccountName, blobContainerName, containerAccessLevel) - if err != nil { - return err - } + _, err = storageManagers.BlobContainer.CreateBlobContainer(context.Background(), resourceGroupName, storageAccountName, blobContainerName, containerAccessLevel) + if err != nil { + return err + }*/ log.Println("Creating KV:", keyvaultName) _, err = resourcemanagerkeyvaults.AzureKeyVaultManager.CreateVaultWithAccessPolicies(context.Background(), resourceGroupName, keyvaultName, resourcegroupLocation, resourcemanagerconfig.ClientID()) // Key Vault needs to be in "Suceeded" state - finish = time.Now().Add(tc.timeout) + finish := time.Now().Add(tc.timeout) for { if finish.Before(time.Now()) { return fmt.Errorf("time out waiting for keyvault") diff --git a/pkg/resourcemanager/storages/blobcontainer/blob_container.go b/pkg/resourcemanager/storages/blobcontainer/blob_container.go index 2b41cceb721..2096fcb3404 100644 --- a/pkg/resourcemanager/storages/blobcontainer/blob_container.go +++ b/pkg/resourcemanager/storages/blobcontainer/blob_container.go @@ -5,8 +5,6 @@ package blobcontainer import ( "context" - "fmt" - "log" s "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" @@ -68,8 +66,6 @@ func (bc *AzureBlobContainerManager) GetBlobContainer(ctx context.Context, resou // containerName - the name of the container func (bc *AzureBlobContainerManager) DeleteBlobContainer(ctx context.Context, resourceGroupName string, accountName string, containerName string) (result autorest.Response, err error) { containerClient := getContainerClient() - log.Println(fmt.Sprintf("Deleting blob container '%s' for resource group: %s", containerName, accountName)) - return containerClient.Delete(ctx, resourceGroupName, accountName, diff --git a/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go b/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go index ea7c46f44a6..619c6c453f5 100644 --- a/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go +++ b/pkg/resourcemanager/storages/blobcontainer/blob_container_reconcile.go @@ -106,6 +106,7 @@ func (bc *AzureBlobContainerManager) Delete(ctx context.Context, obj runtime.Obj catch = []string{ errhelp.ParentNotFoundErrorCode, errhelp.ResourceGroupNotFoundErrorCode, + errhelp.NotFoundErrorCode, } if helpers.ContainsString(catch, azerr.Type) { return false, nil diff --git a/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go index 45efda9e48c..f9ee1bd5235 100644 --- a/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go +++ b/pkg/resourcemanager/storages/storageaccount/storageaccount_reconcile.go @@ -168,11 +168,14 @@ func (sa *azureStorageManager) Delete(ctx context.Context, obj runtime.Object, o name := instance.ObjectMeta.Name groupName := instance.Spec.ResourceGroup + _, err = sa.DeleteStorage(ctx, groupName, name) if err != nil { catch := []string{ errhelp.ValidationError, + errhelp.ResourceGroupNotFoundErrorCode, } + err = errhelp.NewAzureError(err) if azerr, ok := err.(*errhelp.AzureError); ok { if helpers.ContainsString(catch, azerr.Type) { From d259e00459cad94b81da3e5cfa16b55570cffb87 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Mon, 20 Apr 2020 23:33:58 -0600 Subject: [PATCH 19/27] PR comments --- pkg/resourcemanager/psql/server/server_reconcile.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/resourcemanager/psql/server/server_reconcile.go b/pkg/resourcemanager/psql/server/server_reconcile.go index b879adac85a..68f4bc235b9 100644 --- a/pkg/resourcemanager/psql/server/server_reconcile.go +++ b/pkg/resourcemanager/psql/server/server_reconcile.go @@ -61,13 +61,11 @@ func (p *PSQLServerClient) Ensure(ctx context.Context, obj runtime.Object, opts instance.Status.ResourceId = *getServer.ID instance.Status.Provisioned = true instance.Status.Provisioning = false - instance.Status.State = string(getServer.UserVisibleState) return true, nil } // the database exists but has not provisioned yet - so keep waiting instance.Status.Message = "Postgres server exists but may not be ready" - instance.Status.State = string(getServer.UserVisibleState) return false, nil } From 00c8187b03ff0ee6a2f9890585cef9e182ee2366 Mon Sep 17 00:00:00 2001 From: jpflueger Date: Tue, 21 Apr 2020 10:07:55 -0600 Subject: [PATCH 20/27] cosmosdb: allowing ip firewall rules --- api/v1alpha1/cosmosdb_types.go | 1 + config/samples/azure_v1alpha1_cosmosdb.yaml | 13 +++++++++---- pkg/resourcemanager/cosmosdbs/cosmosdb.go | 9 +++++++++ pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go | 2 +- pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go | 3 ++- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index 1a0c32794fa..321d1707577 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -23,6 +23,7 @@ type CosmosDBSpec struct { Properties CosmosDBProperties `json:"properties,omitempty"` VirtualNetworkRules *[]CosmosDBVirtualNetworkRule `json:"virtualNetworkRules,omitempty"` KeyVaultToStoreSecrets string `json:"keyVaultToStoreSecrets,omitempty"` + IPRules *[]string `json:"ipRules,omitempty"` } // CosmosDBKind enumerates the values for kind. diff --git a/config/samples/azure_v1alpha1_cosmosdb.yaml b/config/samples/azure_v1alpha1_cosmosdb.yaml index 2442ba22685..af960a5c725 100644 --- a/config/samples/azure_v1alpha1_cosmosdb.yaml +++ b/config/samples/azure_v1alpha1_cosmosdb.yaml @@ -8,7 +8,7 @@ spec: resourceGroup: resourcegroup-azure-operators properties: databaseAccountOfferType: Standard - enableMultipleWriteLocations: false + #enableMultipleWriteLocations: false # optionally set the mongoDBVersion to "3.2" or "3.6", if omitted the default is "3.2" # NOTE: kind must be set to MongoDB for this to take effect #mongoDBVersion: "3.6" @@ -19,6 +19,11 @@ spec: # - subnetId: /subscriptions/{subscription_id}/resourceGroups/{resourcegroup}/providers/Microsoft.Network/virtualNetworks/{vnet_name}/subnets/{subnet_name} # ignoreMissingServiceEndpoint: false - # Use the field below to optionally specify a different keyvault - # to store the connectiong string secrets in - #keyVaultToStoreSecrets: asoSecretKeyVault \ No newline at end of file +# optionally configure different CIDR IP ranges for allowed-list, omitting allows all or falls back to vNetRules +# ipRules: +# # these rules allow Azure Portal access +# - 104.42.195.92 +# - 40.76.54.131 +# - 52.176.6.30 +# - 52.169.50.45 +# - 52.187.184.26 \ No newline at end of file diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb.go b/pkg/resourcemanager/cosmosdbs/cosmosdb.go index 8c00b59f7ee..8665da4084b 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net/http" + "strings" "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb" "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -44,6 +45,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( location string, kind v1alpha1.CosmosDBKind, networkRule *[]v1alpha1.CosmosDBVirtualNetworkRule, + ipRules *[]string, properties v1alpha1.CosmosDBProperties, tags map[string]*string) (*documentdb.DatabaseAccount, error) { cosmosDBClient, err := getCosmosDBClient() @@ -98,6 +100,12 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( }) } } + + sIPRules := "" + if ipRules != nil { + sIPRules = strings.Join(*ipRules, ",") + } + createUpdateParams := documentdb.DatabaseAccountCreateUpdateParameters{ Location: to.StringPtr(location), Tags: tags, @@ -112,6 +120,7 @@ func (*AzureCosmosDBManager) CreateOrUpdateCosmosDB( EnableMultipleWriteLocations: &bWriteLocal, Locations: &locationsArray, Capabilities: &capabilities, + IPRangeFilter: &sIPRules, }, } createUpdateFuture, err := cosmosDBClient.CreateOrUpdate( diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go index d0b01fc3f05..23001b49bd0 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_manager.go @@ -21,7 +21,7 @@ func NewAzureCosmosDBManager(secretClient secrets.SecretClient) *AzureCosmosDBMa // CosmosDBManager client functions type CosmosDBManager interface { // CreateOrUpdateCosmosDB creates a new cosmos database account - CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, networkRule *[]v1alpha1.CosmosDBVirtualNetworkRule, properties v1alpha1.CosmosDBProperties, tags map[string]*string) (*documentdb.DatabaseAccount, error) + CreateOrUpdateCosmosDB(ctx context.Context, groupName string, cosmosDBName string, location string, kind v1alpha1.CosmosDBKind, networkRule *[]v1alpha1.CosmosDBVirtualNetworkRule, ipRules *[]string, properties v1alpha1.CosmosDBProperties, tags map[string]*string) (*documentdb.DatabaseAccount, error) // GetCosmosDB gets a cosmos database account GetCosmosDB(ctx context.Context, groupName string, cosmosDBName string) (*documentdb.DatabaseAccount, error) diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 88120e15809..55dc073b5f9 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -95,6 +95,7 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o location := instance.Spec.Location kind := instance.Spec.Kind networkRule := instance.Spec.VirtualNetworkRules + ipRules := instance.Spec.IPRules cosmosDBProperties := v1alpha1.CosmosDBProperties{ DatabaseAccountOfferType: instance.Spec.Properties.DatabaseAccountOfferType, @@ -103,7 +104,7 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o IsVirtualNetworkFilterEnabled: instance.Spec.Properties.IsVirtualNetworkFilterEnabled, } - db, err = m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, networkRule, cosmosDBProperties, tags) + db, err = m.CreateOrUpdateCosmosDB(ctx, groupName, accountName, location, kind, networkRule, ipRules, cosmosDBProperties, tags) if err != nil { azerr := errhelp.NewAzureErrorAzureError(err) instance.Status.Message = err.Error() From f56f92b5e4fbc072e125838b242bc07c93ed72a8 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 21 Apr 2020 10:42:47 -0600 Subject: [PATCH 21/27] fix compile issues --- controllers/azuresqlserver_controller_test.go | 1 - controllers/azuresqluser_controller_test.go | 6 +----- controllers/azurevnetrule_controller_test.go | 5 ----- controllers/eventhub_storageaccount_controller_test.go | 7 +++---- controllers/keyvault_controller_test.go | 4 ---- controllers/resourcegroup_controller_test.go | 6 ------ 6 files changed, 4 insertions(+), 25 deletions(-) diff --git a/controllers/azuresqlserver_controller_test.go b/controllers/azuresqlserver_controller_test.go index ecedcbe46a0..86e42e0e420 100644 --- a/controllers/azuresqlserver_controller_test.go +++ b/controllers/azuresqlserver_controller_test.go @@ -18,7 +18,6 @@ func TestAzureSqlServerControllerNoResourceGroup(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) sqlServerName := GenerateTestResourceNameWithRandom("sqlserver-dev", 10) diff --git a/controllers/azuresqluser_controller_test.go b/controllers/azuresqluser_controller_test.go index 2c5b0f99558..9e18f595936 100644 --- a/controllers/azuresqluser_controller_test.go +++ b/controllers/azuresqluser_controller_test.go @@ -7,7 +7,6 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" @@ -15,17 +14,14 @@ import ( "github.com/Azure/azure-service-operator/pkg/helpers" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestAzureSQLUserControllerNoAdminSecret(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) - var err error + var sqlServerName string var sqlDatabaseName string var sqlUser *azurev1alpha1.AzureSQLUser diff --git a/controllers/azurevnetrule_controller_test.go b/controllers/azurevnetrule_controller_test.go index a32f4007d9e..96cdffa198b 100644 --- a/controllers/azurevnetrule_controller_test.go +++ b/controllers/azurevnetrule_controller_test.go @@ -7,23 +7,18 @@ package controllers import ( "context" - "strings" "testing" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - "github.com/stretchr/testify/assert" "github.com/Azure/azure-service-operator/pkg/errhelp" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestAzureSqlVNetRuleControllerNoResourceGroup(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) // Add any setup steps that needs to be executed before each test sqlServerName := GenerateTestResourceNameWithRandom("sqlvnetrule-test-srv", 10) diff --git a/controllers/eventhub_storageaccount_controller_test.go b/controllers/eventhub_storageaccount_controller_test.go index 9fdbbf6bd66..181c34ed80c 100644 --- a/controllers/eventhub_storageaccount_controller_test.go +++ b/controllers/eventhub_storageaccount_controller_test.go @@ -7,16 +7,15 @@ package controllers import ( "context" - "fmt" "testing" s "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" - //"github.com/Azure/azure-service-operator/pkg/errhelp" + "github.com/Azure/azure-service-operator/pkg/errhelp" - //kvhelper "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" - //kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" + kvhelper "github.com/Azure/azure-service-operator/pkg/resourcemanager/keyvaults" + kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/controllers/keyvault_controller_test.go b/controllers/keyvault_controller_test.go index 2398bf6ba87..a4552b05361 100644 --- a/controllers/keyvault_controller_test.go +++ b/controllers/keyvault_controller_test.go @@ -8,7 +8,6 @@ package controllers import ( "context" "net/http" - "strings" "testing" "time" @@ -19,7 +18,6 @@ import ( kvsecrets "github.com/Azure/azure-service-operator/pkg/secrets/keyvault" "github.com/stretchr/testify/assert" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -236,7 +234,6 @@ func TestKeyvaultControllerInvalidName(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) keyVaultName := "k" @@ -263,7 +260,6 @@ func TestKeyvaultControllerNoResourceGroup(t *testing.T) { t.Parallel() defer PanicRecover(t) ctx := context.Background() - assert := assert.New(t) keyVaultName := helpers.FillWithRandom(GenerateTestResourceName("kv"), 24) diff --git a/controllers/resourcegroup_controller_test.go b/controllers/resourcegroup_controller_test.go index 436b1f96a9c..08997d640e5 100644 --- a/controllers/resourcegroup_controller_test.go +++ b/controllers/resourcegroup_controller_test.go @@ -13,9 +13,7 @@ import ( azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" "github.com/stretchr/testify/assert" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) func TestResourceGroupControllerHappyPath(t *testing.T) { @@ -26,8 +24,6 @@ func TestResourceGroupControllerHappyPath(t *testing.T) { resourceGroupName := GenerateTestResourceNameWithRandom("rg-dev", 10) - var err error - // Create the ResourceGroup object and expect the Reconcile to be created resourceGroupInstance := &azurev1alpha1.ResourceGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -42,8 +38,6 @@ func TestResourceGroupControllerHappyPath(t *testing.T) { // create rg EnsureInstance(ctx, t, tc, resourceGroupInstance) - resourceGroupNamespacedName := types.NamespacedName{Name: resourceGroupName, Namespace: "default"} - // verify rg exists in azure assert.Eventually(func() bool { _, err := tc.resourceGroupManager.CheckExistence(ctx, resourceGroupName) From 41b90f689ee19b455e38477d430a4b743c41dd65 Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 21 Apr 2020 11:30:38 -0600 Subject: [PATCH 22/27] fix test --- .../eventhub_namespace_consumergroup_controller_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/controllers/eventhub_namespace_consumergroup_controller_test.go b/controllers/eventhub_namespace_consumergroup_controller_test.go index 7e50b7a5d84..123436bcdc4 100644 --- a/controllers/eventhub_namespace_consumergroup_controller_test.go +++ b/controllers/eventhub_namespace_consumergroup_controller_test.go @@ -91,11 +91,6 @@ func TestConsumerGroupEventHubAndNamespaceControllerHappy(t *testing.T) { EnsureDelete(ctx, t, tc, consumerGroupInstance) - assert.Eventually(func() bool { - cg, _ := tc.consumerGroupClient.GetConsumerGroup(ctx, rgName, eventhubNamespaceName, eventhubName, azureConsumerGroupName) - return cg.Response.StatusCode != http.StatusOK - }, tc.timeout, tc.retry, "wait for consumergroup to be gone from azure") - // verify eventhub is deleted EnsureDelete(ctx, t, tc, eventhubInstance) From 0cb0fbf83f27db3033ecdf63c969c108f6d6c91f Mon Sep 17 00:00:00 2001 From: Mel Rush Date: Tue, 21 Apr 2020 13:49:18 -0600 Subject: [PATCH 23/27] adding msg and provisioned to kube (#968) --- api/v1alpha1/apimgmt_types.go | 2 ++ api/v1alpha1/apimservice_types.go | 2 ++ api/v1alpha1/appinsights_types.go | 2 ++ api/v1alpha1/azuredatalakegen2filesystem_types.go | 2 ++ api/v1alpha1/azurenetworkinterface_types.go | 2 ++ api/v1alpha1/azurepublicipaddress_types.go | 2 ++ api/v1alpha1/azuresqlaction_types.go | 2 ++ api/v1alpha1/azuresqldatabase_types.go | 2 ++ api/v1alpha1/azuresqlfailovergroup_types.go | 2 ++ api/v1alpha1/azuresqlfirewallrule_types.go | 2 ++ api/v1alpha1/azuresqlserver_types.go | 2 ++ api/v1alpha1/azuresqluser_types.go | 2 ++ api/v1alpha1/azuresqlvnetrule_types.go | 2 ++ api/v1alpha1/azurevirtualmachine_types.go | 2 ++ api/v1alpha1/blobcontainer_types.go | 2 ++ api/v1alpha1/consumergroup_types.go | 2 ++ api/v1alpha1/cosmosdb_types.go | 2 ++ api/v1alpha1/eventhub_types.go | 2 ++ api/v1alpha1/eventhubnamespace_types.go | 2 ++ api/v1alpha1/keyvault_types.go | 2 ++ api/v1alpha1/keyvaultkey_types.go | 2 ++ api/v1alpha1/mysqldatabase_types.go | 2 ++ api/v1alpha1/mysqlfirewallrule_types.go | 2 ++ api/v1alpha1/mysqlserver_types.go | 2 ++ api/v1alpha1/postgresqldatabase_types.go | 2 ++ api/v1alpha1/postgresqlfirewallrule_types.go | 2 ++ api/v1alpha1/postgresqlserver_types.go | 2 ++ api/v1alpha1/rediscache_types.go | 2 ++ api/v1alpha1/resourcegroup_types.go | 2 +- api/v1alpha1/storageaccount_types.go | 2 ++ api/v1alpha1/virtualnetwork_types.go | 2 ++ 31 files changed, 61 insertions(+), 1 deletion(-) diff --git a/api/v1alpha1/apimgmt_types.go b/api/v1alpha1/apimgmt_types.go index c5395021632..ec4d4f1a99b 100644 --- a/api/v1alpha1/apimgmt_types.go +++ b/api/v1alpha1/apimgmt_types.go @@ -18,6 +18,8 @@ type APIMgmtSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type APIMgmtAPI struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/apimservice_types.go b/api/v1alpha1/apimservice_types.go index dda328b5eb7..4dd7b057e17 100644 --- a/api/v1alpha1/apimservice_types.go +++ b/api/v1alpha1/apimservice_types.go @@ -31,6 +31,8 @@ type ApimServiceSpec struct { // +kubebuilder:subresource:status // ApimService is the Schema for the apimservices API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type ApimService struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/appinsights_types.go b/api/v1alpha1/appinsights_types.go index 2c0ad466f0a..c10e15d933f 100644 --- a/api/v1alpha1/appinsights_types.go +++ b/api/v1alpha1/appinsights_types.go @@ -22,6 +22,8 @@ type AppInsightsSpec struct { // +kubebuilder:subresource:status // AppInsights is the Schema for the appinsights API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AppInsights struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azuredatalakegen2filesystem_types.go b/api/v1alpha1/azuredatalakegen2filesystem_types.go index c956252cda2..2e59c0c4431 100644 --- a/api/v1alpha1/azuredatalakegen2filesystem_types.go +++ b/api/v1alpha1/azuredatalakegen2filesystem_types.go @@ -22,6 +22,8 @@ type AzureDataLakeGen2FileSystemSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // AzureDataLakeGen2FileSystem is the Schema for the azuredatalakegen2filesystems API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureDataLakeGen2FileSystem struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azurenetworkinterface_types.go b/api/v1alpha1/azurenetworkinterface_types.go index 28251f75fd3..6b01e08b94d 100644 --- a/api/v1alpha1/azurenetworkinterface_types.go +++ b/api/v1alpha1/azurenetworkinterface_types.go @@ -25,6 +25,8 @@ type AzureNetworkInterfaceSpec struct { // +kubebuilder:subresource:status // AzureNetworkInterface is the Schema for the azurenetworkinterfaces API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureNetworkInterface struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azurepublicipaddress_types.go b/api/v1alpha1/azurepublicipaddress_types.go index 7443aba869c..136d1d49cd4 100644 --- a/api/v1alpha1/azurepublicipaddress_types.go +++ b/api/v1alpha1/azurepublicipaddress_types.go @@ -26,6 +26,8 @@ type AzurePublicIPAddressSpec struct { // +kubebuilder:subresource:status // AzurePublicIPAddress is the Schema for the azurepublicipaddresses API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzurePublicIPAddress struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azuresqlaction_types.go b/api/v1alpha1/azuresqlaction_types.go index 584d5699d9e..33721494b5a 100644 --- a/api/v1alpha1/azuresqlaction_types.go +++ b/api/v1alpha1/azuresqlaction_types.go @@ -28,6 +28,8 @@ type AzureSqlActionSpec struct { // +kubebuilder:subresource:status // AzureSqlAction is the Schema for the azuresqlactions API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureSqlAction struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azuresqldatabase_types.go b/api/v1alpha1/azuresqldatabase_types.go index 2e1c487f112..b1443ae9fd9 100644 --- a/api/v1alpha1/azuresqldatabase_types.go +++ b/api/v1alpha1/azuresqldatabase_types.go @@ -25,6 +25,8 @@ type AzureSqlDatabaseSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // AzureSqlDatabase is the Schema for the azuresqldatabases API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureSqlDatabase struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azuresqlfailovergroup_types.go b/api/v1alpha1/azuresqlfailovergroup_types.go index 36ad6ff83f0..2e4960c2027 100644 --- a/api/v1alpha1/azuresqlfailovergroup_types.go +++ b/api/v1alpha1/azuresqlfailovergroup_types.go @@ -26,6 +26,8 @@ type AzureSqlFailoverGroupSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // AzureSqlFailoverGroup is the Schema for the azuresqlfailovergroups API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureSqlFailoverGroup struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azuresqlfirewallrule_types.go b/api/v1alpha1/azuresqlfirewallrule_types.go index d4601c66a4d..395ec1233ae 100644 --- a/api/v1alpha1/azuresqlfirewallrule_types.go +++ b/api/v1alpha1/azuresqlfirewallrule_types.go @@ -24,6 +24,8 @@ type AzureSqlFirewallRuleSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // AzureSqlFirewallRule is the Schema for the azuresqlfirewallrules API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureSqlFirewallRule struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azuresqlserver_types.go b/api/v1alpha1/azuresqlserver_types.go index 381d5541d7d..67e9e28d222 100644 --- a/api/v1alpha1/azuresqlserver_types.go +++ b/api/v1alpha1/azuresqlserver_types.go @@ -23,6 +23,8 @@ type AzureSqlServerSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // AzureSqlServer is the Schema for the azuresqlservers API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureSqlServer struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azuresqluser_types.go b/api/v1alpha1/azuresqluser_types.go index 6cdd3b6dd5e..0e1fa7457db 100644 --- a/api/v1alpha1/azuresqluser_types.go +++ b/api/v1alpha1/azuresqluser_types.go @@ -31,6 +31,8 @@ type AzureSQLUserSpec struct { // +kubebuilder:subresource:status // AzureSQLUser is the Schema for the sqlusers API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureSQLUser struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azuresqlvnetrule_types.go b/api/v1alpha1/azuresqlvnetrule_types.go index ccb1c6d0883..bf190c80bb6 100644 --- a/api/v1alpha1/azuresqlvnetrule_types.go +++ b/api/v1alpha1/azuresqlvnetrule_types.go @@ -23,6 +23,8 @@ type AzureSQLVNetRuleSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // AzureSQLVNetRule is the Schema for the azuresqlvnetrules API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureSQLVNetRule struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/azurevirtualmachine_types.go b/api/v1alpha1/azurevirtualmachine_types.go index 05555d355a0..47971b50836 100644 --- a/api/v1alpha1/azurevirtualmachine_types.go +++ b/api/v1alpha1/azurevirtualmachine_types.go @@ -37,6 +37,8 @@ const ( // +kubebuilder:subresource:status // AzureVirtualMachine is the Schema for the azurevirtualmachines API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type AzureVirtualMachine struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/blobcontainer_types.go b/api/v1alpha1/blobcontainer_types.go index e372e4e3ddb..1dc36b58d52 100644 --- a/api/v1alpha1/blobcontainer_types.go +++ b/api/v1alpha1/blobcontainer_types.go @@ -25,6 +25,8 @@ type BlobContainerSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // BlobContainer is the Schema for the blobcontainers API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type BlobContainer struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/consumergroup_types.go b/api/v1alpha1/consumergroup_types.go index c821ce23b1f..4617cd5e405 100644 --- a/api/v1alpha1/consumergroup_types.go +++ b/api/v1alpha1/consumergroup_types.go @@ -25,6 +25,8 @@ type ConsumerGroupSpec struct { // +kubebuilder:subresource:status // ConsumerGroup is the Schema for the consumergroups API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type ConsumerGroup struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/cosmosdb_types.go b/api/v1alpha1/cosmosdb_types.go index 1a0c32794fa..a4b6259243a 100644 --- a/api/v1alpha1/cosmosdb_types.go +++ b/api/v1alpha1/cosmosdb_types.go @@ -69,6 +69,8 @@ type CosmosDBLocation struct { // +kubebuilder:subresource:status // CosmosDB is the Schema for the cosmosdbs API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type CosmosDB struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/eventhub_types.go b/api/v1alpha1/eventhub_types.go index d2c07a7175a..1e1bd5aca9e 100644 --- a/api/v1alpha1/eventhub_types.go +++ b/api/v1alpha1/eventhub_types.go @@ -91,6 +91,8 @@ type EventhubProperties struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // Eventhub is the Schema for the eventhubs API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type Eventhub struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/eventhubnamespace_types.go b/api/v1alpha1/eventhubnamespace_types.go index ff4374cd56a..41c10917ebe 100644 --- a/api/v1alpha1/eventhubnamespace_types.go +++ b/api/v1alpha1/eventhubnamespace_types.go @@ -25,6 +25,8 @@ type EventhubNamespaceSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // EventhubNamespace is the Schema for the eventhubnamespaces API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type EventhubNamespace struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/keyvault_types.go b/api/v1alpha1/keyvault_types.go index 3b21a2c1f67..0abd6b40a00 100644 --- a/api/v1alpha1/keyvault_types.go +++ b/api/v1alpha1/keyvault_types.go @@ -57,6 +57,8 @@ type Permissions struct { // +kubebuilder:subresource:status // KeyVault is the Schema for the keyvaults API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type KeyVault struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/keyvaultkey_types.go b/api/v1alpha1/keyvaultkey_types.go index d3724412891..bec2672837a 100644 --- a/api/v1alpha1/keyvaultkey_types.go +++ b/api/v1alpha1/keyvaultkey_types.go @@ -27,6 +27,8 @@ type KeyVaultKeySpec struct { // +kubebuilder:subresource:status // KeyVaultKey is the Schema for the keyvaultkeys API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type KeyVaultKey struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/mysqldatabase_types.go b/api/v1alpha1/mysqldatabase_types.go index f6626672582..32ef46908d0 100644 --- a/api/v1alpha1/mysqldatabase_types.go +++ b/api/v1alpha1/mysqldatabase_types.go @@ -20,6 +20,8 @@ type MySQLDatabaseSpec struct { // +kubebuilder:subresource:status // MySQLDatabase is the Schema for the mysqldatabases API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type MySQLDatabase struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/mysqlfirewallrule_types.go b/api/v1alpha1/mysqlfirewallrule_types.go index b213b5f08ce..a5d646a7c44 100644 --- a/api/v1alpha1/mysqlfirewallrule_types.go +++ b/api/v1alpha1/mysqlfirewallrule_types.go @@ -22,6 +22,8 @@ type MySQLFirewallRuleSpec struct { // +kubebuilder:subresource:status // MySQLFirewallRule is the Schema for the mysqlfirewallrules API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type MySQLFirewallRule struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/mysqlserver_types.go b/api/v1alpha1/mysqlserver_types.go index 3a3c8386512..25a24adbf30 100644 --- a/api/v1alpha1/mysqlserver_types.go +++ b/api/v1alpha1/mysqlserver_types.go @@ -26,6 +26,8 @@ type MySQLServerSpec struct { // +kubebuilder:subresource:status // MySQLServer is the Schema for the mysqlservers API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type MySQLServer struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/postgresqldatabase_types.go b/api/v1alpha1/postgresqldatabase_types.go index 83bfd40424e..f01daf465b3 100644 --- a/api/v1alpha1/postgresqldatabase_types.go +++ b/api/v1alpha1/postgresqldatabase_types.go @@ -22,6 +22,8 @@ type PostgreSQLDatabaseSpec struct { // +kubebuilder:subresource:status // PostgreSQLDatabase is the Schema for the postgresqldatabases API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type PostgreSQLDatabase struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/postgresqlfirewallrule_types.go b/api/v1alpha1/postgresqlfirewallrule_types.go index 82becf2bb5a..e2ff53dee81 100644 --- a/api/v1alpha1/postgresqlfirewallrule_types.go +++ b/api/v1alpha1/postgresqlfirewallrule_types.go @@ -24,6 +24,8 @@ type PostgreSQLFirewallRuleSpec struct { // +kubebuilder:subresource:status // PostgreSQLFirewallRule is the Schema for the postgresqlfirewallrules API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type PostgreSQLFirewallRule struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/postgresqlserver_types.go b/api/v1alpha1/postgresqlserver_types.go index 2489d079a3d..9dd64dd1b83 100644 --- a/api/v1alpha1/postgresqlserver_types.go +++ b/api/v1alpha1/postgresqlserver_types.go @@ -75,6 +75,8 @@ const ( // +kubebuilder:subresource:status // PostgreSQLServer is the Schema for the postgresqlservers API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type PostgreSQLServer struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/rediscache_types.go b/api/v1alpha1/rediscache_types.go index 01abff59a93..54fd0fe544d 100644 --- a/api/v1alpha1/rediscache_types.go +++ b/api/v1alpha1/rediscache_types.go @@ -63,6 +63,8 @@ const ( // +kubebuilder:subresource:status // RedisCache is the Schema for the rediscaches API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type RedisCache struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/resourcegroup_types.go b/api/v1alpha1/resourcegroup_types.go index 8902d539f53..8cbf2f740e5 100644 --- a/api/v1alpha1/resourcegroup_types.go +++ b/api/v1alpha1/resourcegroup_types.go @@ -23,7 +23,7 @@ type ResourceGroupSpec struct { // ResourceGroup is the Schema for the resourcegroups API // +kubebuilder:resource:shortName=rg,path=resourcegroups // +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" -// +kubebuilder:printcolumn:name="Provisioning",type="string",JSONPath=".status.provisioning" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type ResourceGroup struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/storageaccount_types.go b/api/v1alpha1/storageaccount_types.go index 41ba60b368c..730f5fc47ad 100644 --- a/api/v1alpha1/storageaccount_types.go +++ b/api/v1alpha1/storageaccount_types.go @@ -65,6 +65,8 @@ type StorageAccountAccessTier string // +kubebuilder:subresource:status // StorageAccount is the Schema for the storages API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type StorageAccount struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/virtualnetwork_types.go b/api/v1alpha1/virtualnetwork_types.go index 8723e16a841..803ba6917d6 100644 --- a/api/v1alpha1/virtualnetwork_types.go +++ b/api/v1alpha1/virtualnetwork_types.go @@ -32,6 +32,8 @@ type VirtualNetworkSpec struct { // +kubebuilder:subresource:status // VirtualNetwork is the Schema for the virtualnetworks API +// +kubebuilder:printcolumn:name="Provisioned",type="string",JSONPath=".status.provisioned" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.message" type VirtualNetwork struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` From 866e1676a872c9ec05f86d944ce72bbdd35488ee Mon Sep 17 00:00:00 2001 From: jpflueger Date: Tue, 21 Apr 2020 14:03:22 -0600 Subject: [PATCH 24/27] fixing sample comment indentation and update path --- config/samples/azure_v1alpha1_cosmosdb.yaml | 31 ++++++++++--------- .../cosmosdbs/cosmosdb_reconcile.go | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/config/samples/azure_v1alpha1_cosmosdb.yaml b/config/samples/azure_v1alpha1_cosmosdb.yaml index af960a5c725..e4b857a7a66 100644 --- a/config/samples/azure_v1alpha1_cosmosdb.yaml +++ b/config/samples/azure_v1alpha1_cosmosdb.yaml @@ -8,22 +8,23 @@ spec: resourceGroup: resourcegroup-azure-operators properties: databaseAccountOfferType: Standard - #enableMultipleWriteLocations: false + enableMultipleWriteLocations: false + # optionally set the mongoDBVersion to "3.2" or "3.6", if omitted the default is "3.2" # NOTE: kind must be set to MongoDB for this to take effect - #mongoDBVersion: "3.6" + # mongoDBVersion: "3.6" + + # enable virtual network rules if configured below + # isVirtualNetworkFilterEnabled: true -#optional for network rule set -# isVirtualNetworkFilterEnabled: true -# virtualNetworkRules: -# - subnetId: /subscriptions/{subscription_id}/resourceGroups/{resourcegroup}/providers/Microsoft.Network/virtualNetworks/{vnet_name}/subnets/{subnet_name} -# ignoreMissingServiceEndpoint: false + # optionally restrict access to specific virtual networks + # virtualNetworkRules: + # - subnetId: /subscriptions/{subscription_id}/resourceGroups/{resourcegroup}/providers/Microsoft.Network/virtualNetworks/{vnet_name}/subnets/{subnet_name} + # ignoreMissingServiceEndpoint: false -# optionally configure different CIDR IP ranges for allowed-list, omitting allows all or falls back to vNetRules -# ipRules: -# # these rules allow Azure Portal access -# - 104.42.195.92 -# - 40.76.54.131 -# - 52.176.6.30 -# - 52.169.50.45 -# - 52.187.184.26 \ No newline at end of file + # optionally configure different CIDR IP ranges for allowed-list, omitting allows all or falls back to vNetRules + # ipRules: + # # the ips in this rule are needed to access your db from the portal + # - 104.42.195.92,40.76.54.131,52.176.6.30,52.169.50.45,52.187.184.26 + # # add additional ips you would like to grant access + # - 73.153.28.188 \ No newline at end of file diff --git a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go index 55dc073b5f9..b06554f1fd1 100644 --- a/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go +++ b/pkg/resourcemanager/cosmosdbs/cosmosdb_reconcile.go @@ -60,7 +60,7 @@ func (m *AzureCosmosDBManager) Ensure(ctx context.Context, obj runtime.Object, o instance.Status.State = *db.ProvisioningState } - if instance.Status.State == "Creating" { + if instance.Status.State == "Creating" || instance.Status.State == "Updating" { // avoid multiple CreateOrUpdate requests while resource is already creating return false, nil } From 982f427bc80c3aa617381c1f5a0cf3b6dd15d49c Mon Sep 17 00:00:00 2001 From: jananivMS Date: Tue, 21 Apr 2020 14:28:24 -0600 Subject: [PATCH 25/27] remove comments --- controllers/suite_test.go | 80 ++------------------------------------- 1 file changed, 4 insertions(+), 76 deletions(-) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 47cee1c297c..9018ad9cc14 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -73,14 +73,6 @@ func setup() error { resourceGroupName := GenerateTestResourceName("rg-prime") resourcegroupLocation := resourcemanagerconfig.DefaultLocation() - //eventhubNamespaceName := GenerateTestResourceName("evns-prime") - //eventhubName := GenerateTestResourceName("ev-prime") - //namespaceLocation := resourcemanagerconfig.DefaultLocation() - - //storageAccountName := GenerateAlphaNumTestResourceName("saprime") - //blobContainerName := GenerateTestResourceName("blob-prime") - //containerAccessLevel := s.PublicAccessContainer - keyvaultName := GenerateAlphaNumTestResourceName("kv-prime") var timeout time.Duration @@ -683,26 +675,11 @@ func setup() error { _, _ = resourceGroupManager.CreateGroup(context.Background(), resourceGroupName, resourcegroupLocation) } - /* - log.Println("Creating EHNS:", eventhubNamespaceName) - eventHubNSManager := eventHubManagers.EventHubNamespace - - // Create the Eventhub namespace resource - _, err = eventHubNSManager.CreateNamespaceAndWait(context.Background(), resourceGroupName, eventhubNamespaceName, namespaceLocation) - if err != nil { - return err - }*/ - tc = TestContext{ - k8sClient: k8sClient, - secretClient: secretClient, - resourceGroupName: resourceGroupName, - resourceGroupLocation: resourcegroupLocation, - //eventhubNamespaceName: eventhubNamespaceName, - //eventhubName: eventhubName, - //namespaceLocation: namespaceLocation, - //storageAccountName: storageAccountName, - //blobContainerName: blobContainerName, + k8sClient: k8sClient, + secretClient: secretClient, + resourceGroupName: resourceGroupName, + resourceGroupLocation: resourcegroupLocation, keyvaultName: keyvaultName, eventHubManagers: eventHubManagers, eventhubClient: eventhubClient, @@ -720,55 +697,6 @@ func setup() error { retry: time.Second * 3, consumerGroupClient: consumerGroupClient, } - /* - var pstate *string - finish := time.Now().Add(tc.timeout) - for { - if finish.Before(time.Now()) { - return fmt.Errorf("time out waiting for eventhub namespace") - } - - namespace, _ := eventHubManagers.EventHubNamespace.GetNamespace(context.Background(), resourceGroupName, eventhubNamespaceName) - pstate = namespace.ProvisioningState - if pstate != nil && *pstate == "Succeeded" { - break - } - time.Sleep(tc.retry) - } - - log.Println("Creating EH:", eventhubName) - // Create the Eventhub resource - _, err = eventHubManagers.EventHub.CreateHub(context.Background(), resourceGroupName, eventhubNamespaceName, eventhubName, int32(7), int32(2), nil) - if err != nil { - return err - } - - log.Println("Creating SA:", storageAccountName) - // Create the Storage Account and Container - _, _, _ = storageAccountManager.CreateStorage(context.Background(), resourceGroupName, storageAccountName, resourcegroupLocation, azurev1alpha1.StorageAccountSku{ - Name: "Standard_LRS", - }, "Storage", map[string]*string{}, "", nil, nil, nil) - - // Storage account needs to be in "Suceeded" state - // for container create to succeed - finish = time.Now().Add(tc.timeout) - for { - - if finish.Before(time.Now()) { - return fmt.Errorf("time out waiting for storage account") - } - - result, _ := storageAccountManager.GetStorage(context.Background(), resourceGroupName, storageAccountName) - if result.ProvisioningState == s.Succeeded { - break - } - time.Sleep(tc.retry) - } - - _, err = storageManagers.BlobContainer.CreateBlobContainer(context.Background(), resourceGroupName, storageAccountName, blobContainerName, containerAccessLevel) - if err != nil { - return err - }*/ log.Println("Creating KV:", keyvaultName) _, err = resourcemanagerkeyvaults.AzureKeyVaultManager.CreateVaultWithAccessPolicies(context.Background(), resourceGroupName, keyvaultName, resourcegroupLocation, resourcemanagerconfig.ClientID()) From beef6468b9dfcf01355cdbd8e51a82552c0bb3a1 Mon Sep 17 00:00:00 2001 From: Erin Corson Date: Tue, 21 Apr 2020 16:02:27 -0600 Subject: [PATCH 26/27] register vm controller with test operator --- controllers/suite_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 9018ad9cc14..2c143a82683 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -44,6 +44,7 @@ import ( resourcemanagerstorages "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages" resourcemanagerblobcontainer "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages/blobcontainer" resourcemanagerstorageaccount "github.com/Azure/azure-service-operator/pkg/resourcemanager/storages/storageaccount" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/vm" resourcemanagervnet "github.com/Azure/azure-service-operator/pkg/resourcemanager/vnet" telemetry "github.com/Azure/azure-service-operator/pkg/telemetry" @@ -509,6 +510,25 @@ func setup() error { return err } + err = (&AzureVirtualMachineReconciler{ + Reconciler: &AsyncReconciler{ + Client: k8sManager.GetClient(), + AzureClient: vm.NewAzureVirtualMachineClient( + secretClient, + k8sManager.GetScheme(), + ), + Telemetry: telemetry.InitializeTelemetryDefault( + "VirtualMachine", + ctrl.Log.WithName("controllers").WithName("VirtualMachine"), + ), + Recorder: k8sManager.GetEventRecorderFor("VirtualMachine-controller"), + Scheme: scheme.Scheme, + }, + }).SetupWithManager(k8sManager) + if err != nil { + return err + } + err = (&AzureSqlActionReconciler{ Reconciler: &AsyncReconciler{ Client: k8sManager.GetClient(), From f7cffbd27a40fa87311c6ff2ba221314596f97f8 Mon Sep 17 00:00:00 2001 From: Claudia Nadolny Date: Wed, 22 Apr 2020 11:43:07 -0700 Subject: [PATCH 27/27] Adding MySQL VNET Rule Operator (#958) * initial changes for vnet rule operator * added mysql docs * remaining changes, added tests + controller logic * minor cleanup * fixed test * updated docs Co-authored-by: William Mortl <32373900+WilliamMortlMicrosoft@users.noreply.github.com> Co-authored-by: Erin Corson --- PROJECT | 3 + api/v1alpha1/mysqlvnetrule_types.go | 47 +++++ config/crd/kustomization.yaml | 3 + .../cainjection_in_mysqlvnetrules.yaml | 8 + .../patches/webhook_in_mysqlvnetrules.yaml | 17 ++ .../samples/azure_v1alpha1_mysqlvnetrule.yaml | 11 ++ controllers/mysql_combined_test.go | 3 + controllers/mysqlvnetrule_controller.go | 28 +++ controllers/mysqlvnetrule_controller_test.go | 131 ++++++++++++++ docs/mysql/mysql.md | 8 + main.go | 16 ++ pkg/resourcemanager/mysql/vnetrule/client.go | 102 +++++++++++ pkg/resourcemanager/mysql/vnetrule/manager.go | 18 ++ .../mysql/vnetrule/reconcile.go | 171 ++++++++++++++++++ 14 files changed, 566 insertions(+) create mode 100644 api/v1alpha1/mysqlvnetrule_types.go create mode 100644 config/crd/patches/cainjection_in_mysqlvnetrules.yaml create mode 100644 config/crd/patches/webhook_in_mysqlvnetrules.yaml create mode 100644 config/samples/azure_v1alpha1_mysqlvnetrule.yaml create mode 100644 controllers/mysqlvnetrule_controller.go create mode 100644 controllers/mysqlvnetrule_controller_test.go create mode 100644 pkg/resourcemanager/mysql/vnetrule/client.go create mode 100644 pkg/resourcemanager/mysql/vnetrule/manager.go create mode 100644 pkg/resourcemanager/mysql/vnetrule/reconcile.go diff --git a/PROJECT b/PROJECT index 00905ec53b7..59918be0af4 100644 --- a/PROJECT +++ b/PROJECT @@ -86,6 +86,9 @@ resources: - group: azure version: v1alpha1 kind: MySQLFirewallRule +- group: azure + version: v1alpha1 + kind: MySQLVNetRule - group: azure version: v1alpha1 kind: AzureVirtualMachine diff --git a/api/v1alpha1/mysqlvnetrule_types.go b/api/v1alpha1/mysqlvnetrule_types.go new file mode 100644 index 00000000000..a4345d0f583 --- /dev/null +++ b/api/v1alpha1/mysqlvnetrule_types.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// MySQLVNetRuleSpec defines the desired state of MySQLVNetRule +type MySQLVNetRuleSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + ResourceGroup string `json:"resourceGroup"` + Server string `json:"server"` + VNetResourceGroup string `json:"vNetResourceGroup"` + VNetName string `json:"vNetName"` + SubnetName string `json:"subnetName"` + IgnoreMissingServiceEndpoint bool `json:"ignoreMissingServiceEndpoint,omitempty"` +} + +// +kubebuilder:object:root=true + +// MySQLVNetRule is the Schema for the mysqlvnetrules API +type MySQLVNetRule struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec MySQLVNetRuleSpec `json:"spec,omitempty"` + Status ASOStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// MySQLVNetRuleList contains a list of MySQLVNetRule +type MySQLVNetRuleList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []MySQLVNetRule `json:"items"` +} + +func init() { + SchemeBuilder.Register(&MySQLVNetRule{}, &MySQLVNetRuleList{}) +} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index d32a5f49f90..22f54886eb2 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -33,6 +33,7 @@ resources: - bases/azure.microsoft.com_mysqlfirewallrules.yaml - bases/azure.microsoft.com_azurepublicipaddresses.yaml - bases/azure.microsoft.com_azurenetworkinterfaces.yaml +- bases/azure.microsoft.com_mysqlvnetrules.yaml - bases/azure.microsoft.com_azurevirtualmachines.yaml # +kubebuilder:scaffold:crdkustomizeresource @@ -66,6 +67,7 @@ resources: #- patches/webhook_in_storageaccounts.yaml #- patches/webhook_in_azurepublicipaddresses.yaml #- patches/webhook_in_azurenetworkinterfaces.yaml +#- patches/webhook_in_mysqlvnetrules.yaml #- patches/webhook_in_azurevirtualmachines.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch @@ -98,6 +100,7 @@ resources: #- patches/cainjection_in_storageaccounts.yaml #- patches/cainjection_in_azurepublicipaddresses.yaml #- patches/cainjection_in_azurenetworkinterfaces.yaml +#- patches/cainjection_in_mysqlvnetrules.yaml #- patches/cainjection_in_azurevirtualmachines.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch diff --git a/config/crd/patches/cainjection_in_mysqlvnetrules.yaml b/config/crd/patches/cainjection_in_mysqlvnetrules.yaml new file mode 100644 index 00000000000..370c08038db --- /dev/null +++ b/config/crd/patches/cainjection_in_mysqlvnetrules.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + certmanager.k8s.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: mysqlvnetrules.azure.microsoft.com diff --git a/config/crd/patches/webhook_in_mysqlvnetrules.yaml b/config/crd/patches/webhook_in_mysqlvnetrules.yaml new file mode 100644 index 00000000000..e0f6bda0df2 --- /dev/null +++ b/config/crd/patches/webhook_in_mysqlvnetrules.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: mysqlvnetrules.azure.microsoft.com +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/samples/azure_v1alpha1_mysqlvnetrule.yaml b/config/samples/azure_v1alpha1_mysqlvnetrule.yaml new file mode 100644 index 00000000000..e27496c3843 --- /dev/null +++ b/config/samples/azure_v1alpha1_mysqlvnetrule.yaml @@ -0,0 +1,11 @@ +apiVersion: azure.microsoft.com/v1alpha1 +kind: MySQLVNetRule +metadata: + name: mysqlvnetrule-sample +spec: + resourceGroup: resourcegroup-azure-operators + server: mysqlserver-sample + vNetResourceGroup: resourcegroup-vnet + vNetName: virtualnetwork-sample + subnetName: test1 + ignoreMissingServiceEndpoint: true diff --git a/controllers/mysql_combined_test.go b/controllers/mysql_combined_test.go index a0cc952fbbe..1e570f76783 100644 --- a/controllers/mysql_combined_test.go +++ b/controllers/mysql_combined_test.go @@ -68,6 +68,9 @@ func TestMySQLHappyPath(t *testing.T) { EnsureInstance(ctx, t, tc, ruleInstance) + // Create VNet and VNetRules ----- + RunMySqlVNetRuleHappyPath(t, mySQLServerName, rgLocation) + EnsureDelete(ctx, t, tc, ruleInstance) EnsureDelete(ctx, t, tc, mySQLDBInstance) EnsureDelete(ctx, t, tc, mySQLServerInstance) diff --git a/controllers/mysqlvnetrule_controller.go b/controllers/mysqlvnetrule_controller.go new file mode 100644 index 00000000000..18b1442f109 --- /dev/null +++ b/controllers/mysqlvnetrule_controller.go @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package controllers + +import ( + ctrl "sigs.k8s.io/controller-runtime" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" +) + +// MySQLVNetRuleReconciler reconciles a MySQLVNetRule object +type MySQLVNetRuleReconciler struct { + Reconciler *AsyncReconciler +} + +// +kubebuilder:rbac:groups=azure.microsoft.com,resources=mysqlvnetrules,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=azure.microsoft.com,resources=mysqlvnetrules/status,verbs=get;update;patch + +func (r *MySQLVNetRuleReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + return r.Reconciler.Reconcile(req, &azurev1alpha1.MySQLVNetRule{}) +} + +func (r *MySQLVNetRuleReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&azurev1alpha1.MySQLVNetRule{}). + Complete(r) +} diff --git a/controllers/mysqlvnetrule_controller_test.go b/controllers/mysqlvnetrule_controller_test.go new file mode 100644 index 00000000000..2281c6a39f9 --- /dev/null +++ b/controllers/mysqlvnetrule_controller_test.go @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// +build all mysql + +package controllers + +import ( + "context" + "strings" + "testing" + + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/stretchr/testify/assert" + + "github.com/Azure/azure-service-operator/pkg/errhelp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func TestMySqlVNetRuleControllerNoResourceGroup(t *testing.T) { + t.Parallel() + defer PanicRecover(t) + ctx := context.Background() + assert := assert.New(t) + + // Add any setup steps that needs to be executed before each test + mySqlServerName := GenerateTestResourceNameWithRandom("mysqlvnetrule-test-srv", 10) + mySqlVNetRuleName := GenerateTestResourceNameWithRandom("vnetrule-dev", 10) + + // Create the SqlVnetRule object and expect the Reconcile to be created + mySqlVNetRuleInstance := &azurev1alpha1.MySQLVNetRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: mySqlVNetRuleName, + Namespace: "default", + }, + Spec: azurev1alpha1.MySQLVNetRuleSpec{ + ResourceGroup: GenerateTestResourceNameWithRandom("rg-fake-srv", 10), + Server: mySqlServerName, + VNetResourceGroup: "vnet-rg", + VNetName: "test-vnet", + SubnetName: "subnet1", + IgnoreMissingServiceEndpoint: true, + }, + } + + err := tc.k8sClient.Create(ctx, mySqlVNetRuleInstance) + assert.Equal(nil, err, "create mysqlvnetrule in k8s") + + mySqlVNETRuleNamespacedName := types.NamespacedName{Name: mySqlVNetRuleName, Namespace: "default"} + + assert.Eventually(func() bool { + err = tc.k8sClient.Get(ctx, mySqlVNETRuleNamespacedName, mySqlVNetRuleInstance) + if err == nil { + return HasFinalizer(mySqlVNetRuleInstance, finalizerName) + } else { + return false + } + }, tc.timeout, tc.retry, "wait for mysqlvnetrule to have finalizer") + + assert.Eventually(func() bool { + err = tc.k8sClient.Get(ctx, mySqlVNETRuleNamespacedName, mySqlVNetRuleInstance) + if err == nil { + return strings.Contains(mySqlVNetRuleInstance.Status.Message, errhelp.ResourceGroupNotFoundErrorCode) + } else { + return false + } + }, tc.timeout, tc.retry, "wait for mysqlvnetrule to have rg not found error") + + err = tc.k8sClient.Delete(ctx, mySqlVNetRuleInstance) + assert.Equal(nil, err, "delete mysqlvnetrule in k8s") + + assert.Eventually(func() bool { + err = tc.k8sClient.Get(ctx, mySqlVNETRuleNamespacedName, mySqlVNetRuleInstance) + return apierrors.IsNotFound(err) + }, tc.timeout, tc.retry, "wait for mysqlvnetrule to be gone from k8s") +} + +func RunMySqlVNetRuleHappyPath(t *testing.T, mySqlServerName string, rgLocation string) { + defer PanicRecover(t) + ctx := context.Background() + + mySqlVNetRuleName := GenerateTestResourceNameWithRandom("vnet-rule", 10) + VNetName := GenerateTestResourceNameWithRandom("vnet", 10) + subnetName := "subnet-test" + VNetSubNetInstance := azurev1alpha1.VNetSubnets{ + SubnetName: subnetName, + SubnetAddressPrefix: "110.1.0.0/16", + } + + // Create a VNET + VNetInstance := &azurev1alpha1.VirtualNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: VNetName, + Namespace: "default", + }, + Spec: azurev1alpha1.VirtualNetworkSpec{ + Location: rgLocation, + ResourceGroup: tc.resourceGroupName, + AddressSpace: "110.0.0.0/8", + Subnets: []azurev1alpha1.VNetSubnets{VNetSubNetInstance}, + }, + } + + EnsureInstance(ctx, t, tc, VNetInstance) + + // Create a VNet Rule + + mySqlVNetRuleInstance := &azurev1alpha1.MySQLVNetRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: mySqlVNetRuleName, + Namespace: "default", + }, + Spec: azurev1alpha1.MySQLVNetRuleSpec{ + ResourceGroup: tc.resourceGroupName, + Server: mySqlServerName, + VNetResourceGroup: tc.resourceGroupName, + VNetName: VNetName, + SubnetName: subnetName, + IgnoreMissingServiceEndpoint: true, + }, + } + + // Create VNet Rule and ensure it was created + EnsureInstance(ctx, t, tc, mySqlVNetRuleInstance) + + // Delete a VNet Rule and ensure it was deleted + EnsureDelete(ctx, t, tc, mySqlVNetRuleInstance) + +} diff --git a/docs/mysql/mysql.md b/docs/mysql/mysql.md index 4185aac9197..3a4c7e229d3 100644 --- a/docs/mysql/mysql.md +++ b/docs/mysql/mysql.md @@ -55,6 +55,14 @@ The `server` indicates the MySQL server on which you want to configure the new M *Note*: When the `startIpAddress` and `endIpAddress` are 0.0.0.0, it denotes a special case that adds a firewall rule to allow all Azure services to access the server. +### MySQL virtual network rule + +The MySQL virtual network rule operator allows you to add virtual network rules to the MySQL server. + +The `server` indicates the MySQL server on which you want to configure the new MySQL virtual network rule on and `resourceGroup` is the resource group of the MySQL server. Provide the virtual network name and subnet name in the variables `vNetName` and `subnetName`, and `vNetResourceGroup` is the resource group the virtual network is located in. The `ignoreMissingServiceEndpoint` indicates whether or not to create virtual network rule before the virtual network has vnet service endpoint enabled. + +*Note*: When using MySQL Virtual Network Rules, the `Basic` SKU is not a valid op + ## Deploy, view and delete resources You can follow the steps [here](/docs/customresource.md) to deploy, view and delete resources. diff --git a/main.go b/main.go index b675d0a7b3e..a46481c13c1 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( mysqldatabase "github.com/Azure/azure-service-operator/pkg/resourcemanager/mysql/database" mysqlfirewall "github.com/Azure/azure-service-operator/pkg/resourcemanager/mysql/firewallrule" mysqlserver "github.com/Azure/azure-service-operator/pkg/resourcemanager/mysql/server" + mysqlvnetrule "github.com/Azure/azure-service-operator/pkg/resourcemanager/mysql/vnetrule" nic "github.com/Azure/azure-service-operator/pkg/resourcemanager/nic" pip "github.com/Azure/azure-service-operator/pkg/resourcemanager/pip" psqldatabase "github.com/Azure/azure-service-operator/pkg/resourcemanager/psql/database" @@ -644,6 +645,21 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "NetworkInterface") os.Exit(1) } + if err = (&controllers.MySQLVNetRuleReconciler{ + Reconciler: &controllers.AsyncReconciler{ + Client: mgr.GetClient(), + AzureClient: mysqlvnetrule.NewMySQLVNetRuleClient(), + Telemetry: telemetry.InitializeTelemetryDefault( + "MySQLVNetRule", + ctrl.Log.WithName("controllers").WithName("MySQLVNetRule"), + ), + Recorder: mgr.GetEventRecorderFor("MySQLVNetRule-controller"), + Scheme: scheme, + }, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "MySQLVNetRule") + os.Exit(1) + } if err = (&controllers.AzureVirtualMachineReconciler{ Reconciler: &controllers.AsyncReconciler{ diff --git a/pkg/resourcemanager/mysql/vnetrule/client.go b/pkg/resourcemanager/mysql/vnetrule/client.go new file mode 100644 index 00000000000..4f264b60b19 --- /dev/null +++ b/pkg/resourcemanager/mysql/vnetrule/client.go @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package server + +import ( + "context" + + mysql "github.com/Azure/azure-sdk-for-go/services/mysql/mgmt/2017-12-01/mysql" + network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-09-01/network" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/config" + "github.com/Azure/azure-service-operator/pkg/resourcemanager/iam" +) + +type MySQLVNetRuleClient struct { +} + +func NewMySQLVNetRuleClient() *MySQLVNetRuleClient { + return &MySQLVNetRuleClient{} +} + +func getMySQLVNetRulesClient() mysql.VirtualNetworkRulesClient { + VNetRulesClient := mysql.NewVirtualNetworkRulesClientWithBaseURI(config.BaseURI(), config.SubscriptionID()) + a, _ := iam.GetResourceManagementAuthorizer() + VNetRulesClient.Authorizer = a + VNetRulesClient.AddToUserAgent(config.UserAgent()) + return VNetRulesClient +} + +// GetNetworkSubnetClient retrieves a Subnetclient +func GetGoNetworkSubnetClient() network.SubnetsClient { + SubnetsClient := network.NewSubnetsClientWithBaseURI(config.BaseURI(), config.SubscriptionID()) + a, _ := iam.GetResourceManagementAuthorizer() + SubnetsClient.Authorizer = a + SubnetsClient.AddToUserAgent(config.UserAgent()) + return SubnetsClient +} + +// GetSQLVNetRule returns a VNet rule +func (vr *MySQLVNetRuleClient) GetSQLVNetRule(ctx context.Context, resourceGroupName string, serverName string, ruleName string) (result mysql.VirtualNetworkRule, err error) { + VNetRulesClient := getMySQLVNetRulesClient() + + return VNetRulesClient.Get( + ctx, + resourceGroupName, + serverName, + ruleName, + ) +} + +// DeleteSQLVNetRule deletes a VNet rule +func (vr *MySQLVNetRuleClient) DeleteSQLVNetRule(ctx context.Context, resourceGroupName string, serverName string, ruleName string) (err error) { + + // check to see if the rule exists, if it doesn't then short-circuit + _, err = vr.GetSQLVNetRule(ctx, resourceGroupName, serverName, ruleName) + if err != nil { + return nil + } + + VNetRulesClient := getMySQLVNetRulesClient() + _, err = VNetRulesClient.Delete( + ctx, + resourceGroupName, + serverName, + ruleName, + ) + + return err +} + +// CreateOrUpdateSQLVNetRule creates or updates a VNet rule +// based on code from: https://godoc.org/github.com/Azure/azure-sdk-for-go/services/preview/sql/mgmt/2015-05-01-preview/sql#VirtualNetworkRulesClient.CreateOrUpdate +func (vr *MySQLVNetRuleClient) CreateOrUpdateSQLVNetRule(ctx context.Context, resourceGroupName string, serverName string, ruleName string, VNetRG string, VNetName string, SubnetName string, IgnoreServiceEndpoint bool) (vnr mysql.VirtualNetworkRule, err error) { + + VNetRulesClient := getMySQLVNetRulesClient() + SubnetClient := GetGoNetworkSubnetClient() + + // Get ARM Resource ID of Subnet based on the VNET name, Subnet name and Subnet Address Prefix + subnet, err := SubnetClient.Get(ctx, VNetRG, VNetName, SubnetName, "") + if err != nil { + return vnr, err + } + subnetResourceID := *subnet.ID + + // Populate parameters with the right ID + parameters := mysql.VirtualNetworkRule{ + VirtualNetworkRuleProperties: &mysql.VirtualNetworkRuleProperties{ + VirtualNetworkSubnetID: &subnetResourceID, + IgnoreMissingVnetServiceEndpoint: &IgnoreServiceEndpoint, + }, + } + + // Call CreateOrUpdate + result, err := VNetRulesClient.CreateOrUpdate( + ctx, + resourceGroupName, + serverName, + ruleName, + parameters, + ) + return result.Result(VNetRulesClient) +} diff --git a/pkg/resourcemanager/mysql/vnetrule/manager.go b/pkg/resourcemanager/mysql/vnetrule/manager.go new file mode 100644 index 00000000000..c7cd5f657fd --- /dev/null +++ b/pkg/resourcemanager/mysql/vnetrule/manager.go @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package server + +import ( + "context" + + mysql "github.com/Azure/azure-sdk-for-go/services/mysql/mgmt/2017-12-01/mysql" + "github.com/Azure/azure-service-operator/pkg/resourcemanager" +) + +type MySqlVNetRuleManager interface { + CreateOrUpdateMySQLVNetRule(ctx context.Context, resourceGroupName string, serverName string, ruleName string, VNetRG string, VNetName string, SubnetName string, IgnoreServiceEndpoint bool) (result mysql.VirtualNetworkRule, err error) + DeleteMySQLVNetRule(ctx context.Context, resourceGroupName string, serverName string, ruleName string) (err error) + GetMySQLVNetRule(ctx context.Context, resourceGroupName string, serverName string, ruleName string) (result mysql.VirtualNetworkRule, err error) + resourcemanager.ARMClient +} diff --git a/pkg/resourcemanager/mysql/vnetrule/reconcile.go b/pkg/resourcemanager/mysql/vnetrule/reconcile.go new file mode 100644 index 00000000000..e49c0f8dc97 --- /dev/null +++ b/pkg/resourcemanager/mysql/vnetrule/reconcile.go @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package server + +import ( + "context" + "fmt" + "strings" + + mysql "github.com/Azure/azure-sdk-for-go/services/mysql/mgmt/2017-12-01/mysql" + azurev1alpha1 "github.com/Azure/azure-service-operator/api/v1alpha1" + "github.com/Azure/azure-service-operator/pkg/errhelp" + "github.com/Azure/azure-service-operator/pkg/helpers" + "github.com/Azure/azure-service-operator/pkg/resourcemanager" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" +) + +// Ensure creates a sqlvnetrule +func (vr *MySQLVNetRuleClient) Ensure(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { + instance, err := vr.convert(obj) + if err != nil { + return false, err + } + + groupName := instance.Spec.ResourceGroup + server := instance.Spec.Server + ruleName := instance.ObjectMeta.Name + virtualNetworkRG := instance.Spec.VNetResourceGroup + virtualnetworkname := instance.Spec.VNetName + subnetName := instance.Spec.SubnetName + ignoreendpoint := instance.Spec.IgnoreMissingServiceEndpoint + + vnetrule, err := vr.GetSQLVNetRule(ctx, groupName, server, ruleName) + if err == nil { + if vnetrule.VirtualNetworkRuleProperties != nil && vnetrule.VirtualNetworkRuleProperties.State == mysql.Ready { + instance.Status.Provisioning = false + instance.Status.Provisioned = true + instance.Status.Message = resourcemanager.SuccessMsg + instance.Status.ResourceId = *vnetrule.ID + return true, nil + } + return false, nil + } + instance.Status.Message = fmt.Sprintf("MySQLVNetRule Get error %s", err.Error()) + requeuErrors := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + } + azerr := errhelp.NewAzureErrorAzureError(err) + if helpers.ContainsString(requeuErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil + } + + instance.Status.Provisioning = true + _, err = vr.CreateOrUpdateSQLVNetRule(ctx, groupName, server, ruleName, virtualNetworkRG, virtualnetworkname, subnetName, ignoreendpoint) + if err != nil { + instance.Status.Message = err.Error() + azerr := errhelp.NewAzureErrorAzureError(err) + + if azerr.Type == errhelp.AsyncOpIncompleteError { + instance.Status.Provisioning = true + instance.Status.Message = "Resource request submitted to Azure successfully" + return false, nil + } + + ignorableErrors := []string{ + errhelp.ResourceGroupNotFoundErrorCode, + errhelp.ParentNotFoundErrorCode, + errhelp.ResourceNotFound, + } + if helpers.ContainsString(ignorableErrors, azerr.Type) { + instance.Status.Provisioning = false + return false, nil + } + + // this happens when we try to create the VNet rule and the server doesnt exist yet + errorString := err.Error() + if strings.Contains(errorString, "does not have the server") { + instance.Status.Provisioning = false + return false, nil + } + + return false, err + } + + return false, nil // We requeue so the success can be caught in the Get() path +} + +// Delete drops a sqlvnetrule +func (vr *MySQLVNetRuleClient) Delete(ctx context.Context, obj runtime.Object, opts ...resourcemanager.ConfigOption) (bool, error) { + instance, err := vr.convert(obj) + if err != nil { + return false, err + } + + groupName := instance.Spec.ResourceGroup + server := instance.Spec.Server + ruleName := instance.ObjectMeta.Name + + err = vr.DeleteSQLVNetRule(ctx, groupName, server, ruleName) + if err != nil { + instance.Status.Message = err.Error() + + azerr := errhelp.NewAzureErrorAzureError(err) + // these errors are expected + ignore := []string{ + errhelp.AsyncOpIncompleteError, + } + + // this means the thing doesn't exist + finished := []string{ + errhelp.ResourceNotFound, + errhelp.ParentNotFoundErrorCode, + } + + if helpers.ContainsString(ignore, azerr.Type) { + return true, nil //requeue + } + + if helpers.ContainsString(finished, azerr.Type) { + return false, nil //end reconcile + } + return false, err + } + + return false, nil +} + +// GetParents returns the parents of sqlvnetrule +func (vr *MySQLVNetRuleClient) GetParents(obj runtime.Object) ([]resourcemanager.KubeParent, error) { + instance, err := vr.convert(obj) + if err != nil { + return nil, err + } + + return []resourcemanager.KubeParent{ + { + Key: types.NamespacedName{ + Namespace: instance.Namespace, + Name: instance.Spec.Server, + }, + Target: &azurev1alpha1.MySQLServer{}, + }, + { + Key: types.NamespacedName{ + Namespace: instance.Namespace, + Name: instance.Spec.ResourceGroup, + }, + Target: &azurev1alpha1.ResourceGroup{}, + }, + }, nil +} + +func (vr *MySQLVNetRuleClient) GetStatus(obj runtime.Object) (*azurev1alpha1.ASOStatus, error) { + instance, err := vr.convert(obj) + if err != nil { + return nil, err + } + return &instance.Status, nil +} + +func (vr *MySQLVNetRuleClient) convert(obj runtime.Object) (*azurev1alpha1.MySQLVNetRule, error) { + local, ok := obj.(*azurev1alpha1.MySQLVNetRule) + if !ok { + return nil, fmt.Errorf("failed type assertion on kind: %s", obj.GetObjectKind().GroupVersionKind().String()) + } + return local, nil +}