From c1d1fd8fb5c01db4d097d58cc6d0696c65b4d282 Mon Sep 17 00:00:00 2001 From: Deepak Muley Date: Mon, 29 Apr 2024 11:07:55 -0700 Subject: [PATCH] added modular tests for cluster with topology for all supported k8s versions --- test/e2e/capx_quick_start_test.go | 2 +- test/e2e/cluster_topology.go | 316 ++++++++++++++++++ test/e2e/cluster_topology_basic_test.go | 113 +++++++ test/e2e/cluster_topology_conformance_test.go | 122 +++++++ test/e2e/cluster_topology_mega_scale_test.go | 113 +++++++ test/e2e/cluster_topology_scale_in_test.go | 188 +++++++++++ test/e2e/cluster_topology_scale_out_test.go | 188 +++++++++++ .../cluster_topology_scale_up_down_test.go | 312 +++++++++++++++++ test/e2e/cluster_topology_upgrade_test.go | 155 +++++++++ test/e2e/cluster_upgrade_test.go | 2 + test/e2e/clusterclass_changes_test.go | 2 +- test/e2e/config/nutanix.yaml | 10 +- test/e2e/test_helpers.go | 38 +++ 13 files changed, 1558 insertions(+), 3 deletions(-) create mode 100644 test/e2e/cluster_topology.go create mode 100644 test/e2e/cluster_topology_basic_test.go create mode 100644 test/e2e/cluster_topology_conformance_test.go create mode 100644 test/e2e/cluster_topology_mega_scale_test.go create mode 100644 test/e2e/cluster_topology_scale_in_test.go create mode 100644 test/e2e/cluster_topology_scale_out_test.go create mode 100644 test/e2e/cluster_topology_scale_up_down_test.go create mode 100644 test/e2e/cluster_topology_upgrade_test.go diff --git a/test/e2e/capx_quick_start_test.go b/test/e2e/capx_quick_start_test.go index 54561a3064..094b15a9cf 100644 --- a/test/e2e/capx_quick_start_test.go +++ b/test/e2e/capx_quick_start_test.go @@ -36,7 +36,7 @@ var _ = Describe("When following the Cluster API quick-start", Label("quickstart }) }) -var _ = Describe("When following the Cluster API quick-start with ClusterClass", Label("quickstart", "capx-feature-test"), func() { +var _ = Describe("When following the Cluster API quick-start with ClusterClass", Label("quickstart", "clusterclass", "capx-feature-test"), func() { capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { return capi_e2e.QuickStartSpecInput{ E2EConfig: e2eConfig, diff --git a/test/e2e/cluster_topology.go b/test/e2e/cluster_topology.go new file mode 100644 index 0000000000..8cc71b30e8 --- /dev/null +++ b/test/e2e/cluster_topology.go @@ -0,0 +1,316 @@ +//go:build e2e + +/* +Copyright 2024 Nutanix + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/kubetest" +) + +type NutanixE2ETest struct { + testHelper testHelperInterface + e2eConfig *clusterctl.E2EConfig + bootstrapClusterProxy framework.ClusterProxy + artifactFolder string + clusterctlConfigPath string + flavor string + namespace *corev1.Namespace + testSpecName string + kubetestConfigPath string +} + +type NutanixE2ETestOption func(*NutanixE2ETest) + +func NewNutanixE2ETest(options ...NutanixE2ETestOption) *NutanixE2ETest { + nutanixE2ETest := &NutanixE2ETest{} + for _, o := range options { + o(nutanixE2ETest) + } + return nutanixE2ETest +} + +func WithE2ETestSpecName(testSpecName string) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.testSpecName = testSpecName + } +} + +func WithE2ETestHelper(testHelper testHelperInterface) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.testHelper = testHelper + } +} + +func WithE2ETestConfig(e2eConfig *clusterctl.E2EConfig) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.e2eConfig = e2eConfig + } +} + +func WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy framework.ClusterProxy) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.bootstrapClusterProxy = bootstrapClusterProxy + } +} + +func WithE2ETestArtifactFolder(artifactFolder string) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.artifactFolder = artifactFolder + } +} + +func WithE2ETestClusterctlConfigPath(clusterctlConfigPath string) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.clusterctlConfigPath = clusterctlConfigPath + } +} + +func WithE2ETestKubetestConfigPath(kubetestConfigPath string) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.kubetestConfigPath = kubetestConfigPath + } +} + +func WithE2ETestClusterTemplateFlavor(flavor string) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.flavor = flavor + } +} + +func WithE2ETestNamespace(namespace *corev1.Namespace) NutanixE2ETestOption { + return func(nutanixE2ETest *NutanixE2ETest) { + nutanixE2ETest.namespace = namespace + } +} + +func (e2eTest *NutanixE2ETest) CreateCluster(ctx context.Context, clusterTopologyConfig *ClusterTopologyConfig) (*clusterctl.ApplyClusterTemplateAndWaitResult, error) { + configClusterInput := clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(e2eTest.artifactFolder, "clusters", e2eTest.bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: e2eTest.clusterctlConfigPath, + KubeconfigPath: e2eTest.bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: e2eTest.flavor, + Namespace: e2eTest.namespace.Name, + ClusterName: clusterTopologyConfig.name, + KubernetesVersion: clusterTopologyConfig.k8sVersion, + ControlPlaneMachineCount: ptr.To(int64(clusterTopologyConfig.cpNodeCount)), + WorkerMachineCount: ptr.To(int64(clusterTopologyConfig.workerNodeCount)), + } + + clusterResources := new(clusterctl.ApplyClusterTemplateAndWaitResult) + + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: e2eTest.bootstrapClusterProxy, + ConfigCluster: configClusterInput, + WaitForClusterIntervals: e2eTest.e2eConfig.GetIntervals("", "wait-cluster"), + WaitForControlPlaneIntervals: e2eTest.e2eConfig.GetIntervals("", "wait-control-plane"), + WaitForMachineDeployments: e2eTest.e2eConfig.GetIntervals("", "wait-worker-nodes"), + }, clusterResources) + + return clusterResources, nil +} + +func (e2eTest *NutanixE2ETest) UpgradeCluster(ctx context.Context, clusterTopologyConfig *ClusterTopologyConfig) (*clusterctl.ApplyClusterTemplateAndWaitResult, error) { + configClusterInput := clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(e2eTest.artifactFolder, "clusters", e2eTest.bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: e2eTest.clusterctlConfigPath, + KubeconfigPath: e2eTest.bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: e2eTest.flavor, + Namespace: e2eTest.namespace.Name, + ClusterName: clusterTopologyConfig.name, + KubernetesVersion: clusterTopologyConfig.k8sVersion, + ControlPlaneMachineCount: ptr.To(int64(clusterTopologyConfig.cpNodeCount)), + WorkerMachineCount: ptr.To(int64(clusterTopologyConfig.workerNodeCount)), + } + + clusterResources := new(clusterctl.ApplyClusterTemplateAndWaitResult) + + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: e2eTest.bootstrapClusterProxy, + ConfigCluster: configClusterInput, + WaitForClusterIntervals: e2eTest.e2eConfig.GetIntervals("", "wait-cluster"), + WaitForControlPlaneIntervals: e2eTest.e2eConfig.GetIntervals("", "wait-control-plane"), + WaitForMachineDeployments: e2eTest.e2eConfig.GetIntervals("", "wait-worker-nodes"), + }, clusterResources) + + return clusterResources, nil +} + +func (e2eTest *NutanixE2ETest) WaitForControlPlaneMachinesToBeUpgraded(ctx context.Context, clusterTopologyConfig *ClusterTopologyConfig, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) { + waitForMachinesToBeUpgraded := e2eTest.e2eConfig.GetIntervals("", "wait-machine-upgrade") + mgmtClient := e2eTest.bootstrapClusterProxy.GetClient() + framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{ + Lister: mgmtClient, + Cluster: clusterResources.Cluster, + MachineCount: int(clusterTopologyConfig.cpNodeCount), + KubernetesUpgradeVersion: clusterTopologyConfig.k8sVersion, + }, waitForMachinesToBeUpgraded...) +} + +func (e2eTest *NutanixE2ETest) WaitForMachineDeploymentMachinesToBeUpgraded(ctx context.Context, clusterTopologyConfig *ClusterTopologyConfig, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) { + waitForMachinesToBeUpgraded := e2eTest.e2eConfig.GetIntervals("", "wait-machine-upgrade") + mgmtClient := e2eTest.bootstrapClusterProxy.GetClient() + for _, deployment := range clusterResources.MachineDeployments { + if *deployment.Spec.Replicas > 0 { + framework.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, framework.WaitForMachineDeploymentMachinesToBeUpgradedInput{ + Lister: mgmtClient, + Cluster: clusterResources.Cluster, + MachineCount: int(*deployment.Spec.Replicas), + KubernetesUpgradeVersion: clusterTopologyConfig.k8sVersion, + MachineDeployment: *deployment, + }, waitForMachinesToBeUpgraded...) + } + } +} + +func (e2eTest *NutanixE2ETest) WaitForNodesReady(ctx context.Context, targetKubernetesVersion string, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) { + workloadProxy := e2eTest.bootstrapClusterProxy.GetWorkloadCluster(ctx, e2eTest.namespace.Name, clusterResources.Cluster.Name) + workloadClient := workloadProxy.GetClient() + framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{ + Lister: workloadClient, + KubernetesVersion: targetKubernetesVersion, + Count: int(clusterResources.ExpectedTotalNodes()), + WaitForNodesReady: e2eTest.e2eConfig.GetIntervals(e2eTest.testSpecName, "wait-nodes-ready"), + }) +} + +func (e2eTest *NutanixE2ETest) RunConformanceTest(ctx context.Context, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) error { + workloadProxy := e2eTest.bootstrapClusterProxy.GetWorkloadCluster(ctx, e2eTest.namespace.Name, clusterResources.Cluster.Name) + // Start running the conformance test suite. + return kubetest.Run( + ctx, + kubetest.RunInput{ + ClusterProxy: workloadProxy, + NumberOfNodes: int(clusterResources.ExpectedWorkerNodes()), + ArtifactsDirectory: e2eTest.artifactFolder, + ConfigFilePath: e2eTest.kubetestConfigPath, + GinkgoNodes: int(clusterResources.ExpectedWorkerNodes()), + }, + ) +} + +func (e2eTest *NutanixE2ETest) InstallAddonPackage(ctx context.Context, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult) error { + return fmt.Errorf("Not implemented yet.") +} + +type ClusterTopologyConfig struct { + name string + k8sVersion string + cpNodeCount int + workerNodeCount int + cpImageName string + workerImageName string + machineMemorySize string + machineSystemDiskSize string + machineVCPUSockets int64 + machineVCPUSPerSocket int64 +} + +type ClusterTopologyConfigOption func(*ClusterTopologyConfig) + +func NewClusterTopologyConfig(options ...func(*ClusterTopologyConfig)) *ClusterTopologyConfig { + clusterTopologyConfig := &ClusterTopologyConfig{} + for _, o := range options { + o(clusterTopologyConfig) + } + return clusterTopologyConfig +} + +// Start +// Option Pattern functions for ClusterTopologyConfig +// + +func WithName(name string) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.name = name + } +} + +func WithKubernetesVersion(k8sVersion string) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.k8sVersion = k8sVersion + } +} + +func WithControlPlaneCount(nodeCount int) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.cpNodeCount = nodeCount + } +} + +func WithWorkerNodeCount(nodeCount int) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.workerNodeCount = nodeCount + } +} + +func WithControlPlaneMachineTemplateImage(imageName string) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.cpImageName = imageName + os.Setenv("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME", imageName) + } +} + +func WithWorkerMachineTemplateImage(imageName string) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.workerImageName = imageName + os.Setenv("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME", imageName) + } +} + +func WithMachineMemorySize(machineMemorySize string) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.machineMemorySize = machineMemorySize + os.Setenv("NUTANIX_MACHINE_MEMORY_SIZE", machineMemorySize) + } +} + +func WithMachineSystemDiskSize(machineSystemDiskSize string) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.machineSystemDiskSize = machineSystemDiskSize + os.Setenv("NUTANIX_SYSTEMDISK_SIZE", machineSystemDiskSize) + } +} + +func WithMachineVCPUSockets(machineVCPUSockets int64) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.machineVCPUSockets = machineVCPUSockets + os.Setenv("NUTANIX_MACHINE_VCPU_SOCKET", fmt.Sprint(machineVCPUSockets)) + } +} + +func WithMachineVCPUSPerSocket(machineVCPUSPerSocket int64) ClusterTopologyConfigOption { + return func(clusterTopologyConfig *ClusterTopologyConfig) { + clusterTopologyConfig.machineVCPUSPerSocket = machineVCPUSPerSocket + os.Setenv("NUTANIX_MACHINE_VCPU_PER_SOCKET", fmt.Sprint(machineVCPUSPerSocket)) + } +} + +// +// Option Pattern functions for ClusterTopologyConfig +// End diff --git a/test/e2e/cluster_topology_basic_test.go b/test/e2e/cluster_topology_basic_test.go new file mode 100644 index 0000000000..b99125fc69 --- /dev/null +++ b/test/e2e/cluster_topology_basic_test.go @@ -0,0 +1,113 @@ +//go:build e2e + +/* +Copyright 2024 Nutanix + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +var _ = Describe("When creating a cluster with topology with simple workflow", Label("clusterclass", "only-for-validation"), func() { + const specName = "cluster-with-topology-simple-workflow" + const flavor = "topology" + + var ( + namespace *corev1.Namespace + clusterName string + cluster *clusterv1.Cluster + cancelWatches context.CancelFunc + testHelper testHelperInterface + nutanixE2ETest *NutanixE2ETest + ) + + BeforeEach(func() { + testHelper = newTestHelper(e2eConfig) + Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil") + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + Expect(namespace).NotTo(BeNil()) + nutanixE2ETest = NewNutanixE2ETest( + WithE2ETestSpecName(specName), + WithE2ETestHelper(testHelper), + WithE2ETestConfig(e2eConfig), + WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy), + WithE2ETestArtifactFolder(artifactFolder), + WithE2ETestClusterctlConfigPath(clusterctlConfigPath), + WithE2ETestClusterTemplateFlavor(flavor), + WithE2ETestNamespace(namespace), + ) + }) + + basicWorkflow := func(targetKubeVer, targetImageName string) { + By("Creating a workload cluster with topology") + clusterTopologyConfig := NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithControlPlaneCount(1), + WithWorkerNodeCount(1), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err := nutanixE2ETest.CreateCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + // Set test specific variable so that cluster can be cleaned up after each test + cluster = clusterResources.Cluster + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + By("PASSED!") + } + + AfterEach(func() { + dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, cluster, e2eConfig.GetIntervals, skipCleanup) + }) + + It("Create a cluster with topology with version Kube127", Label("Kube127", "cluster-topology-basic"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + basicWorkflow(kube127, kube127Image) + }) + + It("Create a cluster with topology with version Kube128", Label("Kube128", "cluster-topology-basic"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + basicWorkflow(kube128, kube128Image) + }) + + It("Create a cluster with topology with version Kube129", Label("Kube129", "cluster-topology-basic"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + basicWorkflow(kube129, kube129Image) + }) +}) diff --git a/test/e2e/cluster_topology_conformance_test.go b/test/e2e/cluster_topology_conformance_test.go new file mode 100644 index 0000000000..6d5178cf59 --- /dev/null +++ b/test/e2e/cluster_topology_conformance_test.go @@ -0,0 +1,122 @@ +//go:build e2e + +/* +Copyright 2024 Nutanix + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +var _ = Describe("When creating a cluster with topology and running kubetest/conformance", Label("clusterclass", "only-for-validation"), func() { + const specName = "cluster-with-topology-conformance" + const flavor = "topology" + + var ( + namespace *corev1.Namespace + clusterName string + cluster *clusterv1.Cluster + cancelWatches context.CancelFunc + testHelper testHelperInterface + nutanixE2ETest *NutanixE2ETest + ) + + BeforeEach(func() { + testHelper = newTestHelper(e2eConfig) + Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil") + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + Expect(namespace).NotTo(BeNil()) + + kubetestConfigFilePath := testHelper.getVariableFromE2eConfig("KUBETEST_CONFIGURATION") + Expect(kubetestConfigFilePath).To(BeAnExistingFile(), "%s should be a valid kubetest config file") + + nutanixE2ETest = NewNutanixE2ETest( + WithE2ETestSpecName(specName), + WithE2ETestHelper(testHelper), + WithE2ETestConfig(e2eConfig), + WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy), + WithE2ETestArtifactFolder(artifactFolder), + WithE2ETestClusterctlConfigPath(clusterctlConfigPath), + WithE2ETestKubetestConfigPath(kubetestConfigFilePath), + WithE2ETestClusterTemplateFlavor(flavor), + WithE2ETestNamespace(namespace), + ) + }) + + conformanceWorkflow := func(targetKubeVer, targetImageName string) { + By("Creating a workload cluster with topology") + clusterTopologyConfig := NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithControlPlaneCount(1), + WithWorkerNodeCount(1), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err := nutanixE2ETest.CreateCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + // Set test specific variable so that cluster can be cleaned up after each test + cluster = clusterResources.Cluster + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + By("Running conformance tests") + err = nutanixE2ETest.RunConformanceTest(ctx, clusterResources) + Expect(err).ToNot(HaveOccurred(), "Failed to run Kubernetes conformance") + + By("PASSED!") + } + + AfterEach(func() { + dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, cluster, e2eConfig.GetIntervals, skipCleanup) + }) + + It("Create a cluster with topology with version Kube127", Label("Kube127", "cluster-topology-conformance"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + conformanceWorkflow(kube127, kube127Image) + }) + + It("Create a cluster with topology with version Kube128", Label("Kube128", "cluster-topology-conformance"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + conformanceWorkflow(kube128, kube128Image) + }) + + It("Create a cluster with topology with version Kube129", Label("Kube129", "cluster-topology-conformance"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + conformanceWorkflow(kube129, kube129Image) + }) +}) diff --git a/test/e2e/cluster_topology_mega_scale_test.go b/test/e2e/cluster_topology_mega_scale_test.go new file mode 100644 index 0000000000..a2a0482c0a --- /dev/null +++ b/test/e2e/cluster_topology_mega_scale_test.go @@ -0,0 +1,113 @@ +//go:build e2e + +/* +Copyright 2024 Nutanix + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +var _ = Describe("When creating a cluster with topology with mega scale workflow", Label("clusterclass", "only-for-validation"), func() { + const specName = "cluster-with-topology-mega-scale-workflow" + const flavor = "topology" + + var ( + namespace *corev1.Namespace + clusterName string + cluster *clusterv1.Cluster + cancelWatches context.CancelFunc + testHelper testHelperInterface + nutanixE2ETest *NutanixE2ETest + ) + + BeforeEach(func() { + testHelper = newTestHelper(e2eConfig) + Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil") + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + Expect(namespace).NotTo(BeNil()) + nutanixE2ETest = NewNutanixE2ETest( + WithE2ETestSpecName(specName), + WithE2ETestHelper(testHelper), + WithE2ETestConfig(e2eConfig), + WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy), + WithE2ETestArtifactFolder(artifactFolder), + WithE2ETestClusterctlConfigPath(clusterctlConfigPath), + WithE2ETestClusterTemplateFlavor(flavor), + WithE2ETestNamespace(namespace), + ) + }) + + megaScaleWorkflow := func(targetKubeVer, targetImageName string, cpNodeCount, workerNodeCount int) { + By("Creating a workload cluster with topology") + clusterTopologyConfig := NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithControlPlaneCount(cpNodeCount), + WithWorkerNodeCount(workerNodeCount), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err := nutanixE2ETest.CreateCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + // Set test specific variable so that cluster can be cleaned up after each test + cluster = clusterResources.Cluster + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + By("PASSED!") + } + + AfterEach(func() { + dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, cluster, e2eConfig.GetIntervals, skipCleanup) + }) + + It("Create a cluster with topology with version Kube127 and 7 CP and 100 workers", Label("Kube127", "cluster-topology-mega-scale"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + megaScaleWorkflow(kube127, kube127Image, 7, 100) + }) + + It("Create a cluster with topology with version Kube128 and 7 CP and 100 workers", Label("Kube128", "cluster-topology-mega-scale"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + megaScaleWorkflow(kube128, kube128Image, 7, 100) + }) + + It("Create a cluster with topology with version Kube129 and 7 CP and 100 workers", Label("Kube129", "cluster-topology-mega-scale"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + megaScaleWorkflow(kube129, kube129Image, 7, 100) + }) +}) diff --git a/test/e2e/cluster_topology_scale_in_test.go b/test/e2e/cluster_topology_scale_in_test.go new file mode 100644 index 0000000000..6b56ac6ada --- /dev/null +++ b/test/e2e/cluster_topology_scale_in_test.go @@ -0,0 +1,188 @@ +//go:build e2e + +/* +Copyright 2024 Nutanix + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +var _ = Describe("When scaling in cluster with topology ", Label("clusterclass", "only-for-validation"), func() { + const specName = "cluster-with-topology-scale-in-workflow" + const flavor = "topology" + + var ( + namespace *corev1.Namespace + clusterName string + cluster *clusterv1.Cluster + cancelWatches context.CancelFunc + testHelper testHelperInterface + nutanixE2ETest *NutanixE2ETest + ) + + BeforeEach(func() { + testHelper = newTestHelper(e2eConfig) + Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil") + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + Expect(namespace).NotTo(BeNil()) + nutanixE2ETest = NewNutanixE2ETest( + WithE2ETestSpecName(specName), + WithE2ETestHelper(testHelper), + WithE2ETestConfig(e2eConfig), + WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy), + WithE2ETestArtifactFolder(artifactFolder), + WithE2ETestClusterctlConfigPath(clusterctlConfigPath), + WithE2ETestClusterTemplateFlavor(flavor), + WithE2ETestNamespace(namespace), + ) + }) + + AfterEach(func() { + dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, cluster, e2eConfig.GetIntervals, skipCleanup) + }) + + scaleOutWorkflow := func(targetKubeVer, targetImageName string, fromCPNodeCount, fromWorkerNodeCount, toCPNodeCount, toWorkerNodeCount int) { + By("Creating a workload cluster with topology") + clusterTopologyConfig := NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithControlPlaneCount(fromCPNodeCount), + WithWorkerNodeCount(fromWorkerNodeCount), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err := nutanixE2ETest.CreateCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + // Set test specific variable so that cluster can be cleaned up after each test + cluster = clusterResources.Cluster + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + clusterTopologyConfig = NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithControlPlaneCount(toCPNodeCount), + WithWorkerNodeCount(toWorkerNodeCount), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err = nutanixE2ETest.UpgradeCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + By("Waiting for control-plane machines to have the upgraded Kubernetes version") + nutanixE2ETest.WaitForControlPlaneMachinesToBeUpgraded(ctx, clusterTopologyConfig, clusterResources) + + By("Waiting for machine deployment machines to have the upgraded Kubernetes version") + nutanixE2ETest.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, clusterTopologyConfig, clusterResources) + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + By("PASSED!") + } + + It("Scale in a cluster with topology from 3 CP node to 1 CP nodes with Kube127", Label("Kube127", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleOutWorkflow(kube127, kube127Image, 3, 1, 1, 1) + }) + + It("Scale in a cluster with topology from 3 Worker node to 1 Worker nodes with Kube127", Label("Kube127", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleOutWorkflow(kube127, kube127Image, 1, 3, 1, 1) + }) + + It("Scale in a cluster with topology from 3 CP and Worker node to 1 CP and Worker nodes with Kube127", Label("Kube127", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleOutWorkflow(kube127, kube127Image, 3, 3, 1, 1) + }) + + It("Scale in a cluster with topology from 3 CP node to 1 CP nodes with Kube128", Label("Kube128", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleOutWorkflow(kube128, kube128Image, 3, 1, 1, 1) + }) + + It("Scale in a cluster with topology from 3 Worker node to 1 Worker nodes with Kube128", Label("Kube128", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleOutWorkflow(kube128, kube128Image, 1, 3, 1, 1) + }) + + It("Scale in a cluster with topology from 3 CP and Worker node to 1 CP and Worker nodes with Kube128", Label("Kube128", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleOutWorkflow(kube128, kube128Image, 3, 3, 1, 1) + }) + + It("Scale in a cluster with topology from 3 CP node to 1 CP nodes with Kube129", Label("Kube129", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleOutWorkflow(kube129, kube129Image, 3, 1, 1, 1) + }) + + It("Scale in a cluster with topology from 3 Worker node to 1 Worker nodes with Kube129", Label("Kube129", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleOutWorkflow(kube129, kube129Image, 1, 3, 1, 1) + }) + + It("Scale in a cluster with topology from 3 CP and Worker node to 1 CP and Worker nodes with Kube129", Label("Kube129", "cluster-topology-scale-in"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleOutWorkflow(kube129, kube129Image, 3, 3, 1, 1) + }) +}) diff --git a/test/e2e/cluster_topology_scale_out_test.go b/test/e2e/cluster_topology_scale_out_test.go new file mode 100644 index 0000000000..4dd0697ec3 --- /dev/null +++ b/test/e2e/cluster_topology_scale_out_test.go @@ -0,0 +1,188 @@ +//go:build e2e + +/* +Copyright 2024 Nutanix + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +var _ = Describe("When scaling out cluster with topology ", Label("clusterclass", "only-for-validation"), func() { + const specName = "cluster-with-topology-scale-out-workflow" + const flavor = "topology" + + var ( + namespace *corev1.Namespace + clusterName string + cluster *clusterv1.Cluster + cancelWatches context.CancelFunc + testHelper testHelperInterface + nutanixE2ETest *NutanixE2ETest + ) + + BeforeEach(func() { + testHelper = newTestHelper(e2eConfig) + Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil") + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + Expect(namespace).NotTo(BeNil()) + nutanixE2ETest = NewNutanixE2ETest( + WithE2ETestSpecName(specName), + WithE2ETestHelper(testHelper), + WithE2ETestConfig(e2eConfig), + WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy), + WithE2ETestArtifactFolder(artifactFolder), + WithE2ETestClusterctlConfigPath(clusterctlConfigPath), + WithE2ETestClusterTemplateFlavor(flavor), + WithE2ETestNamespace(namespace), + ) + }) + + AfterEach(func() { + dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, cluster, e2eConfig.GetIntervals, skipCleanup) + }) + + scaleOutWorkflow := func(targetKubeVer, targetImageName string, fromCPNodeCount, fromWorkerNodeCount, toCPNodeCount, toWorkerNodeCount int) { + By("Creating a workload cluster with topology") + clusterTopologyConfig := NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithControlPlaneCount(fromCPNodeCount), + WithWorkerNodeCount(fromWorkerNodeCount), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err := nutanixE2ETest.CreateCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + // Set test specific variable so that cluster can be cleaned up after each test + cluster = clusterResources.Cluster + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + clusterTopologyConfig = NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithControlPlaneCount(toCPNodeCount), + WithWorkerNodeCount(toWorkerNodeCount), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err = nutanixE2ETest.UpgradeCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + By("Waiting for control-plane machines to have the upgraded Kubernetes version") + nutanixE2ETest.WaitForControlPlaneMachinesToBeUpgraded(ctx, clusterTopologyConfig, clusterResources) + + By("Waiting for machine deployment machines to have the upgraded Kubernetes version") + nutanixE2ETest.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, clusterTopologyConfig, clusterResources) + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + By("PASSED!") + } + + It("Scale out a cluster with topology from 1 CP node to 3 CP nodes with Kube127", Label("Kube127", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleOutWorkflow(kube127, kube127Image, 1, 1, 3, 1) + }) + + It("Scale out a cluster with topology from 1 Worker node to 3 Worker nodes with Kube127", Label("Kube127", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleOutWorkflow(kube127, kube127Image, 1, 1, 1, 3) + }) + + It("Scale out a cluster with topology from 1 CP and Worker node to 3 CP and Worker nodes with Kube127", Label("Kube127", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleOutWorkflow(kube127, kube127Image, 1, 1, 3, 3) + }) + + It("Scale out a cluster with topology from 1 CP node to 3 CP nodes with Kube128", Label("Kube128", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleOutWorkflow(kube128, kube128Image, 1, 1, 3, 1) + }) + + It("Scale out a cluster with topology from 1 Worker node to 3 Worker nodes with Kube128", Label("Kube128", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleOutWorkflow(kube128, kube128Image, 1, 1, 1, 3) + }) + + It("Scale out a cluster with topology from 1 CP and Worker node to 3 CP and Worker nodes with Kube128", Label("Kube128", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleOutWorkflow(kube128, kube128Image, 1, 1, 3, 3) + }) + + It("Scale out a cluster with topology from 1 CP node to 3 CP nodes with Kube129", Label("Kube129", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleOutWorkflow(kube129, kube129Image, 1, 1, 3, 1) + }) + + It("Scale out a cluster with topology from 1 Worker node to 3 Worker nodes with Kube129", Label("Kube129", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleOutWorkflow(kube129, kube129Image, 1, 1, 1, 3) + }) + + It("Scale out a cluster with topology from 1 CP and Worker node to 3 CP and Worker nodes with Kube129", Label("Kube129", "cluster-topology-scale-out"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleOutWorkflow(kube129, kube129Image, 1, 1, 3, 3) + }) +}) diff --git a/test/e2e/cluster_topology_scale_up_down_test.go b/test/e2e/cluster_topology_scale_up_down_test.go new file mode 100644 index 0000000000..92aeb394e2 --- /dev/null +++ b/test/e2e/cluster_topology_scale_up_down_test.go @@ -0,0 +1,312 @@ +//go:build e2e + +/* +Copyright 2024 Nutanix + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +var _ = Describe("When scaling up/down cluster with topology ", Label("clusterclass", "only-for-validation"), func() { + const specName = "cluster-with-topology-scale-up-down-workflow" + const flavor = "topology" + + var ( + namespace *corev1.Namespace + clusterName string + cluster *clusterv1.Cluster + cancelWatches context.CancelFunc + testHelper testHelperInterface + nutanixE2ETest *NutanixE2ETest + ) + + BeforeEach(func() { + testHelper = newTestHelper(e2eConfig) + Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil") + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + Expect(namespace).NotTo(BeNil()) + nutanixE2ETest = NewNutanixE2ETest( + WithE2ETestSpecName(specName), + WithE2ETestHelper(testHelper), + WithE2ETestConfig(e2eConfig), + WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy), + WithE2ETestArtifactFolder(artifactFolder), + WithE2ETestClusterctlConfigPath(clusterctlConfigPath), + WithE2ETestClusterTemplateFlavor(flavor), + WithE2ETestNamespace(namespace), + ) + }) + + AfterEach(func() { + dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, cluster, e2eConfig.GetIntervals, skipCleanup) + }) + + scaleUpDownWorkflow := func( + targetKubeVer, + targetImageName, + fromMachineMemorySizeGibStr, + toMachineMemorySizeGibStr, + fromMachineSystemDiskSizeGibStr, + toMachineSystemDiskSizeGibStr string, + fromMachineVCPUSockets, + toMachineVCPUSockets, + fromMachineVCPUsPerSocket, + toMachineVCPUsPerSocket int64, + ) { + By("Creating a workload cluster with topology") + clusterTopologyConfig := NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithMachineMemorySize(fromMachineMemorySizeGibStr), + WithMachineSystemDiskSize(fromMachineSystemDiskSizeGibStr), + WithMachineVCPUSockets(fromMachineVCPUSockets), + WithMachineVCPUSPerSocket(fromMachineVCPUsPerSocket), + WithControlPlaneCount(1), + WithWorkerNodeCount(1), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err := nutanixE2ETest.CreateCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + // Set test specific variable so that cluster can be cleaned up after each test + cluster = clusterResources.Cluster + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + clusterTopologyConfig = NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(targetKubeVer), + WithMachineMemorySize(toMachineMemorySizeGibStr), + WithMachineSystemDiskSize(toMachineSystemDiskSizeGibStr), + WithMachineVCPUSockets(toMachineVCPUSockets), + WithMachineVCPUSPerSocket(toMachineVCPUsPerSocket), + WithControlPlaneCount(1), + WithWorkerNodeCount(1), + WithControlPlaneMachineTemplateImage(targetImageName), + WithWorkerMachineTemplateImage(targetImageName), + ) + + clusterResources, err = nutanixE2ETest.UpgradeCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + By("Waiting for control-plane machines to have the upgraded Kubernetes version") + nutanixE2ETest.WaitForControlPlaneMachinesToBeUpgraded(ctx, clusterTopologyConfig, clusterResources) + + By("Waiting for machine deployment machines to have the upgraded Kubernetes version") + nutanixE2ETest.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, clusterTopologyConfig, clusterResources) + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, targetKubeVer, clusterResources) + + var toMachineMemorySizeGib int64 + fmt.Sscan(toMachineMemorySizeGibStr, &toMachineMemorySizeGib) + var toMachineSystemDiskSizeGib int64 + fmt.Sscan(toMachineSystemDiskSizeGibStr, &toMachineSystemDiskSizeGib) + By("Check if all the machines have scaled down resource config (memory size, VCPUSockets, vcpusPerSocket)") + testHelper.verifyResourceConfigOnNutanixMachines(ctx, verifyResourceConfigOnNutanixMachinesParams{ + clusterName: clusterName, + namespace: namespace, + toMachineMemorySizeGib: toMachineMemorySizeGib, + toMachineSystemDiskSizeGib: toMachineSystemDiskSizeGib, + toMachineVCPUSockets: toMachineVCPUSockets, + toMachineVCPUsPerSocket: toMachineVCPUsPerSocket, + bootstrapClusterProxy: bootstrapClusterProxy, + }) + + By("PASSED!") + } + + // Scale Down Test Start + It("Scale down a cluster with CP and Worker node machine memory size from 4Gi to 3Gi with Kube127", Label("Kube127", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleUpDownWorkflow(kube127, kube127Image, "4Gi", "3Gi", "40Gi", "40Gi", 2, 2, 1, 1) + }) + + It("Scale down a cluster with CP and Worker node machine memory size from 4Gi to 3Gi with Kube128", Label("Kube128", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleUpDownWorkflow(kube128, kube128Image, "4Gi", "3Gi", "40Gi", "40Gi", 2, 2, 1, 1) + }) + + It("Scale down a cluster with CP and Worker node machine memory size from 4Gi to 3Gi with Kube129", Label("Kube129", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + Kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + Kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleUpDownWorkflow(Kube129, Kube129Image, "4Gi", "3Gi", "40Gi", "40Gi", 2, 2, 1, 1) + }) + + It("Scale down a cluster with CP and Worker node VCPUSockets from 3 to 2 with Kube127", Label("Kube127", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleUpDownWorkflow(kube127, kube127Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 2, 1, 1) + }) + + It("Scale down a cluster with CP and Worker node VCPUSockets from 3 to 2 with Kube128", Label("Kube128", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleUpDownWorkflow(kube128, kube128Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 2, 1, 1) + }) + + It("Scale down a cluster with CP and Worker node machine VCPUSockets from 3 to 2 with Kube129", Label("Kube129", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + Kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + Kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleUpDownWorkflow(Kube129, Kube129Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 2, 1, 1) + }) + + It("Scale down a cluster with CP and Worker node vcpu per socket from 2 to 1 with Kube127", Label("Kube127", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleUpDownWorkflow(kube127, kube127Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 3, 2, 1) + }) + + It("Scale down a cluster with CP and Worker node vcpu per socket from 2 to 1 with Kube128", Label("Kube128", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleUpDownWorkflow(kube128, kube128Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 3, 2, 1) + }) + + It("Scale down a cluster with CP and Worker node machine vcpu per socket from 2 to 1 with Kube129", Label("Kube129", "cluster-topology-scale-down"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + Kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + Kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleUpDownWorkflow(Kube129, Kube129Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 3, 2, 1) + }) + + // TODO add system disk scale down tests + // Scale Down Test End + + // Scale Up Test Start + It("Scale up a cluster with CP and Worker node machine memory size from 3Gi to 4Gi with Kube127", Label("Kube127", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleUpDownWorkflow(kube127, kube127Image, "3Gi", "4Gi", "40Gi", "40Gi", 2, 2, 1, 1) + }) + + It("Scale up a cluster with CP and Worker node machine memory size from 3Gi to 4Gi with Kube128", Label("Kube128", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleUpDownWorkflow(kube128, kube128Image, "3Gi", "4Gi", "40Gi", "40Gi", 2, 2, 1, 1) + }) + + It("Scale up a cluster with CP and Worker node machine memory size from 3Gi to 4Gi with Kube129", Label("Kube129", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + Kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + Kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleUpDownWorkflow(Kube129, Kube129Image, "3Gi", "4Gi", "40Gi", "40Gi", 2, 2, 1, 1) + }) + + It("Scale up a cluster with CP and Worker node VCPUSockets from 2 to 3 with Kube127", Label("Kube127", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleUpDownWorkflow(kube127, kube127Image, "4Gi", "4Gi", "40Gi", "40Gi", 2, 3, 1, 1) + }) + + It("Scale up a cluster with CP and Worker node VCPUSockets from 2 to 3 with Kube128", Label("Kube128", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleUpDownWorkflow(kube128, kube128Image, "4Gi", "4Gi", "40Gi", "40Gi", 2, 3, 1, 1) + }) + + It("Scale up a cluster with CP and Worker node machine VCPUSockets from 2 to 3 with Kube129", Label("Kube129", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + Kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + Kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleUpDownWorkflow(Kube129, Kube129Image, "4Gi", "4Gi", "40Gi", "40Gi", 2, 3, 1, 1) + }) + + It("Scale up a cluster with CP and Worker node vcpu per socket from 1 to 2 with Kube127", Label("Kube127", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + scaleUpDownWorkflow(kube127, kube127Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 3, 1, 2) + }) + + It("Scale up a cluster with CP and Worker node vcpu per socket from 1 to 2 with Kube128", Label("Kube128", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + scaleUpDownWorkflow(kube128, kube128Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 3, 1, 2) + }) + + It("Scale up a cluster with CP and Worker node machine vcpu per socket from 1 to 2 with Kube129", Label("Kube129", "cluster-topology-scale-up"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + Kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + Kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + scaleUpDownWorkflow(Kube129, Kube129Image, "4Gi", "4Gi", "40Gi", "40Gi", 3, 3, 1, 2) + }) + + // TODO add system disk scale up tests + // Scale Up Test End +}) diff --git a/test/e2e/cluster_topology_upgrade_test.go b/test/e2e/cluster_topology_upgrade_test.go new file mode 100644 index 0000000000..43654caf71 --- /dev/null +++ b/test/e2e/cluster_topology_upgrade_test.go @@ -0,0 +1,155 @@ +//go:build e2e + +/* +Copyright 2024 Nutanix + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +var _ = Describe("When upgrading the k8s version of cluster with topology", Label("clusterclass", "only-for-validation"), func() { + const specName = "cluster-with-topology-upgrade-workflow" + const flavor = "topology" + + var ( + namespace *corev1.Namespace + clusterName string + cluster *clusterv1.Cluster + cancelWatches context.CancelFunc + testHelper testHelperInterface + nutanixE2ETest *NutanixE2ETest + ) + + BeforeEach(func() { + testHelper = newTestHelper(e2eConfig) + Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil") + namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) + Expect(namespace).NotTo(BeNil()) + nutanixE2ETest = NewNutanixE2ETest( + WithE2ETestSpecName(specName), + WithE2ETestHelper(testHelper), + WithE2ETestConfig(e2eConfig), + WithE2ETestBootstrapClusterProxy(bootstrapClusterProxy), + WithE2ETestArtifactFolder(artifactFolder), + WithE2ETestClusterctlConfigPath(clusterctlConfigPath), + WithE2ETestClusterTemplateFlavor(flavor), + WithE2ETestNamespace(namespace), + ) + }) + + AfterEach(func() { + dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, cluster, e2eConfig.GetIntervals, skipCleanup) + }) + + upgradeWorkflow := func(fromKubeVer, fromImageName, toKubeVer, toImageName string) { + By("Creating a workload cluster with topology") + clusterTopologyConfig := NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(fromKubeVer), + WithControlPlaneCount(1), + WithWorkerNodeCount(1), + WithControlPlaneMachineTemplateImage(fromImageName), + WithWorkerMachineTemplateImage(fromImageName), + ) + + clusterResources, err := nutanixE2ETest.CreateCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + // Set test specific variable so that cluster can be cleaned up after each test + cluster = clusterResources.Cluster + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, fromKubeVer, clusterResources) + + clusterTopologyConfig = NewClusterTopologyConfig( + WithName(clusterName), + WithKubernetesVersion(toKubeVer), + WithControlPlaneCount(1), + WithWorkerNodeCount(1), + WithControlPlaneMachineTemplateImage(toImageName), + WithWorkerMachineTemplateImage(toImageName), + ) + + clusterResources, err = nutanixE2ETest.UpgradeCluster(ctx, clusterTopologyConfig) + Expect(err).ToNot(HaveOccurred()) + + By("Waiting for control-plane machines to have the upgraded Kubernetes version") + nutanixE2ETest.WaitForControlPlaneMachinesToBeUpgraded(ctx, clusterTopologyConfig, clusterResources) + + By("Waiting for machine deployment machines to have the upgraded Kubernetes version") + nutanixE2ETest.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, clusterTopologyConfig, clusterResources) + + By("Waiting until nodes are ready") + nutanixE2ETest.WaitForNodesReady(ctx, toKubeVer, clusterResources) + + By("PASSED!") + } + + It("Upgrade a cluster with topology from version Kube126 to Kube127", Label("Kube127", "cluster-topology-upgrade"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube126 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_26") + kube126Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_26") + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + upgradeWorkflow(kube126, kube126Image, kube127, kube127Image) + }) + + It("Upgrade a cluster with topology from version Kube127 to Kube128", Label("Kube128", "cluster-topology-upgrade"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube127 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_27") + kube127Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27") + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + upgradeWorkflow(kube127, kube127Image, kube128, kube128Image) + }) + + // It("Upgrade a cluster with topology from version Kube126 to Kube128 and expect failure", Label("Kube128", "cluster-topology-upgrade-test"), func() { + // // NOTE: Following test will always fail and did not find a way to not fail as function we call has Expect call + // // For more info on why refer https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/operate-cluster + // // we get following error + // // The Cluster "cluster-with-topology-upgrade-workflow-u318rg" is invalid: spec.topology.version: Forbidden: version cannot be increased from "1.26.13" to "1.28.6" + // clusterName = testHelper.generateTestClusterName(specName) + // Expect(clusterName).NotTo(BeNil()) + + // kube126 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_26") + // kube126Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_26") + // kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + // kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + // upgradeWorkflow(kube126, kube126Image, kube128, kube128Image) + // }) + + It("Upgrade a cluster with topology from version Kube128 to Kube129", Label("Kube129", "cluster-topology-upgrade"), func() { + clusterName = testHelper.generateTestClusterName(specName) + Expect(clusterName).NotTo(BeNil()) + + kube128 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_28") + kube128Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28") + kube129 := testHelper.getVariableFromE2eConfig("KUBERNETES_VERSION_v1_29") + kube129Image := testHelper.getVariableFromE2eConfig("NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29") + upgradeWorkflow(kube128, kube128Image, kube129, kube129Image) + }) +}) diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index 7ce015dbff..59d559e8d0 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -37,6 +37,8 @@ var _ = Describe("When upgrading a workload cluster and testing K8S conformance" }) }) +// NOTE: following test does not work as nutanix provider needs cp and md imageName as an extra variable to be updated with +// correct image path with the upgradeTo k8s version. // var _ = Describe("When upgrading a workload cluster using ClusterClass", func() { // ClusterUpgradeConformanceSpec(ctx, func() ClusterUpgradeConformanceSpecInput { // return ClusterUpgradeConformanceSpecInput{ diff --git a/test/e2e/clusterclass_changes_test.go b/test/e2e/clusterclass_changes_test.go index b46f7fdddd..17ac49f52f 100644 --- a/test/e2e/clusterclass_changes_test.go +++ b/test/e2e/clusterclass_changes_test.go @@ -25,7 +25,7 @@ import ( capi_e2e "sigs.k8s.io/cluster-api/test/e2e" ) -var _ = Describe("When testing ClusterClass changes [ClusterClass]", Label("clusterclass", "slow", "network"), func() { +var _ = Describe("When mutating ClusterClass fields", Label("clusterclass", "capx-feature-test"), func() { capi_e2e.ClusterClassChangesSpec(ctx, func() capi_e2e.ClusterClassChangesSpecInput { return capi_e2e.ClusterClassChangesSpecInput{ E2EConfig: e2eConfig, diff --git a/test/e2e/config/nutanix.yaml b/test/e2e/config/nutanix.yaml index 0ae45e2e62..7e220e87c4 100644 --- a/test/e2e/config/nutanix.yaml +++ b/test/e2e/config/nutanix.yaml @@ -234,13 +234,21 @@ variables: NUTANIX_INSECURE: true NUTANIX_ADDITIONAL_TRUST_BUNDLE: "" KUBERNETES_VERSION: "v1.29.2" + KUBERNETES_VERSION_v1_26: "v1.26.13" + KUBERNETES_VERSION_v1_27: "v1.27.9" + KUBERNETES_VERSION_v1_28: "v1.28.6" + KUBERNETES_VERSION_v1_29: "v1.29.2" NUTANIX_SSH_AUTHORIZED_KEY: "" CONTROL_PLANE_ENDPOINT_IP: "" CONTROL_PLANE_ENDPOINT_IP_V124: "" CONTROL_PLANE_MACHINE_COUNT: 3 WORKER_MACHINE_COUNT: 3 NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: "" - NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: "ubuntu-2204-kube-v1.29.2.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: "ubuntu-2004-kube-v1.29.2.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_26: "ubuntu-2204-kube-v1.26.13.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_27: "ubuntu-2204-kube-v1.27.9.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_28: "ubuntu-2204-kube-v1.28.6.qcow2" + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME_v1_29: "ubuntu-2204-kube-v1.29.2.qcow2" CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO: "cluster-upgrade-conformance" WORKERS_MACHINE_TEMPLATE_UPGRADE_TO: "cluster-upgrade-conformance" NUTANIX_MACHINE_TEMPLATE_IMAGE_UPGRADE_TO: "ubuntu-2204-kube-v1.29.2.qcow2" diff --git a/test/e2e/test_helpers.go b/test/e2e/test_helpers.go index a60ee45d5e..74e76e411e 100644 --- a/test/e2e/test_helpers.go +++ b/test/e2e/test_helpers.go @@ -151,6 +151,7 @@ type testHelperInterface interface { verifyFailureMessageOnClusterMachines(ctx context.Context, params verifyFailureMessageOnClusterMachinesParams) verifyGPUNutanixMachines(ctx context.Context, params verifyGPUNutanixMachinesParams) verifyProjectNutanixMachines(ctx context.Context, params verifyProjectNutanixMachinesParams) + verifyResourceConfigOnNutanixMachines(ctx context.Context, params verifyResourceConfigOnNutanixMachinesParams) } type testHelper struct { @@ -580,6 +581,43 @@ func (t testHelper) verifyCategoriesNutanixMachines(ctx context.Context, cluster } } +type verifyResourceConfigOnNutanixMachinesParams struct { + clusterName string + namespace *corev1.Namespace + toMachineMemorySizeGib int64 + toMachineSystemDiskSizeGib int64 + toMachineVCPUSockets int64 + toMachineVCPUsPerSocket int64 + bootstrapClusterProxy framework.ClusterProxy +} + +func (t testHelper) verifyResourceConfigOnNutanixMachines(ctx context.Context, params verifyResourceConfigOnNutanixMachinesParams) { + Eventually( + func(g Gomega) { + nutanixMachines := t.getMachinesForCluster(ctx, + params.clusterName, + params.namespace.Name, + params.bootstrapClusterProxy) + for _, m := range nutanixMachines.Items { + machineProviderID := m.Spec.ProviderID + g.Expect(machineProviderID).NotTo(BeNil()) + machineVmUUID := t.stripNutanixIDFromProviderID(*machineProviderID) + vm, err := t.nutanixClient.V3.GetVM(ctx, machineVmUUID) + g.Expect(err).ShouldNot(HaveOccurred()) + vmMemorySizeInMib := *vm.Status.Resources.MemorySizeMib + g.Expect(vmMemorySizeInMib).To(Equal(params.toMachineMemorySizeGib*1024), "expected memory size of VMs to be equal to %d but was %d", params.toMachineMemorySizeGib*1024, vmMemorySizeInMib) + vmNumSockets := *vm.Status.Resources.NumSockets + g.Expect(vmNumSockets).To(Equal(params.toMachineVCPUSockets), "expected num sockets of VMs to be equal to %d but was %d", params.toMachineVCPUSockets, vmNumSockets) + vmNumVcpusPerSocket := *vm.Status.Resources.NumVcpusPerSocket + g.Expect(vmNumVcpusPerSocket).To(Equal(params.toMachineVCPUsPerSocket), "expected vcpu per socket of VMs to be equal to %d but was %d", params.toMachineVCPUsPerSocket, vmNumVcpusPerSocket) + // TODO check system disk size as well + } + }, + defaultTimeout, + defaultInterval, + ).Should(Succeed()) +} + type verifyConditionParams struct { bootstrapClusterProxy framework.ClusterProxy clusterName string