diff --git a/test/e2e/e2eutil/common.go b/test/e2e/e2eutil/common.go index a392fe57..75a8e2b9 100644 --- a/test/e2e/e2eutil/common.go +++ b/test/e2e/e2eutil/common.go @@ -14,6 +14,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -33,6 +34,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const TestIDLabel = "test-id" + func WithReplicas(obj *unstructured.Unstructured, replicas int) *unstructured.Unstructured { err := unstructured.SetNestedField(obj.Object, int64(replicas), "spec", "replicas") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -270,9 +273,7 @@ func Run(ch <-chan event.Event) error { return err } -func RunWithNoErr(ch <-chan event.Event) { - RunCollectNoErr(ch) -} +var RunWithNoErr = RunCollectNoErr func RunCollect(ch <-chan event.Event) []event.Event { var events []event.Event @@ -282,14 +283,71 @@ func RunCollect(ch <-chan event.Event) []event.Event { return events } -func RunCollectNoErr(ch <-chan event.Event) []event.Event { - events := RunCollect(ch) - for _, e := range events { - gomega.Expect(e.Type).NotTo(gomega.Equal(event.ErrorType)) +func RunCollectNoErr(ch <-chan event.Event, callerSkip ...int) []event.Event { + skip := 0 + if len(callerSkip) > 0 { + skip = callerSkip[0] } + + events := RunCollect(ch) + ExpectNoEventErrors(events, skip+1) + ExpectNoReconcileTimeouts(events, skip+1) return events } +func ExpectNoEventErrors(events []event.Event, callerSkip ...int) { + skip := 0 + if len(callerSkip) > 0 { + skip = callerSkip[0] + } + + gomega.Expect(events).WithOffset(skip + 1).NotTo( + gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, + gstruct.Fields{ + "Type": gomega.Equal(event.ErrorType), + }))) + gomega.Expect(events).WithOffset(skip + 1).NotTo( + gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, + gstruct.Fields{ + "Type": gomega.Equal(event.ApplyType), + "ApplyEvent": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Status": gomega.Equal(event.ApplyFailed), + }), + }))) + gomega.Expect(events).WithOffset(skip + 1).NotTo( + gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, + gstruct.Fields{ + "Type": gomega.Equal(event.PruneType), + "PruneEvent": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Status": gomega.Equal(event.PruneFailed), + }), + }))) + gomega.Expect(events).WithOffset(skip + 1).NotTo( + gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, + gstruct.Fields{ + "Type": gomega.Equal(event.DeleteType), + "DeleteEvent": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Status": gomega.Equal(event.DeleteFailed), + }), + }))) +} + +func ExpectNoReconcileTimeouts(events []event.Event, callerSkip ...int) { + skip := 0 + if len(callerSkip) > 0 { + skip = callerSkip[0] + } + + gomega.Expect(events).WithOffset(skip + 1).NotTo( + gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, + gstruct.Fields{ + "Type": gomega.Equal(event.WaitType), + "WaitEvent": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Status": gomega.Equal(event.ReconcileTimeout), + }), + }))) +} + func ManifestToUnstructured(manifest []byte) *unstructured.Unstructured { u := make(map[string]interface{}) err := yaml.Unmarshal(manifest, &u) diff --git a/test/stress/stress_test.go b/test/stress/stress_test.go index b2a3f97b..b9810985 100644 --- a/test/stress/stress_test.go +++ b/test/stress/stress_test.go @@ -101,6 +101,10 @@ var _ = Describe("Stress", func() { thousandDeploymentsTest(ctx, c, invConfig, inventoryName, namespace.GetName()) }) + It("ThousandDeploymentsRetry", func() { + thousandDeploymentsRetryTest(ctx, c, invConfig, inventoryName, namespace.GetName()) + }) + It("ThousandNamespaces", func() { thousandNamespacesTest(ctx, c, invConfig, inventoryName, namespace.GetName()) }) diff --git a/test/stress/thousand_deployments_retry_test.go b/test/stress/thousand_deployments_retry_test.go new file mode 100644 index 00000000..e4587999 --- /dev/null +++ b/test/stress/thousand_deployments_retry_test.go @@ -0,0 +1,155 @@ +// Copyright 2022 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package stress + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" + "sigs.k8s.io/cli-utils/pkg/apply" + "sigs.k8s.io/cli-utils/pkg/apply/event" + "sigs.k8s.io/cli-utils/pkg/common" + "sigs.k8s.io/cli-utils/pkg/inventory" + "sigs.k8s.io/cli-utils/test/e2e/e2eutil" + "sigs.k8s.io/cli-utils/test/e2e/invconfig" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// thousandDeploymentsRetryTest tests one pre-existing namespace with 1,000 +// Deployments in it. The wait timeout is set too short to confirm +// reconciliation, but the apply/destroy is retried until success. +// +// The Deployments themselves are easy to get status on, but with the retrieval +// of generated resource status (ReplicaSets & Pods), this becomes expensive. +func thousandDeploymentsRetryTest(ctx context.Context, c client.Client, invConfig invconfig.InventoryConfig, inventoryName, namespaceName string) { + By("Apply LOTS of resources") + applier := invConfig.ApplierFactoryFunc() + inventoryID := fmt.Sprintf("%s-%s", inventoryName, namespaceName) + + inventoryInfo := invconfig.CreateInventoryInfo(invConfig, inventoryName, namespaceName, inventoryID) + + resources := []*unstructured.Unstructured{} + + deploymentObjTemplate := e2eutil.ManifestToUnstructured([]byte(deploymentYaml)) + deploymentObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID}) + + objectCount := 1000 + + for i := 1; i <= objectCount; i++ { + deploymentObj := deploymentObjTemplate.DeepCopy() + deploymentObj.SetNamespace(namespaceName) + + // change name & selector labels to avoid overlap between deployments + name := fmt.Sprintf("nginx-%d", i) + deploymentObj.SetName(name) + err := unstructured.SetNestedField(deploymentObj.Object, name, "spec", "selector", "matchLabels", "app") + Expect(err).ToNot(HaveOccurred()) + err = unstructured.SetNestedField(deploymentObj.Object, name, "spec", "template", "metadata", "labels", "app") + Expect(err).ToNot(HaveOccurred()) + + resources = append(resources, deploymentObj) + } + + defer func() { + By("Cleanup Deployments") + e2eutil.DeleteAllUnstructuredIfExists(ctx, c, deploymentObjTemplate) + }() + + startTotal := time.Now() + + var applierEvents []event.Event + + maxAttempts := 15 + reconcileTimeout := 2 * time.Minute + + for attempt := 1; attempt <= maxAttempts; attempt++ { + start := time.Now() + + applierEvents = e2eutil.RunCollect(applier.Run(ctx, inventoryInfo, resources, apply.ApplierOptions{ + // SSA reduces GET+PATCH to just PATCH, which is faster + ServerSideOptions: common.ServerSideOptions{ + ServerSideApply: true, + ForceConflicts: true, + FieldManager: "cli-utils.kubernetes.io", + }, + ReconcileTimeout: reconcileTimeout, + EmitStatusEvents: false, + })) + + duration := time.Since(start) + klog.Infof("Applier.Run execution time (attempt: %d): %v", attempt, duration) + + e2eutil.ExpectNoEventErrors(applierEvents) + + // Retry if ReconcileTimeout + retry := false + for _, e := range applierEvents { + if e.Type == event.WaitType && e.WaitEvent.Status == event.ReconcileTimeout { + retry = true + } + } + if !retry { + break + } + } + + durationTotal := time.Since(startTotal) + klog.Infof("Applier.Run total execution time (attempts: %d): %v", maxAttempts, durationTotal) + + e2eutil.ExpectNoReconcileTimeouts(applierEvents) + + By("Verify inventory created") + invConfig.InvSizeVerifyFunc(ctx, c, inventoryName, namespaceName, inventoryID, len(resources), len(resources)) + + By(fmt.Sprintf("Verify %d Deployments created", objectCount)) + e2eutil.AssertUnstructuredCount(ctx, c, deploymentObjTemplate, objectCount) + + By("Destroy LOTS of resources") + destroyer := invConfig.DestroyerFactoryFunc() + + startTotal = time.Now() + + var destroyerEvents []event.Event + + for attempt := 1; attempt <= maxAttempts; attempt++ { + start := time.Now() + + destroyerEvents = e2eutil.RunCollect(destroyer.Run(ctx, inventoryInfo, apply.DestroyerOptions{ + InventoryPolicy: inventory.PolicyAdoptIfNoInventory, + DeleteTimeout: reconcileTimeout, + })) + + duration := time.Since(start) + klog.Infof("Destroyer.Run execution time (attempt: %d): %v", attempt, duration) + + e2eutil.ExpectNoEventErrors(destroyerEvents) + + // Retry if ReconcileTimeout + retry := false + for _, e := range applierEvents { + if e.Type == event.WaitType && e.WaitEvent.Status == event.ReconcileTimeout { + retry = true + } + } + if !retry { + break + } + } + + durationTotal = time.Since(startTotal) + klog.Infof("Destroyer.Run total execution time (attempts: %d): %v", maxAttempts, durationTotal) + + e2eutil.ExpectNoReconcileTimeouts(applierEvents) + + By("Verify inventory deleted") + invConfig.InvNotExistsFunc(ctx, c, inventoryName, namespaceName, inventoryID) + + By(fmt.Sprintf("Verify %d Deployments deleted", objectCount)) + e2eutil.AssertUnstructuredCount(ctx, c, deploymentObjTemplate, 0) +} diff --git a/test/stress/thousand_deployments_test.go b/test/stress/thousand_deployments_test.go index bd850c81..9b871a51 100644 --- a/test/stress/thousand_deployments_test.go +++ b/test/stress/thousand_deployments_test.go @@ -13,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/klog/v2" "sigs.k8s.io/cli-utils/pkg/apply" - "sigs.k8s.io/cli-utils/pkg/apply/event" "sigs.k8s.io/cli-utils/pkg/common" "sigs.k8s.io/cli-utils/pkg/inventory" "sigs.k8s.io/cli-utils/test/e2e/e2eutil" @@ -35,11 +34,8 @@ func thousandDeploymentsTest(ctx context.Context, c client.Client, invConfig inv resources := []*unstructured.Unstructured{} - labelKey := "created-for" - labelValue := "stress-test" - deploymentObjTemplate := e2eutil.ManifestToUnstructured([]byte(deploymentYaml)) - deploymentObjTemplate.SetLabels(map[string]string{labelKey: labelValue}) + deploymentObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID}) objectCount := 1000 @@ -79,17 +75,8 @@ func thousandDeploymentsTest(ctx context.Context, c client.Client, invConfig inv duration := time.Since(start) klog.Infof("Applier.Run execution time: %v", duration) - for _, e := range applierEvents { - Expect(e.ErrorEvent.Err).To(BeNil()) - } - for _, e := range applierEvents { - Expect(e.ApplyEvent.Error).To(BeNil(), "ApplyEvent: %v", e.ApplyEvent) - } - for _, e := range applierEvents { - if e.Type == event.WaitType { - Expect(e.WaitEvent.Status).To(BeElementOf(event.ReconcilePending, event.ReconcileSuccessful), "WaitEvent: %v", e.WaitEvent) - } - } + e2eutil.ExpectNoEventErrors(applierEvents) + e2eutil.ExpectNoReconcileTimeouts(applierEvents) By("Verify inventory created") invConfig.InvSizeVerifyFunc(ctx, c, inventoryName, namespaceName, inventoryID, len(resources), len(resources)) @@ -110,17 +97,8 @@ func thousandDeploymentsTest(ctx context.Context, c client.Client, invConfig inv duration = time.Since(start) klog.Infof("Destroyer.Run execution time: %v", duration) - for _, e := range destroyerEvents { - Expect(e.ErrorEvent.Err).To(BeNil()) - } - for _, e := range destroyerEvents { - Expect(e.PruneEvent.Error).To(BeNil(), "PruneEvent: %v", e.PruneEvent) - } - for _, e := range destroyerEvents { - if e.Type == event.WaitType { - Expect(e.WaitEvent.Status).To(BeElementOf(event.ReconcilePending, event.ReconcileSuccessful), "WaitEvent: %v", e.WaitEvent) - } - } + e2eutil.ExpectNoEventErrors(destroyerEvents) + e2eutil.ExpectNoReconcileTimeouts(destroyerEvents) By("Verify inventory deleted") invConfig.InvNotExistsFunc(ctx, c, inventoryName, namespaceName, inventoryID) diff --git a/test/stress/thousand_namespaces_test.go b/test/stress/thousand_namespaces_test.go index 48b75776..207e17af 100644 --- a/test/stress/thousand_namespaces_test.go +++ b/test/stress/thousand_namespaces_test.go @@ -41,17 +41,14 @@ func thousandNamespacesTest(ctx context.Context, c client.Client, invConfig invc resources := []*unstructured.Unstructured{crdObj} - labelKey := "created-for" - labelValue := "stress-test" - namespaceObjTemplate := e2eutil.ManifestToUnstructured([]byte(namespaceYaml)) - namespaceObjTemplate.SetLabels(map[string]string{labelKey: labelValue}) + namespaceObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID}) configMapObjTemplate := e2eutil.ManifestToUnstructured([]byte(configMapYaml)) - configMapObjTemplate.SetLabels(map[string]string{labelKey: labelValue}) + configMapObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID}) cronTabObjTemplate := e2eutil.ManifestToUnstructured([]byte(cronTabYaml)) - cronTabObjTemplate.SetLabels(map[string]string{labelKey: labelValue}) + cronTabObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID}) objectCount := 1000