Skip to content

Commit

Permalink
Merge pull request #587 from karlkfi/karl-stress-retry
Browse files Browse the repository at this point in the history
chore: add stress test that validates retries
  • Loading branch information
k8s-ci-robot authored Jun 7, 2022
2 parents 0cb95ee + 9dcbe66 commit a323326
Show file tree
Hide file tree
Showing 5 changed files with 232 additions and 40 deletions.
72 changes: 65 additions & 7 deletions test/e2e/e2eutil/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
"github.com/onsi/gomega/gstruct"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
Expand All @@ -33,6 +34,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)

const TestIDLabel = "test-id"

func WithReplicas(obj *unstructured.Unstructured, replicas int) *unstructured.Unstructured {
err := unstructured.SetNestedField(obj.Object, int64(replicas), "spec", "replicas")
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expand Down Expand Up @@ -270,9 +273,7 @@ func Run(ch <-chan event.Event) error {
return err
}

func RunWithNoErr(ch <-chan event.Event) {
RunCollectNoErr(ch)
}
var RunWithNoErr = RunCollectNoErr

func RunCollect(ch <-chan event.Event) []event.Event {
var events []event.Event
Expand All @@ -282,14 +283,71 @@ func RunCollect(ch <-chan event.Event) []event.Event {
return events
}

func RunCollectNoErr(ch <-chan event.Event) []event.Event {
events := RunCollect(ch)
for _, e := range events {
gomega.Expect(e.Type).NotTo(gomega.Equal(event.ErrorType))
func RunCollectNoErr(ch <-chan event.Event, callerSkip ...int) []event.Event {
skip := 0
if len(callerSkip) > 0 {
skip = callerSkip[0]
}

events := RunCollect(ch)
ExpectNoEventErrors(events, skip+1)
ExpectNoReconcileTimeouts(events, skip+1)
return events
}

func ExpectNoEventErrors(events []event.Event, callerSkip ...int) {
skip := 0
if len(callerSkip) > 0 {
skip = callerSkip[0]
}

gomega.Expect(events).WithOffset(skip + 1).NotTo(
gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras,
gstruct.Fields{
"Type": gomega.Equal(event.ErrorType),
})))
gomega.Expect(events).WithOffset(skip + 1).NotTo(
gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras,
gstruct.Fields{
"Type": gomega.Equal(event.ApplyType),
"ApplyEvent": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
"Status": gomega.Equal(event.ApplyFailed),
}),
})))
gomega.Expect(events).WithOffset(skip + 1).NotTo(
gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras,
gstruct.Fields{
"Type": gomega.Equal(event.PruneType),
"PruneEvent": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
"Status": gomega.Equal(event.PruneFailed),
}),
})))
gomega.Expect(events).WithOffset(skip + 1).NotTo(
gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras,
gstruct.Fields{
"Type": gomega.Equal(event.DeleteType),
"DeleteEvent": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
"Status": gomega.Equal(event.DeleteFailed),
}),
})))
}

func ExpectNoReconcileTimeouts(events []event.Event, callerSkip ...int) {
skip := 0
if len(callerSkip) > 0 {
skip = callerSkip[0]
}

gomega.Expect(events).WithOffset(skip + 1).NotTo(
gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras,
gstruct.Fields{
"Type": gomega.Equal(event.WaitType),
"WaitEvent": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
"Status": gomega.Equal(event.ReconcileTimeout),
}),
})))
}

func ManifestToUnstructured(manifest []byte) *unstructured.Unstructured {
u := make(map[string]interface{})
err := yaml.Unmarshal(manifest, &u)
Expand Down
4 changes: 4 additions & 0 deletions test/stress/stress_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ var _ = Describe("Stress", func() {
thousandDeploymentsTest(ctx, c, invConfig, inventoryName, namespace.GetName())
})

It("ThousandDeploymentsRetry", func() {
thousandDeploymentsRetryTest(ctx, c, invConfig, inventoryName, namespace.GetName())
})

It("ThousandNamespaces", func() {
thousandNamespacesTest(ctx, c, invConfig, inventoryName, namespace.GetName())
})
Expand Down
155 changes: 155 additions & 0 deletions test/stress/thousand_deployments_retry_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
// Copyright 2022 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package stress

import (
"context"
"fmt"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/klog/v2"
"sigs.k8s.io/cli-utils/pkg/apply"
"sigs.k8s.io/cli-utils/pkg/apply/event"
"sigs.k8s.io/cli-utils/pkg/common"
"sigs.k8s.io/cli-utils/pkg/inventory"
"sigs.k8s.io/cli-utils/test/e2e/e2eutil"
"sigs.k8s.io/cli-utils/test/e2e/invconfig"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// thousandDeploymentsRetryTest tests one pre-existing namespace with 1,000
// Deployments in it. The wait timeout is set too short to confirm
// reconciliation, but the apply/destroy is retried until success.
//
// The Deployments themselves are easy to get status on, but with the retrieval
// of generated resource status (ReplicaSets & Pods), this becomes expensive.
func thousandDeploymentsRetryTest(ctx context.Context, c client.Client, invConfig invconfig.InventoryConfig, inventoryName, namespaceName string) {
By("Apply LOTS of resources")
applier := invConfig.ApplierFactoryFunc()
inventoryID := fmt.Sprintf("%s-%s", inventoryName, namespaceName)

inventoryInfo := invconfig.CreateInventoryInfo(invConfig, inventoryName, namespaceName, inventoryID)

resources := []*unstructured.Unstructured{}

deploymentObjTemplate := e2eutil.ManifestToUnstructured([]byte(deploymentYaml))
deploymentObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID})

objectCount := 1000

for i := 1; i <= objectCount; i++ {
deploymentObj := deploymentObjTemplate.DeepCopy()
deploymentObj.SetNamespace(namespaceName)

// change name & selector labels to avoid overlap between deployments
name := fmt.Sprintf("nginx-%d", i)
deploymentObj.SetName(name)
err := unstructured.SetNestedField(deploymentObj.Object, name, "spec", "selector", "matchLabels", "app")
Expect(err).ToNot(HaveOccurred())
err = unstructured.SetNestedField(deploymentObj.Object, name, "spec", "template", "metadata", "labels", "app")
Expect(err).ToNot(HaveOccurred())

resources = append(resources, deploymentObj)
}

defer func() {
By("Cleanup Deployments")
e2eutil.DeleteAllUnstructuredIfExists(ctx, c, deploymentObjTemplate)
}()

startTotal := time.Now()

var applierEvents []event.Event

maxAttempts := 15
reconcileTimeout := 2 * time.Minute

for attempt := 1; attempt <= maxAttempts; attempt++ {
start := time.Now()

applierEvents = e2eutil.RunCollect(applier.Run(ctx, inventoryInfo, resources, apply.ApplierOptions{
// SSA reduces GET+PATCH to just PATCH, which is faster
ServerSideOptions: common.ServerSideOptions{
ServerSideApply: true,
ForceConflicts: true,
FieldManager: "cli-utils.kubernetes.io",
},
ReconcileTimeout: reconcileTimeout,
EmitStatusEvents: false,
}))

duration := time.Since(start)
klog.Infof("Applier.Run execution time (attempt: %d): %v", attempt, duration)

e2eutil.ExpectNoEventErrors(applierEvents)

// Retry if ReconcileTimeout
retry := false
for _, e := range applierEvents {
if e.Type == event.WaitType && e.WaitEvent.Status == event.ReconcileTimeout {
retry = true
}
}
if !retry {
break
}
}

durationTotal := time.Since(startTotal)
klog.Infof("Applier.Run total execution time (attempts: %d): %v", maxAttempts, durationTotal)

e2eutil.ExpectNoReconcileTimeouts(applierEvents)

By("Verify inventory created")
invConfig.InvSizeVerifyFunc(ctx, c, inventoryName, namespaceName, inventoryID, len(resources), len(resources))

By(fmt.Sprintf("Verify %d Deployments created", objectCount))
e2eutil.AssertUnstructuredCount(ctx, c, deploymentObjTemplate, objectCount)

By("Destroy LOTS of resources")
destroyer := invConfig.DestroyerFactoryFunc()

startTotal = time.Now()

var destroyerEvents []event.Event

for attempt := 1; attempt <= maxAttempts; attempt++ {
start := time.Now()

destroyerEvents = e2eutil.RunCollect(destroyer.Run(ctx, inventoryInfo, apply.DestroyerOptions{
InventoryPolicy: inventory.PolicyAdoptIfNoInventory,
DeleteTimeout: reconcileTimeout,
}))

duration := time.Since(start)
klog.Infof("Destroyer.Run execution time (attempt: %d): %v", attempt, duration)

e2eutil.ExpectNoEventErrors(destroyerEvents)

// Retry if ReconcileTimeout
retry := false
for _, e := range applierEvents {
if e.Type == event.WaitType && e.WaitEvent.Status == event.ReconcileTimeout {
retry = true
}
}
if !retry {
break
}
}

durationTotal = time.Since(startTotal)
klog.Infof("Destroyer.Run total execution time (attempts: %d): %v", maxAttempts, durationTotal)

e2eutil.ExpectNoReconcileTimeouts(applierEvents)

By("Verify inventory deleted")
invConfig.InvNotExistsFunc(ctx, c, inventoryName, namespaceName, inventoryID)

By(fmt.Sprintf("Verify %d Deployments deleted", objectCount))
e2eutil.AssertUnstructuredCount(ctx, c, deploymentObjTemplate, 0)
}
32 changes: 5 additions & 27 deletions test/stress/thousand_deployments_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/klog/v2"
"sigs.k8s.io/cli-utils/pkg/apply"
"sigs.k8s.io/cli-utils/pkg/apply/event"
"sigs.k8s.io/cli-utils/pkg/common"
"sigs.k8s.io/cli-utils/pkg/inventory"
"sigs.k8s.io/cli-utils/test/e2e/e2eutil"
Expand All @@ -35,11 +34,8 @@ func thousandDeploymentsTest(ctx context.Context, c client.Client, invConfig inv

resources := []*unstructured.Unstructured{}

labelKey := "created-for"
labelValue := "stress-test"

deploymentObjTemplate := e2eutil.ManifestToUnstructured([]byte(deploymentYaml))
deploymentObjTemplate.SetLabels(map[string]string{labelKey: labelValue})
deploymentObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID})

objectCount := 1000

Expand Down Expand Up @@ -79,17 +75,8 @@ func thousandDeploymentsTest(ctx context.Context, c client.Client, invConfig inv
duration := time.Since(start)
klog.Infof("Applier.Run execution time: %v", duration)

for _, e := range applierEvents {
Expect(e.ErrorEvent.Err).To(BeNil())
}
for _, e := range applierEvents {
Expect(e.ApplyEvent.Error).To(BeNil(), "ApplyEvent: %v", e.ApplyEvent)
}
for _, e := range applierEvents {
if e.Type == event.WaitType {
Expect(e.WaitEvent.Status).To(BeElementOf(event.ReconcilePending, event.ReconcileSuccessful), "WaitEvent: %v", e.WaitEvent)
}
}
e2eutil.ExpectNoEventErrors(applierEvents)
e2eutil.ExpectNoReconcileTimeouts(applierEvents)

By("Verify inventory created")
invConfig.InvSizeVerifyFunc(ctx, c, inventoryName, namespaceName, inventoryID, len(resources), len(resources))
Expand All @@ -110,17 +97,8 @@ func thousandDeploymentsTest(ctx context.Context, c client.Client, invConfig inv
duration = time.Since(start)
klog.Infof("Destroyer.Run execution time: %v", duration)

for _, e := range destroyerEvents {
Expect(e.ErrorEvent.Err).To(BeNil())
}
for _, e := range destroyerEvents {
Expect(e.PruneEvent.Error).To(BeNil(), "PruneEvent: %v", e.PruneEvent)
}
for _, e := range destroyerEvents {
if e.Type == event.WaitType {
Expect(e.WaitEvent.Status).To(BeElementOf(event.ReconcilePending, event.ReconcileSuccessful), "WaitEvent: %v", e.WaitEvent)
}
}
e2eutil.ExpectNoEventErrors(destroyerEvents)
e2eutil.ExpectNoReconcileTimeouts(destroyerEvents)

By("Verify inventory deleted")
invConfig.InvNotExistsFunc(ctx, c, inventoryName, namespaceName, inventoryID)
Expand Down
9 changes: 3 additions & 6 deletions test/stress/thousand_namespaces_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,14 @@ func thousandNamespacesTest(ctx context.Context, c client.Client, invConfig invc

resources := []*unstructured.Unstructured{crdObj}

labelKey := "created-for"
labelValue := "stress-test"

namespaceObjTemplate := e2eutil.ManifestToUnstructured([]byte(namespaceYaml))
namespaceObjTemplate.SetLabels(map[string]string{labelKey: labelValue})
namespaceObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID})

configMapObjTemplate := e2eutil.ManifestToUnstructured([]byte(configMapYaml))
configMapObjTemplate.SetLabels(map[string]string{labelKey: labelValue})
configMapObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID})

cronTabObjTemplate := e2eutil.ManifestToUnstructured([]byte(cronTabYaml))
cronTabObjTemplate.SetLabels(map[string]string{labelKey: labelValue})
cronTabObjTemplate.SetLabels(map[string]string{e2eutil.TestIDLabel: inventoryID})

objectCount := 1000

Expand Down

0 comments on commit a323326

Please sign in to comment.