throttle existing-non-labeled resources finding (#336)

Co-authored-by: Dmitriy Kalinin <dkalinin@vmware.com>
carvel-dev · Oct 6, 2021 · 3dd4181 · 3dd4181
1 parent 200fb90
commit 3dd4181
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 7 deletions.
diff --git a/pkg/kapp/cmd/app/deploy.go b/pkg/kapp/cmd/app/deploy.go
@@ -286,7 +286,10 @@ func (o *DeployOptions) existingResources(newResources []ctlres.Resource,
 	}
 
 	matchingOpts := ctlres.AllAndMatchingOpts{
-		SkipResourceOwnershipCheck: o.DeployFlags.OverrideOwnershipOfExistingResources,
+		ExistingNonLabeledResourcesCheck:            o.DeployFlags.ExistingNonLabeledResourcesCheck,
+		ExistingNonLabeledResourcesCheckConcurrency: o.DeployFlags.ExistingNonLabeledResourcesCheckConcurrency,
+		SkipResourceOwnershipCheck:                  o.DeployFlags.OverrideOwnershipOfExistingResources,
+
 		// Prevent accidently overriding kapp state records
 		DisallowedResourcesByLabelKeys: []string{ctlapp.KappIsAppLabelKey},
 		LabelErrorResolutionFunc:       labelErrorResolutionFunc,

diff --git a/pkg/kapp/cmd/app/deploy_flags.go b/pkg/kapp/cmd/app/deploy_flags.go
@@ -15,7 +15,9 @@ type DeployFlags struct {
 	Patch      bool
 	AllowEmpty bool
 
-	OverrideOwnershipOfExistingResources bool
+	ExistingNonLabeledResourcesCheck            bool
+	ExistingNonLabeledResourcesCheckConcurrency int
+	OverrideOwnershipOfExistingResources        bool
 
 	AppChangesMaxToKeep int
 
@@ -35,6 +37,10 @@ func (s *DeployFlags) Set(cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(&s.Patch, "patch", "p", false, "Add or update provided resources")
 	cmd.Flags().BoolVar(&s.AllowEmpty, "dangerous-allow-empty-list-of-resources", false, "Allow to apply empty set of resources (same as running kapp delete)")
 
+	cmd.Flags().BoolVar(&s.ExistingNonLabeledResourcesCheck, "existing-non-labeled-resources-check",
+		true, "Find and consider existing non-labeled resources in diff")
+	cmd.Flags().IntVar(&s.ExistingNonLabeledResourcesCheckConcurrency, "existing-non-labeled-resources-check-concurrency",
+		100, "Concurrency to check for existing non-labeled resources")
 	cmd.Flags().BoolVar(&s.OverrideOwnershipOfExistingResources, "dangerous-override-ownership-of-existing-resources",
 		false, "Steal existing resources from another app")
 

diff --git a/pkg/kapp/resources/labeled_resources.go b/pkg/kapp/resources/labeled_resources.go
@@ -9,6 +9,7 @@ import (
 	"sync"
 
 	"github.com/k14s/kapp/pkg/kapp/logger"
+	"github.com/k14s/kapp/pkg/kapp/util"
 	"k8s.io/apimachinery/pkg/labels"
 )
 
@@ -84,7 +85,10 @@ func (a *LabeledResources) All() ([]Resource, error) {
 }
 
 type AllAndMatchingOpts struct {
-	SkipResourceOwnershipCheck     bool
+	ExistingNonLabeledResourcesCheck            bool
+	ExistingNonLabeledResourcesCheckConcurrency int
+	SkipResourceOwnershipCheck                  bool
+
 	DisallowedResourcesByLabelKeys []string
 	LabelErrorResolutionFunc       func(string, string) string
 }
@@ -101,9 +105,15 @@ func (a *LabeledResources) AllAndMatching(newResources []Resource, opts AllAndMa
 		return nil, err
 	}
 
-	nonLabeledResources, err := a.findNonLabeledResources(resources, newResources)
-	if err != nil {
-		return nil, err
+	var nonLabeledResources []Resource
+
+	if opts.ExistingNonLabeledResourcesCheck {
+		var err error
+		nonLabeledResources, err = a.findNonLabeledResources(
+			resources, newResources, opts.ExistingNonLabeledResourcesCheckConcurrency)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if !opts.SkipResourceOwnershipCheck && len(nonLabeledResources) > 0 {
@@ -182,7 +192,7 @@ func (a *LabeledResources) checkDisallowedLabels(resources []Resource, disallowe
 	return nil
 }
 
-func (a *LabeledResources) findNonLabeledResources(labeledResources, newResources []Resource) ([]Resource, error) {
+func (a *LabeledResources) findNonLabeledResources(labeledResources, newResources []Resource, concurrency int) ([]Resource, error) {
 	defer a.logger.DebugFunc("findNonLabeledResources").Finish()
 
 	var foundResources []Resource
@@ -193,6 +203,8 @@ func (a *LabeledResources) findNonLabeledResources(labeledResources, newResource
 	}
 
 	var wg sync.WaitGroup
+	throttle := util.NewThrottle(concurrency)
+
 	errCh := make(chan error, len(newResources))
 	resCh := make(chan Resource, len(newResources))
 
@@ -202,6 +214,9 @@ func (a *LabeledResources) findNonLabeledResources(labeledResources, newResource
 		if _, found := rsMap[NewUniqueResourceKey(res).String()]; !found {
 			wg.Add(1)
 			go func() {
+				throttle.Take()
+				defer throttle.Done()
+
 				defer func() { wg.Done() }()
 
 				exists, err := a.identifiedResources.Exists(res, ExistsOpts{})