disable concurrent reconcile to avoid OOM issue in the 3K managed clu…

…ster scale test

Signed-off-by: Xiangjing Li <xiangli@redhat.com>

build/e2e-kc.sh

@@ -304,17 +304,22 @@ echo "STARTING test 11-helm-hub-dryrun"
 kubectl config use-context kind-hub
 kubectl apply -f test/e2e/cases/11-helm-hub-dryrun/
 sleep 30
-if kubectl get subscriptions.apps.open-cluster-management.io ingress-appsub | grep Propagated; then
+if kubectl get subscriptions.apps.open-cluster-management.io -n default ingress-appsub | grep Propagated; then
     echo "11-helm-hub-dryrun: ingress-appsub status is Propagated"
 else
     echo "11-helm-hub-dryruns FAILED:  ingress-appsub status is not Propagated"
     exit 1
 fi
+sleep 30
 kubectl config use-context kind-cluster1
-if kubectl get subscriptionstatus.apps.open-cluster-management.io ingress-appsub -o yaml | grep InstallError; then
+if kubectl get subscriptionstatus.apps.open-cluster-management.io -n default ingress-appsub -o yaml | grep InstallError; then
     echo "11-helm-hub-dryrun: found InstallError in subscription status output"
 else
     echo "11-helm-hub-dryrun: FAILED: InstallError is not in the subscription status output"
+    kubectl get appsub -n default ingress-appsub -o yaml
+    kubectl get pods -n open-cluster-management-agent-addon  -l component=application-manager
+    kubectl describe pods -n open-cluster-management-agent-addon  -l component=application-manager
+    kubectl logs -n open-cluster-management-agent-addon  -l component=application-manager
     exit 1
 fi
 kubectl config use-context kind-hub
@@ -541,8 +546,9 @@ else
 fi
 
 echo "19-verify-git-pull-time-metric: patching successful subscription and expeting failed metrics"
-kubectl -n git-pull-time-metric-test annotate subscriptions git-pull-time-metric-sub apps.open-cluster-management.io/git-branch=main1 --overwrite
-sleep 120
+kubectl -n git-pull-time-metric-test annotate appsub git-pull-time-metric-sub apps.open-cluster-management.io/git-branch=main1 --overwrite
+# With high reconcile rate, the appsub is handled every 2 minutes. wait for over 2 minutes to make sure the appsub is handled.
+sleep 150
 
 echo "19-verify-git-pull-time-metric: fetching failed managed cluster metrics"
 collectedFailedMcMetrics=`kubectl exec -n open-cluster-management-agent-addon deploy/application-manager -- curl http://localhost:8388/metrics`

pkg/helmrelease/controller/helmrelease/helmrelease_controller.go

@@ -17,7 +17,7 @@ limitations under the License.
 // install/upgrade/uninstall code ported and modified from:
 // github.com/operator-framework/operator-sdk/internal/helm/controller/reconcile.go
 
-//Package helmrelease controller manages the helmrelease CR
+// Package helmrelease controller manages the helmrelease CR
 package helmrelease
 
 import (

pkg/placementrule/controller/placementrule/placementrule_controller.go

@@ -59,7 +59,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 	// Enable the concurrent reconcile in case some placementrule could take longer to generate cluster decisions
 	c, err := controller.New("placementrule-controller", mgr, controller.Options{
 		Reconciler:              r,
-		MaxConcurrentReconciles: 2,
+		MaxConcurrentReconciles: 1,
 	})
 	if err != nil {
 		return err

pkg/placementrule/controller/placementrule/placementrule_status_controller.go

@@ -65,7 +65,7 @@ func genReconciler(mgr manager.Manager) reconcile.Reconciler {
 func addRec(mgr manager.Manager, r reconcile.Reconciler) error {
 	c, err := controller.New("placementrule-status-controller", mgr, controller.Options{
 		Reconciler:              r,
-		MaxConcurrentReconciles: 2,
+		MaxConcurrentReconciles: 1,
 	})
 	if err != nil {
 		return err

test/e2e/cases/11-helm-hub-dryrun/02-placement.yaml

@@ -2,5 +2,6 @@ apiVersion: apps.open-cluster-management.io/v1
 kind: PlacementRule
 metadata:
   name: nginx-pr
+  namespace: default
 spec:
   clusterReplicas: 10

test/e2e/cases/11-helm-hub-dryrun/02-subscription.yaml

@@ -2,6 +2,7 @@ apiVersion: apps.open-cluster-management.io/v1
 kind: Subscription
 metadata:
   name: ingress-appsub
+  namespace: default
 spec:
   channel: default/dev-helmrepo
   name: ingress