wait on startup to get permissions

The glbc component needs to get a CRD and have permission to read it,
this is not inmidiate when bootstrapping a cluster so, instead of
crashing and depending on a external component to restart the pod,
we can active poll on this conditions before crashing.

This has the benefit of reducing the noise on the logs with panics,
that may be misinterpreted by external tools processing the logs.
It also improves the bootstrap latency.

cmd/glbc/main.go

@@ -25,6 +25,7 @@ import (
 
 	flag "github.com/spf13/pflag"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/ingress-gce/pkg/frontendconfig"
 	"k8s.io/ingress-gce/pkg/ingparams"
 	"k8s.io/ingress-gce/pkg/instancegroups"
@@ -35,6 +36,7 @@ import (
 	"k8s.io/klog/v2"
 
 	crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/kubernetes"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
@@ -111,7 +113,20 @@ func main() {
 	// TODO(rramkumar): Reuse this CRD handler for other CRD's coming.
 	crdHandler := crd.NewCRDHandler(crdClient)
 	backendConfigCRDMeta := backendconfig.CRDMeta()
-	if _, err := crdHandler.EnsureCRD(backendConfigCRDMeta, true); err != nil {
+	// The CRD and/or its permission may take a while, wait for 1 minute to avoid
+	// crashing so we can reduce the noise on the logs.
+	if err := wait.PollImmediate(3*time.Second, 1*time.Minute, func() (bool, error) {
+		_, err := crdHandler.EnsureCRD(backendConfigCRDMeta, true)
+		if err == nil {
+			return true, nil
+		}
+		// retry if kube-apiserver returns a not found or forbidden error
+		if apierrors.IsNotFound(err) || apierrors.IsForbidden(err) {
+			return false, nil
+		}
+		// crash on any other error
+		return false, err
+	}); err != nil {
 		klog.Fatalf("Failed to ensure BackendConfig CRD: %v", err)
 	}