Retrieve every k8s resource generated by a Carvel package installation (alt)

chart/kubeapps/README.md

@@ -497,8 +497,8 @@ Once you have installed Kubeapps follow the [Getting Started Guide](https://gith
 | `kubeappsapis.image.pullPolicy`                                            | Kubeapps-APIs image pull policy                                                           | `IfNotPresent`           |
 | `kubeappsapis.image.pullSecrets`                                           | Kubeapps-APIs image pull secrets                                                          | `[]`                     |
 | `kubeappsapis.replicaCount`                                                | Number of frontend replicas to deploy                                                     | `2`                      |
-| `kubeappsapis.qps`                                                         | KubeappsAPIs Kubernetes API client QPS limit                                              | `10.0`                   |
-| `kubeappsapis.burst`                                                       | KubeappsAPIs Kubernetes API client Burst limit                                            | `15`                     |
+| `kubeappsapis.qps`                                                         | KubeappsAPIs Kubernetes API client QPS limit                                              | `50.0`                   |
+| `kubeappsapis.burst`                                                       | KubeappsAPIs Kubernetes API client Burst limit                                            | `100`                    |
 | `kubeappsapis.terminationGracePeriodSeconds`                               | The grace time period for sig term                                                        | `300`                    |
 | `kubeappsapis.extraEnvVars`                                                | Array with extra environment variables to add to the KubeappsAPIs container               | `[]`                     |
 | `kubeappsapis.extraEnvVarsCM`                                              | Name of existing ConfigMap containing extra env vars for the KubeappsAPIs container       | `""`                     |

chart/kubeapps/values.yaml

@@ -1662,10 +1662,10 @@ kubeappsapis:
   replicaCount: 2
   ## @param kubeappsapis.qps KubeappsAPIs Kubernetes API client QPS limit
   ##
-  qps: "10.0"
+  qps: "50.0"
   ## @param kubeappsapis.burst KubeappsAPIs Kubernetes API client Burst limit
   ##
-  burst: "15"
+  burst: "100"
   ## @param kubeappsapis.terminationGracePeriodSeconds The grace time period for sig term
   ## ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution
   ##

cmd/kubeapps-apis/plugins/kapp_controller/packages/v1alpha1/server.go

@@ -16,6 +16,13 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/cppforlife/go-cli-ui/ui"
+	ctlapp "github.com/k14s/kapp/pkg/kapp/app"
+	kappcmdapp "github.com/k14s/kapp/pkg/kapp/cmd/app"
+	kappcmdcore "github.com/k14s/kapp/pkg/kapp/cmd/core"
+	kappcmdtools "github.com/k14s/kapp/pkg/kapp/cmd/tools"
+	"github.com/k14s/kapp/pkg/kapp/logger"
+	ctlres "github.com/k14s/kapp/pkg/kapp/resources"
 	"github.com/kubeapps/kubeapps/cmd/kubeapps-apis/core"
 	corev1 "github.com/kubeapps/kubeapps/cmd/kubeapps-apis/gen/core/packages/v1alpha1"
 	"github.com/kubeapps/kubeapps/cmd/kubeapps-apis/gen/plugins/kapp_controller/packages/v1alpha1"
@@ -26,6 +33,7 @@ import (
 )
 
 type clientGetter func(context.Context, string) (kubernetes.Interface, dynamic.Interface, error)
+type kappClientsGetter func(ctx context.Context, cluster, namespace string) (ctlapp.Apps, ctlres.IdentifiedResources, *kappcmdapp.FailingAPIServicesPolicy, ctlres.ResourceFilter, error)
 
 const (
 	globalPackagingNamespace = "kapp-controller-packaging-global"
@@ -43,6 +51,7 @@ type Server struct {
 	clientGetter             clientGetter
 	globalPackagingNamespace string
 	globalPackagingCluster   string
+	kappClientsGetter        kappClientsGetter
 }
 
 // NewServer returns a Server automatically configured with a function to obtain
@@ -69,6 +78,50 @@ func NewServer(configGetter core.KubernetesConfigGetter, globalPackagingCluster
 		},
 		globalPackagingNamespace: globalPackagingNamespace,
 		globalPackagingCluster:   globalPackagingCluster,
+		kappClientsGetter: func(ctx context.Context, cluster, namespace string) (ctlapp.Apps, ctlres.IdentifiedResources, *kappcmdapp.FailingAPIServicesPolicy, ctlres.ResourceFilter, error) {
+			if configGetter == nil {
+				return ctlapp.Apps{}, ctlres.IdentifiedResources{}, nil, ctlres.ResourceFilter{}, status.Errorf(codes.Internal, "configGetter arg required")
+			}
+			// Retrieve the k8s REST client from the configGetter
+			config, err := configGetter(ctx, cluster)
+			if err != nil {
+				return ctlapp.Apps{}, ctlres.IdentifiedResources{}, nil, ctlres.ResourceFilter{}, status.Errorf(codes.FailedPrecondition, "unable to get config due to: %v", err)
+			}
+			// Pass the REST client to the (custom) kapp factory
+			configFactory := NewConfigurableConfigFactoryImpl()
+			configFactory.ConfigureRESTConfig(config)
+			depsFactory := kappcmdcore.NewDepsFactoryImpl(configFactory, ui.NewNoopUI())
+
+			// Create an empty resource filter
+			resourceFilterFlags := kappcmdtools.ResourceFilterFlags{}
+			resourceFilter, err := resourceFilterFlags.ResourceFilter()
+			if err != nil {
+				return ctlapp.Apps{}, ctlres.IdentifiedResources{}, nil, ctlres.ResourceFilter{}, status.Errorf(codes.FailedPrecondition, "unable to get config due to: %v", err)
+			}
+
+			// Create the preconfigured resource types flags and a failing policy
+			resourceTypesFlags := kappcmdapp.ResourceTypesFlags{
+				// Allow to ignore failing APIServices
+				IgnoreFailingAPIServices: true,
+				// Scope resource searching to fallback allowed namespaces
+				ScopeToFallbackAllowedNamespaces: true,
+			}
+			failingAPIServicesPolicy := resourceTypesFlags.FailingAPIServicePolicy()
+
+			// Getting namespaced clients (e.g., for fetching an App)
+			supportingNsObjs, err := kappcmdapp.FactoryClients(depsFactory, kappcmdcore.NamespaceFlags{Name: namespace}, resourceTypesFlags, logger.NewNoopLogger())
+			if err != nil {
+				return ctlapp.Apps{}, ctlres.IdentifiedResources{}, nil, ctlres.ResourceFilter{}, status.Errorf(codes.FailedPrecondition, "unable to get config due to: %v", err)
+			}
+
+			// Getting non-namespaced clients (e.g., for fetching every k8s object in the cluster)
+			supportingObjs, err := kappcmdapp.FactoryClients(depsFactory, kappcmdcore.NamespaceFlags{Name: ""}, resourceTypesFlags, logger.NewNoopLogger())
+			if err != nil {
+				return ctlapp.Apps{}, ctlres.IdentifiedResources{}, nil, ctlres.ResourceFilter{}, status.Errorf(codes.FailedPrecondition, "unable to get config due to: %v", err)
+			}
+
+			return supportingNsObjs.Apps, supportingObjs.IdentifiedResources, failingAPIServicesPolicy, resourceFilter, nil
+		},
 	}
 }
 
@@ -83,3 +136,15 @@ func (s *Server) GetClients(ctx context.Context, cluster string) (kubernetes.Int
 	}
 	return typedClient, dynamicClient, nil
 }
+
+// GetKappClients ensures a client getter is available and uses it to return a Kapp Factory.
+func (s *Server) GetKappClients(ctx context.Context, cluster, namespace string) (ctlapp.Apps, ctlres.IdentifiedResources, *kappcmdapp.FailingAPIServicesPolicy, ctlres.ResourceFilter, error) {
+	if s.clientGetter == nil {
+		return ctlapp.Apps{}, ctlres.IdentifiedResources{}, nil, ctlres.ResourceFilter{}, status.Errorf(codes.Internal, "server not configured with configGetter")
+	}
+	appsClient, resourcesClient, failingAPIServicesPolicy, resourceFilter, err := s.kappClientsGetter(ctx, cluster, namespace)
+	if err != nil {
+		return ctlapp.Apps{}, ctlres.IdentifiedResources{}, nil, ctlres.ResourceFilter{}, status.Errorf(codes.FailedPrecondition, fmt.Sprintf("unable to get Kapp Factory : %v", err))
+	}
+	return appsClient, resourcesClient, failingAPIServicesPolicy, resourceFilter, nil
+}

cmd/kubeapps-apis/plugins/kapp_controller/packages/v1alpha1/server_ctrl_packages.go

@@ -735,90 +735,10 @@ func (s *Server) GetInstalledPackageResourceRefs(ctx context.Context, request *c
 		cluster = s.globalPackagingCluster
 	}
 
-	typedClient, _, err := s.GetClients(ctx, cluster)
+	// get the list of every k8s resource matching ResourceRef
+	refs, err := s.inspectKappK8sResources(ctx, cluster, namespace, installedPackageRefId)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "unable to get the k8s client: '%v'", err)
-	}
-
-	refs := []*corev1.ResourceRef{}
-
-	// TODO(agamez): apparently, App CRs not being created by a "kapp deploy"
-	// don't have the proper annotations. So, in order to retrieve the annotation value,
-	// we have to get the ConfigMap <AppName>-ctrl and, then, fetch the
-	// vaulue of the key "labelValue" in data.spec.
-	// See https://kubernetes.slack.com/archives/CH8KCCKA5/p1637842398026700
-	// https://github.com/vmware-tanzu/carvel-kapp-controller/issues/430
-
-	// the ConfigMap name is, by convention, "<appname>-ctrl", but it will change in the near future
-	cmName := fmt.Sprintf("%s-ctrl", installedPackageRefId)
-	cm, err := typedClient.CoreV1().ConfigMaps(namespace).Get(ctx, cmName, metav1.GetOptions{})
-	if err == nil && cm.Data["spec"] != "" {
-
-		appLabelValue := extractValue(cm.Data["spec"], "labelValue")
-		appLabelSelector := fmt.Sprintf("%s=%s", appLabelKey, appLabelValue)
-		listOptions := metav1.ListOptions{LabelSelector: appLabelSelector}
-
-		// TODO(agamez): perform an actual query over all the resources available in the cluster
-		// this is currently just a PoC getting the bare minimum: pods, deployments, services and secrets.
-		// Also, the xxx.Items[i] are not populating the Kind and APIVersion fields. Check why.
-
-		// Fetching all the matching pods
-		pods, err := typedClient.CoreV1().Pods(namespace).List(ctx, listOptions)
-		if err != nil {
-			return nil, statuserror.FromK8sError("get", "Pods", "", err)
-		}
-		for _, resource := range pods.Items {
-			refs = append(refs, &corev1.ResourceRef{
-				ApiVersion: "core/v1",
-				Kind:       "Pod",
-				Name:       resource.Name,
-				Namespace:  resource.Namespace,
-			})
-		}
-
-		// Fetching all the matching deployments
-		deployments, err := typedClient.AppsV1().Deployments(namespace).List(ctx, listOptions)
-		if err != nil {
-			return nil, statuserror.FromK8sError("get", "Deployments", "", err)
-		}
-		for _, resource := range deployments.Items {
-			refs = append(refs, &corev1.ResourceRef{
-				ApiVersion: "apps/v1",
-				Kind:       "Deployment",
-				Name:       resource.ObjectMeta.Name,
-				Namespace:  resource.ObjectMeta.Namespace,
-			})
-		}
-
-		// Fetching all the matching services
-		services, err := typedClient.CoreV1().Services(namespace).List(ctx, listOptions)
-		if err != nil {
-			return nil, statuserror.FromK8sError("get", "services", "", err)
-		}
-		for _, resource := range services.Items {
-			refs = append(refs, &corev1.ResourceRef{
-				ApiVersion: "core/v1",
-				Kind:       "Service",
-				Name:       resource.ObjectMeta.Name,
-				Namespace:  resource.ObjectMeta.Namespace,
-			})
-		}
-
-		// Fetching all the matching secrets
-		secrets, err := typedClient.CoreV1().Secrets(namespace).List(ctx, listOptions)
-		if err != nil {
-			return nil, statuserror.FromK8sError("get", "Secrets", "", err)
-		}
-		for _, resource := range secrets.Items {
-			refs = append(refs, &corev1.ResourceRef{
-				ApiVersion: "core/v1",
-				Kind:       "Secret",
-				Name:       resource.ObjectMeta.Name,
-				Namespace:  resource.ObjectMeta.Namespace,
-			})
-		}
-	} else {
-		log.Warning(statuserror.FromK8sError("get", "ConfigMap", cmName, err))
+		return nil, err
 	}
 
 	return &corev1.GetInstalledPackageResourceRefsResponse{

cmd/kubeapps-apis/plugins/kapp_controller/packages/v1alpha1/server_data_adapters.go

@@ -249,8 +249,6 @@ func (s *Server) buildInstalledPackageDetail(pkgInstall *packagingv1alpha1.Packa
 	deployStderr := ""
 	fetchStdout := ""
 	fetchStderr := ""
-	inspectStdout := ""
-	inspectStderr := ""
 
 	if app.Status.Deploy != nil {
 		deployStdout = app.Status.Deploy.Stdout
@@ -260,10 +258,6 @@ func (s *Server) buildInstalledPackageDetail(pkgInstall *packagingv1alpha1.Packa
 		fetchStdout = app.Status.Fetch.Stdout
 		fetchStderr = app.Status.Fetch.Stderr
 	}
-	if app.Status.Inspect != nil {
-		inspectStdout = app.Status.Inspect.Stdout
-		inspectStderr = app.Status.Inspect.Stderr
-	}
 
 	// Build some custom installation notes based on the available stdout + stderr
 	// TODO(agamez): this is just a temporary solution until come up with a better UX solution
@@ -279,10 +273,6 @@ func (s *Server) buildInstalledPackageDetail(pkgInstall *packagingv1alpha1.Packa
 %s
 
 
-### Inspect:
-%s
-
-
 ## Errors
 
 
@@ -294,10 +284,7 @@ func (s *Server) buildInstalledPackageDetail(pkgInstall *packagingv1alpha1.Packa
 %s
 
 
-### Inspect:
-%s
-
-`, deployStdout, fetchStdout, inspectStdout, deployStderr, fetchStderr, inspectStderr)
+`, deployStdout, fetchStdout, deployStderr, fetchStderr)
 
 	if len(pkgInstall.Status.Conditions) > 1 {
 		log.Warningf("The package install %s has more than one status conditions. Using the first one: %s", pkgInstall.Name, pkgInstall.Status.Conditions[0])

cmd/kubeapps-apis/plugins/kapp_controller/packages/v1alpha1/server_data_resources.go

@@ -14,7 +14,9 @@ package main
 
 import (
 	"context"
+	"fmt"
 
+	corev1 "github.com/kubeapps/kubeapps/cmd/kubeapps-apis/gen/core/packages/v1alpha1"
 	kappctrlv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	packagingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1"
 	datapackagingv1alpha1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1"
@@ -399,3 +401,55 @@ func (s *Server) updatePkgInstall(ctx context.Context, cluster, namespace string
 	}
 	return &pkgInstall, nil
 }
+
+// inspectKappK8sResources returns the list of k8s resources matching the given listOptions
+func (s *Server) inspectKappK8sResources(ctx context.Context, cluster, namespace, packageId string) ([]*corev1.ResourceRef, error) {
+	// As per https://github.com/vmware-tanzu/carvel-kapp-controller/blob/v0.31.0/pkg/deploy/kapp.go#L151
+	appName := fmt.Sprintf("%s-ctrl", packageId)
+
+	refs := []*corev1.ResourceRef{}
+
+	// Get the Kapp different clients
+	appsClient, resourcesClient, failingAPIServicesPolicy, _, err := s.GetKappClients(ctx, cluster, namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	// Fetch the Kapp App
+	app, err := appsClient.Find(appName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Fetch the GroupVersions used by the app
+	usedGVs, err := app.UsedGVs()
+	if err != nil {
+		return nil, err
+	}
+
+	// Mark those GVs as required
+	failingAPIServicesPolicy.MarkRequiredGVs(usedGVs)
+
+	// Create a k8s label selector for the app
+	labelSelector, err := app.LabelSelector()
+	if err != nil {
+		return nil, err
+	}
+
+	// List the k8s resources that match the label selector
+	resources, err := resourcesClient.List(labelSelector, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// For each resource, generate and append the ResourceRef
+	for _, resource := range resources {
+		refs = append(refs, &corev1.ResourceRef{
+			ApiVersion: resource.GroupVersion().String(),
+			Kind:       resource.Kind(),
+			Name:       resource.Name(),
+			Namespace:  resource.Namespace(),
+		})
+	}
+	return refs, nil
+}