From 097a477836fed77cea18f4e0e4711a3ec7890b42 Mon Sep 17 00:00:00 2001 From: Johnnie Chou Date: Sat, 20 Aug 2022 22:27:15 +0000 Subject: [PATCH] add filtering and listing functionalities --- cmd/main.go | 2 +- cmd/status/cmdstatus.go | 207 ++++++++++++++++++---- cmd/status/cmdstatus_test.go | 13 +- pkg/inventory/fake-inventory-client.go | 6 + pkg/inventory/inventory-client-factory.go | 6 +- pkg/inventory/inventory-client.go | 37 ++++ pkg/inventory/inventory-client_test.go | 16 +- pkg/inventory/inventorycm.go | 7 + test/e2e/customprovider/provider.go | 2 +- 9 files changed, 251 insertions(+), 45 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 88752602..dbb985aa 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -71,7 +71,7 @@ func main() { destroy.Command(f, invFactory, loader, ioStreams), diff.NewCommand(f, ioStreams), preview.Command(f, invFactory, loader, ioStreams), - status.Command(f, invFactory, loader), + status.Command(context.TODO(), f, invFactory, status.NewInventoryLoader(loader)), } for _, subCmd := range subCmds { subCmd.PreRunE = preRunE diff --git a/cmd/status/cmdstatus.go b/cmd/status/cmdstatus.go index a485ad1c..fea9b760 100644 --- a/cmd/status/cmdstatus.go +++ b/cmd/status/cmdstatus.go @@ -6,11 +6,13 @@ package status import ( "context" "fmt" + ss "strings" "time" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/slice" "sigs.k8s.io/cli-utils/cmd/flagutils" "sigs.k8s.io/cli-utils/cmd/status/printers" "sigs.k8s.io/cli-utils/cmd/status/printers/printer" @@ -24,18 +26,39 @@ import ( "sigs.k8s.io/cli-utils/pkg/kstatus/status" "sigs.k8s.io/cli-utils/pkg/manifestreader" "sigs.k8s.io/cli-utils/pkg/object" + printcommon "sigs.k8s.io/cli-utils/pkg/print/common" + pkgprinters "sigs.k8s.io/cli-utils/pkg/printers" ) -func GetRunner(factory cmdutil.Factory, invFactory inventory.ClientFactory, loader manifestreader.ManifestLoader) *Runner { +const ( + Known = "known" + Current = "current" + Deleted = "deleted" + Forever = "forever" +) + +const ( + Local = "local" + Remote = "remote" +) + +var ( + PollUntilOptions = []string{Known, Current, Deleted, Forever} +) + +func GetRunner(ctx context.Context, factory cmdutil.Factory, + invFactory inventory.ClientFactory, loader Loader) *Runner { r := &Runner{ + ctx: ctx, factory: factory, invFactory: invFactory, - loader: NewInventoryLoader(loader), - pollerFactoryFunc: pollerFactoryFunc, + loader: loader, + PollerFactoryFunc: pollerFactoryFunc, } c := &cobra.Command{ - Use: "status (DIRECTORY | STDIN)", - RunE: r.runE, + Use: "status (DIRECTORY | STDIN)", + PreRunE: r.preRunE, + RunE: r.runE, } c.Flags().DurationVar(&r.period, "poll-period", 2*time.Second, "Polling period for resource statuses.") @@ -44,18 +67,24 @@ func GetRunner(factory cmdutil.Factory, invFactory inventory.ClientFactory, load c.Flags().StringVar(&r.output, "output", "events", "Output format.") c.Flags().DurationVar(&r.timeout, "timeout", 0, "How long to wait before exiting") + c.Flags().StringVar(&r.invType, "inv-type", Local, "Type of the inventory info, must be local or remote") + c.Flags().StringVar(&r.inventoryNames, "inv-names", "", "Names of targeted inventory: inv1,inv2,...") + c.Flags().StringVar(&r.namespaces, "namespaces", "", "Names of targeted namespaces: ns1,ns2,...") + c.Flags().StringVar(&r.statuses, "statuses", "", "Targeted status: st1,st2...") r.Command = c return r } -func Command(f cmdutil.Factory, invFactory inventory.ClientFactory, loader manifestreader.ManifestLoader) *cobra.Command { - return GetRunner(f, invFactory, loader).Command +func Command(ctx context.Context, f cmdutil.Factory, + invFactory inventory.ClientFactory, loader Loader) *cobra.Command { + return GetRunner(ctx, f, invFactory, loader).Command } // Runner captures the parameters for the command and contains // the run function. type Runner struct { + ctx context.Context Command *cobra.Command factory cmdutil.Factory invFactory inventory.ClientFactory @@ -66,49 +95,167 @@ type Runner struct { timeout time.Duration output string - pollerFactoryFunc func(cmdutil.Factory) (poller.Poller, error) + invType string + inventoryNames string + inventoryNameSet map[string]bool + namespaces string + namespaceSet map[string]bool + statuses string + statusSet map[string]bool + + PollerFactoryFunc func(cmdutil.Factory) (poller.Poller, error) } -// runE implements the logic of the command and will delegate to the -// poller to compute status for each of the resources. One of the printer -// implementations takes care of printing the output. -func (r *Runner) runE(cmd *cobra.Command, args []string) error { +func (r *Runner) preRunE(*cobra.Command, []string) error { + if !slice.ContainsString(PollUntilOptions, r.pollUntil, nil) { + return fmt.Errorf("pollUntil must be one of %s, %s, %s, %s", + Known, Current, Deleted, Forever) + } + + if found := pkgprinters.ValidatePrinterType(r.output); !found { + return fmt.Errorf("unknown output type %q", r.output) + } + + if r.invType != Local && r.invType != Remote { + return fmt.Errorf("inv-type flag should be either local or remote") + } + + if r.invType == Local && r.inventoryNames != "" { + return fmt.Errorf("inv-names flag should only be used when inv-type is set to remote") + } + + if r.inventoryNames != "" { + r.inventoryNameSet = make(map[string]bool) + for _, name := range ss.Split(r.inventoryNames, ",") { + r.inventoryNameSet[name] = true + } + } + + if r.namespaces != "" { + r.namespaceSet = make(map[string]bool) + for _, ns := range ss.Split(r.namespaces, ",") { + r.namespaceSet[ns] = true + } + } + + if r.statuses != "" { + r.statusSet = make(map[string]bool) + for _, st := range ss.Split(r.statuses, ",") { + parsedST := ss.ToLower(st) + r.statusSet[parsedST] = true + } + } + + return nil +} + +// Load inventory info from local storage +// and get info from the cluster based on the local info +// wrap it to be a map mapping from string to objectMetadataSet +func (r *Runner) loadInvFromDisk(cmd *cobra.Command, args []string) (*printer.PrintData, error) { inv, err := r.loader.GetInvInfo(cmd, args) if err != nil { - return err + return nil, err } invClient, err := r.invFactory.NewClient(r.factory) if err != nil { - return err + return nil, err } // Based on the inventory template manifest we look up the inventory // from the live state using the inventory client. identifiers, err := invClient.GetClusterObjs(inv) if err != nil { - return err + return nil, err } - // Exit here if the inventory is empty. - if len(identifiers) == 0 { - _, _ = fmt.Fprint(cmd.OutOrStdout(), "no resources found in the inventory\n") - return nil + printData := printer.PrintData{ + Identifiers: object.ObjMetadataSet{}, + InvNameMap: make(map[object.ObjMetadata]string), + StatusSet: r.statusSet, + } + + for _, obj := range identifiers { + // check if the object is under one of the targeted namespaces + if _, ok := r.namespaceSet[obj.Namespace]; ok || len(r.namespaceSet) == 0 { + // add to the map for future reference + printData.InvNameMap[obj] = inv.Name() + // append to identifiers + printData.Identifiers = append(printData.Identifiers, obj) + } + } + return &printData, nil +} + +// Retrieve a list of inventory object from the cluster +func (r *Runner) listInvFromCluster() (*printer.PrintData, error) { + invClient, err := r.invFactory.NewClient(r.factory) + if err != nil { + return nil, err } // initialize maps in printData printData := printer.PrintData{ - InvNameMap: make(map[object.ObjMetadata]string), - StatusSet: make(map[string]bool), + Identifiers: object.ObjMetadataSet{}, + InvNameMap: make(map[object.ObjMetadata]string), + StatusSet: r.statusSet, } - for _, obj := range identifiers { - // add to the map for future reference - printData.InvNameMap[obj] = inv.Name() - // append to identifiers - printData.Identifiers = append(printData.Identifiers, obj) + + identifiersMap, err := invClient.ListClusterInventoryObjs(r.ctx) + if err != nil { + return nil, err + } + + for invName, identifiers := range identifiersMap { + // Check if there are targeted inventory names and include the current inventory name + if _, ok := r.inventoryNameSet[invName]; !ok && len(r.inventoryNameSet) != 0 { + continue + } + // Filter objects + for _, obj := range identifiers { + // check if the object is under one of the targeted namespaces + if _, ok := r.namespaceSet[obj.Namespace]; ok || len(r.namespaceSet) == 0 { + // add to the map for future reference + printData.InvNameMap[obj] = invName + // append to identifiers + printData.Identifiers = append(printData.Identifiers, obj) + } + } + } + return &printData, nil +} + +// runE implements the logic of the command and will delegate to the +// poller to compute status for each of the resources. One of the printer +// implementations takes care of printing the output. +func (r *Runner) runE(cmd *cobra.Command, args []string) error { + var printData *printer.PrintData + var err error + switch r.invType { + case Local: + if len(args) != 0 { + fmt.Printf("%c[%dm", printcommon.ESC, printcommon.YELLOW) + fmt.Println("Warning: Path is assigned while list flag is enabled, ignore the path") + fmt.Printf("%c[%dm", printcommon.ESC, printcommon.RESET) + } + printData, err = r.loadInvFromDisk(cmd, args) + case Remote: + printData, err = r.listInvFromCluster() + default: + return fmt.Errorf("invType must be either local or remote") + } + if err != nil { + return err + } + + // Exit here if the inventory is empty. + if len(printData.Identifiers) == 0 { + _, _ = fmt.Fprint(cmd.OutOrStdout(), "no resources found in the inventory\n") + return nil } - statusPoller, err := r.pollerFactoryFunc(r.factory) + statusPoller, err := r.PollerFactoryFunc(r.factory) if err != nil { return err } @@ -119,7 +266,7 @@ func (r *Runner) runE(cmd *cobra.Command, args []string) error { In: cmd.InOrStdin(), Out: cmd.OutOrStdout(), ErrOut: cmd.ErrOrStderr(), - }, &printData) + }, printData) if err != nil { return fmt.Errorf("error creating printer: %w", err) } @@ -151,11 +298,11 @@ func (r *Runner) runE(cmd *cobra.Command, args []string) error { return fmt.Errorf("unknown value for pollUntil: %q", r.pollUntil) } - eventChannel := statusPoller.Poll(ctx, identifiers, polling.PollOptions{ + eventChannel := statusPoller.Poll(ctx, printData.Identifiers, polling.PollOptions{ PollInterval: r.period, }) - return printer.Print(eventChannel, identifiers, cancelFunc) + return printer.Print(eventChannel, printData.Identifiers, cancelFunc) } // desiredStatusNotifierFunc returns an Observer function for the diff --git a/cmd/status/cmdstatus_test.go b/cmd/status/cmdstatus_test.go index f60c8959..cf0f196c 100644 --- a/cmd/status/cmdstatus_test.go +++ b/cmd/status/cmdstatus_test.go @@ -83,10 +83,14 @@ func TestCommand(t *testing.T) { expectedOutput string }{ "no inventory template": { + pollUntil: "known", + printer: "events", input: "", expectedErrMsg: "Package uninitialized. Please run \"init\" command.", }, "no inventory in live state": { + pollUntil: "known", + printer: "events", input: inventoryTemplate, expectedOutput: "no resources found in the inventory\n", }, @@ -500,17 +504,19 @@ foo/deployment.apps/default/foo is InProgress: inProgress factory: tf, invFactory: inventory.FakeClientFactory(tc.inventory), loader: NewInventoryLoader(loader), - pollerFactoryFunc: func(c cmdutil.Factory) (poller.Poller, error) { + PollerFactoryFunc: func(c cmdutil.Factory) (poller.Poller, error) { return &fakePoller{tc.events}, nil }, pollUntil: tc.pollUntil, output: tc.printer, timeout: tc.timeout, + invType: Local, } cmd := &cobra.Command{ - RunE: runner.runE, + PreRunE: runner.preRunE, + RunE: runner.runE, } cmd.SetIn(strings.NewReader(tc.input)) var buf bytes.Buffer @@ -542,13 +548,14 @@ foo/deployment.apps/default/foo is InProgress: inProgress factory: tf, invFactory: inventory.FakeClientFactory(tc.inventory), loader: NewInventoryLoader(loader), - pollerFactoryFunc: func(c cmdutil.Factory) (poller.Poller, error) { + PollerFactoryFunc: func(c cmdutil.Factory) (poller.Poller, error) { return &fakePoller{tc.events}, nil }, pollUntil: tc.pollUntil, output: tc.printer, timeout: tc.timeout, + invType: Local, } cmd := &cobra.Command{ diff --git a/pkg/inventory/fake-inventory-client.go b/pkg/inventory/fake-inventory-client.go index 41de174a..e600f6b3 100644 --- a/pkg/inventory/fake-inventory-client.go +++ b/pkg/inventory/fake-inventory-client.go @@ -4,6 +4,8 @@ package inventory import ( + "context" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" cmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/cli-utils/pkg/apis/actuation" @@ -101,3 +103,7 @@ func (fic *FakeClient) GetClusterInventoryInfo(Info) (*unstructured.Unstructured func (fic *FakeClient) GetClusterInventoryObjs(_ Info) (object.UnstructuredSet, error) { return object.UnstructuredSet{}, nil } + +func (fic *FakeClient) ListClusterInventoryObjs(_ context.Context) (map[string]object.ObjMetadataSet, error) { + return map[string]object.ObjMetadataSet{}, nil +} diff --git a/pkg/inventory/inventory-client-factory.go b/pkg/inventory/inventory-client-factory.go index 707a6766..97749bc5 100644 --- a/pkg/inventory/inventory-client-factory.go +++ b/pkg/inventory/inventory-client-factory.go @@ -3,7 +3,9 @@ package inventory -import cmdutil "k8s.io/kubectl/pkg/cmd/util" +import ( + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) var ( _ ClientFactory = ClusterClientFactory{} @@ -20,5 +22,5 @@ type ClusterClientFactory struct { } func (ccf ClusterClientFactory) NewClient(factory cmdutil.Factory) (Client, error) { - return NewClient(factory, WrapInventoryObj, InvInfoToConfigMap, ccf.StatusPolicy) + return NewClient(factory, WrapInventoryObj, InvInfoToConfigMap, ccf.StatusPolicy, ConfigMapGVK) } diff --git a/pkg/inventory/inventory-client.go b/pkg/inventory/inventory-client.go index 82b31464..a7f11643 100644 --- a/pkg/inventory/inventory-client.go +++ b/pkg/inventory/inventory-client.go @@ -11,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" "k8s.io/klog/v2" @@ -44,6 +45,8 @@ type Client interface { GetClusterInventoryInfo(inv Info) (*unstructured.Unstructured, error) // GetClusterInventoryObjs looks up the inventory objects from the cluster. GetClusterInventoryObjs(inv Info) (object.UnstructuredSet, error) + // ListClusterInventoryObjs returns a map mapping from inventory name to a list of cluster inventory objects + ListClusterInventoryObjs(ctx context.Context) (map[string]object.ObjMetadataSet, error) } // ClusterClient is a concrete implementation of the @@ -55,6 +58,7 @@ type ClusterClient struct { InventoryFactoryFunc StorageFactoryFunc invToUnstructuredFunc ToUnstructuredFunc statusPolicy StatusPolicy + gvk schema.GroupVersionKind } var _ Client = &ClusterClient{} @@ -65,6 +69,7 @@ func NewClient(factory cmdutil.Factory, invFunc StorageFactoryFunc, invToUnstructuredFunc ToUnstructuredFunc, statusPolicy StatusPolicy, + gvk schema.GroupVersionKind, ) (*ClusterClient, error) { dc, err := factory.DynamicClient() if err != nil { @@ -85,6 +90,7 @@ func NewClient(factory cmdutil.Factory, InventoryFactoryFunc: invFunc, invToUnstructuredFunc: invToUnstructuredFunc, statusPolicy: statusPolicy, + gvk: gvk, } return &clusterClient, nil } @@ -359,6 +365,37 @@ func (cic *ClusterClient) GetClusterInventoryObjs(inv Info) (object.Unstructured return clusterInvObjects, err } +func (cic *ClusterClient) ListClusterInventoryObjs(ctx context.Context) (map[string]object.ObjMetadataSet, error) { + // Define the mapping + mapping, err := cic.mapper.RESTMapping(cic.gvk.GroupKind(), cic.gvk.Version) + if err != nil { + return nil, err + } + + // retrieve the list from the cluster + clusterInvs, err := cic.dc.Resource(mapping.Resource).List(ctx, metav1.ListOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + if apierrors.IsNotFound(err) { + return map[string]object.ObjMetadataSet{}, nil + } + + identifiers := make(map[string]object.ObjMetadataSet) + + for i, inv := range clusterInvs.Items { + invName := inv.GetName() + identifiers[invName] = object.ObjMetadataSet{} + wrappedInvObjSlice, err := cic.InventoryFactoryFunc(&clusterInvs.Items[i]).Load() + if err != nil { + return nil, err + } + identifiers[invName] = append(identifiers[invName], wrappedInvObjSlice...) + } + + return identifiers, nil +} + // createInventoryObj creates the passed inventory object on the APIServer. func (cic *ClusterClient) createInventoryObj(obj *unstructured.Unstructured, dryRun common.DryRunStrategy) (*unstructured.Unstructured, error) { if dryRun.ClientOrServerDryRun() { diff --git a/pkg/inventory/inventory-client_test.go b/pkg/inventory/inventory-client_test.go index 7eebea16..ccea396c 100644 --- a/pkg/inventory/inventory-client_test.go +++ b/pkg/inventory/inventory-client_test.go @@ -90,7 +90,7 @@ func TestGetClusterInventoryInfo(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { invClient, err := NewClient(tf, - WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy) + WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy, ConfigMapGVK) require.NoError(t, err) var inv *unstructured.Unstructured @@ -201,7 +201,7 @@ func TestMerge(t *testing.T) { tf.FakeDynamicClient.PrependReactor("list", "configmaps", toReactionFunc(tc.clusterObjs)) // Create the local inventory object storing "tc.localObjs" invClient, err := NewClient(tf, - WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy) + WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy, ConfigMapGVK) require.NoError(t, err) // Call "Merge" to create the union of clusterObjs and localObjs. @@ -274,7 +274,7 @@ func TestCreateInventory(t *testing.T) { }) invClient, err := NewClient(tf, - WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy) + WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy, ConfigMapGVK) require.NoError(t, err) inv := invClient.invToUnstructuredFunc(tc.inv) if inv != nil { @@ -367,7 +367,7 @@ func TestReplace(t *testing.T) { // Client and server dry-run do not throw errors. invClient, err := NewClient(tf, - WrapInventoryObj, InvInfoToConfigMap, StatusPolicyAll) + WrapInventoryObj, InvInfoToConfigMap, StatusPolicyAll, ConfigMapGVK) require.NoError(t, err) err = invClient.Replace(copyInventory(), object.ObjMetadataSet{}, nil, common.DryRunClient) if err != nil { @@ -382,7 +382,7 @@ func TestReplace(t *testing.T) { t.Run(name, func(t *testing.T) { // Create inventory client, and store the cluster objs in the inventory object. invClient, err := NewClient(tf, - WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy) + WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy, ConfigMapGVK) require.NoError(t, err) wrappedInv := invClient.InventoryFactoryFunc(inventoryObj) if err := wrappedInv.Store(tc.clusterObjs, tc.objStatus); err != nil { @@ -453,7 +453,7 @@ func TestGetClusterObjs(t *testing.T) { tf.FakeDynamicClient.PrependReactor("list", "configmaps", toReactionFunc(tc.clusterObjs)) invClient, err := NewClient(tf, - WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy) + WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy, ConfigMapGVK) require.NoError(t, err) clusterObjs, err := invClient.GetClusterObjs(tc.localInv) if tc.isError { @@ -516,7 +516,7 @@ func TestDeleteInventoryObj(t *testing.T) { defer tf.Cleanup() invClient, err := NewClient(tf, - WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy) + WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy, ConfigMapGVK) require.NoError(t, err) inv := invClient.invToUnstructuredFunc(tc.inv) if inv != nil { @@ -563,7 +563,7 @@ func TestApplyInventoryNamespace(t *testing.T) { }) invClient, err := NewClient(tf, - WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy) + WrapInventoryObj, InvInfoToConfigMap, tc.statusPolicy, ConfigMapGVK) require.NoError(t, err) err = invClient.ApplyInventoryNamespace(tc.namespace, tc.dryRunStrategy) assert.NoError(t, err) diff --git a/pkg/inventory/inventorycm.go b/pkg/inventory/inventorycm.go index 50bb5468..e51cf989 100644 --- a/pkg/inventory/inventorycm.go +++ b/pkg/inventory/inventorycm.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/klog/v2" "sigs.k8s.io/cli-utils/pkg/apis/actuation" @@ -24,6 +25,12 @@ import ( "sigs.k8s.io/cli-utils/pkg/object" ) +var ConfigMapGVK = schema.GroupVersionKind{ + Group: "", + Kind: "ConfigMap", + Version: "v1", +} + // WrapInventoryObj takes a passed ConfigMap (as a resource.Info), // wraps it with the ConfigMap and upcasts the wrapper as // an the Inventory interface. diff --git a/test/e2e/customprovider/provider.go b/test/e2e/customprovider/provider.go index 805e516a..89d3ede5 100644 --- a/test/e2e/customprovider/provider.go +++ b/test/e2e/customprovider/provider.go @@ -113,7 +113,7 @@ type CustomClientFactory struct { func (CustomClientFactory) NewClient(factory util.Factory) (inventory.Client, error) { return inventory.NewClient(factory, - WrapInventoryObj, invToUnstructuredFunc, inventory.StatusPolicyAll) + WrapInventoryObj, invToUnstructuredFunc, inventory.StatusPolicyAll, inventory.ConfigMapGVK) } func invToUnstructuredFunc(inv inventory.Info) *unstructured.Unstructured {