From 7c9859885807a1bbdbd812d295aec0edfb52af64 Mon Sep 17 00:00:00 2001 From: Robert Brennan Date: Thu, 17 Dec 2020 17:32:01 -0500 Subject: [PATCH] Fix test fixtures, add a test for controllers (#455) * first pass at fixing test fixtures * tests mostly working * add controller test * remove debug stuff * delint * revert test file * remove extra controllers from fixtures * delint * fix messages --- pkg/kube/resources_test.go | 29 ++- pkg/kube/test_files/test_1/deployment2.yaml | 20 ++ pkg/validator/controller_test.go | 81 +++--- pkg/validator/fullaudit_test.go | 38 +-- pkg/validator/pod_test.go | 10 - test/fixtures.go | 264 ++++++++++---------- 6 files changed, 247 insertions(+), 195 deletions(-) create mode 100644 pkg/kube/test_files/test_1/deployment2.yaml diff --git a/pkg/kube/resources_test.go b/pkg/kube/resources_test.go index b6b4caf14..dda9e7b62 100644 --- a/pkg/kube/resources_test.go +++ b/pkg/kube/resources_test.go @@ -27,13 +27,13 @@ func TestGetResourcesFromPath(t *testing.T) { assert.Equal(t, 1, len(resources.Namespaces), "Should have a namespace") assert.Equal(t, "two", resources.Namespaces[0].ObjectMeta.Name) - assert.Equal(t, 8, len(resources.Controllers), "Should have eight controllers") + assert.Equal(t, 9, len(resources.Controllers), "Should have eight controllers") namespaceCount := map[string]int{} for _, controller := range resources.Controllers { namespaceCount[controller.ObjectMeta.GetNamespace()]++ } - assert.Equal(t, 7, namespaceCount[""], "Should have seven controller in default namespace") - assert.Equal(t, 1, namespaceCount["two"], "Should have one controller in namespace 'two'") + assert.Equal(t, 8, namespaceCount[""]) + assert.Equal(t, 1, namespaceCount["two"]) } func TestGetMultipleResourceFromSingleFile(t *testing.T) { @@ -87,10 +87,7 @@ func TestAddResourcesFromReader(t *testing.T) { } func TestGetResourceFromAPI(t *testing.T) { - k8s, dynamicInterface := test.SetupTestAPI() - k8s = test.SetupAddControllers(context.Background(), k8s, "test") - // TODO find a way to mock out the dynamic client - // and create fake pods in order to find all of the controllers. + k8s, dynamicInterface := test.SetupTestAPI(test.GetMockControllers("test")...) resources, err := CreateResourceProviderFromAPI(context.Background(), k8s, "test", &dynamicInterface) assert.Equal(t, nil, err, "Error should be nil") @@ -99,7 +96,19 @@ func TestGetResourceFromAPI(t *testing.T) { assert.IsType(t, time.Now(), resources.CreationTime, "Creation time should be set") assert.Equal(t, 0, len(resources.Nodes), "Should not have any nodes") - assert.Equal(t, 1, len(resources.Controllers), "Should have 1 controller") - - assert.Equal(t, "", resources.Controllers[0].ObjectMeta.GetName()) + assert.Equal(t, 5, len(resources.Controllers), "Should have 5 controllers") + + expectedNames := map[string]bool{ + "deploy": false, + "job": false, + "cronjob": false, + "statefulset": false, + "daemonset": false, + } + for _, ctrl := range resources.Controllers { + expectedNames[ctrl.ObjectMeta.GetName()] = true + } + for name, val := range expectedNames { + assert.Equal(t, true, val, name) + } } diff --git a/pkg/kube/test_files/test_1/deployment2.yaml b/pkg/kube/test_files/test_1/deployment2.yaml new file mode 100644 index 000000000..a705174d1 --- /dev/null +++ b/pkg/kube/test_files/test_1/deployment2.yaml @@ -0,0 +1,20 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: test-deployment-2 +spec: + replicas: 2 + selector: + matchLabels: + app: test-deployment + template: + metadata: + labels: + app: test-deployment + spec: + containers: + - name: ubuntu + image: ubuntu + ports: + - containerPort: 3000 + diff --git a/pkg/validator/controller_test.go b/pkg/validator/controller_test.go index 8bfed5051..bdb70482c 100644 --- a/pkg/validator/controller_test.go +++ b/pkg/validator/controller_test.go @@ -16,6 +16,7 @@ package validator import ( "context" + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -59,41 +60,59 @@ func TestValidateController(t *testing.T) { } func TestControllerLevelChecks(t *testing.T) { - c := conf.Configuration{ - Checks: map[string]conf.Severity{ - "multipleReplicasForDeployment": conf.SeverityDanger, - }, - } - resources, err := kube.CreateResourceProviderFromPath("../kube/test_files/test_1") - - assert.Equal(t, nil, err, "Error should be nil") - - assert.Equal(t, 8, len(resources.Controllers), "Should have eight controllers") - - expectedSum := CountSummary{ - Successes: uint(0), - Warnings: uint(0), - Dangers: uint(1), - } - - expectedResults := ResultSet{ - "multipleReplicasForDeployment": {ID: "multipleReplicasForDeployment", Message: "Only one replica is scheduled", Success: false, Severity: "danger", Category: "Reliability"}, - } - - for _, controller := range resources.Controllers { - if controller.Kind == "Deployment" && controller.ObjectMeta.GetName() == "test-deployment" { - actualResult, err := ValidateController(context.Background(), &c, controller) - if err != nil { - panic(err) + testResources := func(res *kube.ResourceProvider) { + c := conf.Configuration{ + Checks: map[string]conf.Severity{ + "multipleReplicasForDeployment": conf.SeverityDanger, + }, + } + expectedResult := ResultMessage{ + ID: "multipleReplicasForDeployment", + Severity: "danger", + Category: "Reliability", + } + for _, controller := range res.Controllers { + if controller.Kind == "Deployment" { + actualResult, err := ValidateController(context.Background(), &c, controller) + if err != nil { + panic(err) + } + if controller.ObjectMeta.GetName() == "test-deployment-2" { + expectedResult.Success = true + expectedResult.Message = "Multiple replicas are scheduled" + } else if controller.ObjectMeta.GetName() == "test-deployment" { + expectedResult.Success = false + expectedResult.Message = "Only one replica is scheduled" + } + expectedResults := ResultSet{ + "multipleReplicasForDeployment": expectedResult, + } + + assert.Equal(t, "Deployment", actualResult.Kind) + assert.Equal(t, 1, len(actualResult.Results), "should be equal") + assert.EqualValues(t, expectedResults, actualResult.Results, controller.ObjectMeta.GetName()) } - - assert.Equal(t, "Deployment", actualResult.Kind) - assert.Equal(t, 1, len(actualResult.Results), "should be equal") - assert.EqualValues(t, expectedSum, actualResult.GetSummary()) - assert.EqualValues(t, expectedResults, actualResult.Results) } } + res, err := kube.CreateResourceProviderFromPath("../kube/test_files/test_1") + assert.Equal(t, nil, err, "Error should be nil") + assert.Equal(t, 9, len(res.Controllers), "Should have eight controllers") + testResources(res) + + replicaSpec := map[string]interface{}{"replicas": 2} + b, err := json.Marshal(replicaSpec) + assert.NoError(t, err) + err = json.Unmarshal(b, &replicaSpec) + + d1, p1 := test.MockDeploy("test", "test-deployment") + d2, p2 := test.MockDeploy("test", "test-deployment-2") + d2.Object["spec"] = replicaSpec + k8s, dynamicClient := test.SetupTestAPI(&d1, &p1, &d2, &p2) + res, err = kube.CreateResourceProviderFromAPI(context.Background(), k8s, "test", &dynamicClient) + assert.Equal(t, err, nil, "error should be nil") + assert.Equal(t, 2, len(res.Controllers), "Should have two controllers") + testResources(res) } func TestSkipHealthChecks(t *testing.T) { diff --git a/pkg/validator/fullaudit_test.go b/pkg/validator/fullaudit_test.go index 63690da84..5eb193259 100644 --- a/pkg/validator/fullaudit_test.go +++ b/pkg/validator/fullaudit_test.go @@ -11,13 +11,10 @@ import ( ) func TestGetTemplateData(t *testing.T) { - k8s, dynamicClient := test.SetupTestAPI() - k8s = test.SetupAddControllers(context.Background(), k8s, "test") - k8s = test.SetupAddExtraControllerVersions(context.Background(), k8s, "test-extra") - // TODO figure out how to mock out dynamic client. - // and add in pods for all controllers to fill out tests. + k8s, dynamicClient := test.SetupTestAPI(test.GetMockControllers("test")...) resources, err := kube.CreateResourceProviderFromAPI(context.Background(), k8s, "test", &dynamicClient) assert.Equal(t, err, nil, "error should be nil") + assert.Equal(t, 5, len(resources.Controllers)) c := conf.Configuration{ Checks: map[string]conf.Severity{ @@ -28,29 +25,38 @@ func TestGetTemplateData(t *testing.T) { sum := CountSummary{ Successes: uint(0), - Warnings: uint(1), - Dangers: uint(1), + Warnings: uint(3), + Dangers: uint(3), } actualAudit, err := RunAudit(context.Background(), c, resources) - assert.Equal(t, err, nil, "error should be nil") - assert.EqualValues(t, sum, actualAudit.GetSummary()) assert.Equal(t, actualAudit.SourceType, "Cluster", "should be from a cluster") assert.Equal(t, actualAudit.SourceName, "test", "should be from a cluster") - expected := []struct { + expectedResults := []struct { kind string results int }{ - {kind: "Pod", results: 2}, + {kind: "StatefulSet", results: 2}, + {kind: "DaemonSet", results: 2}, + {kind: "Deployment", results: 2}, + {kind: "Job", results: 0}, + {kind: "CronJob", results: 0}, } - assert.Equal(t, len(expected), len(actualAudit.Results)) - for idx, result := range actualAudit.Results { - assert.Equal(t, expected[idx].kind, result.Kind) - assert.Equal(t, 1, len(result.PodResult.ContainerResults)) - assert.Equal(t, expected[idx].results, len(result.PodResult.ContainerResults[0].Results)) + assert.Equal(t, len(expectedResults), len(actualAudit.Results)) + for _, result := range actualAudit.Results { + found := false + for _, expected := range expectedResults { + if expected.kind != result.Kind { + continue + } + found = true + assert.Equal(t, 1, len(result.PodResult.ContainerResults)) + assert.Equal(t, expected.results, len(result.PodResult.ContainerResults[0].Results)) + } + assert.Equal(t, found, true) } } diff --git a/pkg/validator/pod_test.go b/pkg/validator/pod_test.go index bfbc22abb..4e5a7b4b3 100644 --- a/pkg/validator/pod_test.go +++ b/pkg/validator/pod_test.go @@ -36,8 +36,6 @@ func TestValidatePod(t *testing.T) { }, } - k8s, _ := test.SetupTestAPI() - k8s = test.SetupAddControllers(context.Background(), k8s, "test") p := test.MockPod() deployment, err := kube.NewGenericWorkloadFromPod(p, nil) assert.NoError(t, err) @@ -73,8 +71,6 @@ func TestInvalidIPCPod(t *testing.T) { }, } - k8s, _ := test.SetupTestAPI() - k8s = test.SetupAddControllers(context.Background(), k8s, "test") p := test.MockPod() p.Spec.HostIPC = true workload, err := kube.NewGenericWorkloadFromPod(p, nil) @@ -110,8 +106,6 @@ func TestInvalidNeworkPod(t *testing.T) { }, } - k8s, _ := test.SetupTestAPI() - k8s = test.SetupAddControllers(context.Background(), k8s, "test") p := test.MockPod() p.Spec.HostNetwork = true workload, err := kube.NewGenericWorkloadFromPod(p, nil) @@ -148,8 +142,6 @@ func TestInvalidPIDPod(t *testing.T) { }, } - k8s, _ := test.SetupTestAPI() - k8s = test.SetupAddControllers(context.Background(), k8s, "test") p := test.MockPod() p.Spec.HostPID = true workload, err := kube.NewGenericWorkloadFromPod(p, nil) @@ -192,8 +184,6 @@ func TestExemption(t *testing.T) { }, } - k8s, _ := test.SetupTestAPI() - k8s = test.SetupAddControllers(context.Background(), k8s, "test") p := test.MockPod() p.Spec.HostIPC = true p.ObjectMeta = metav1.ObjectMeta{ diff --git a/test/fixtures.go b/test/fixtures.go index 2de46e980..13163c82c 100644 --- a/test/fixtures.go +++ b/test/fixtures.go @@ -1,7 +1,7 @@ package test import ( - "context" + "encoding/json" appsv1 "k8s.io/api/apps/v1" appsv1beta1 "k8s.io/api/apps/v1beta1" @@ -10,6 +10,7 @@ import ( batchv1beta1 "k8s.io/api/batch/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" dynamicFake "k8s.io/client-go/dynamic/fake" @@ -17,6 +18,20 @@ import ( "k8s.io/client-go/kubernetes/fake" ) +func newUnstructured(apiVersion, kind, namespace, name string, spec interface{}) unstructured.Unstructured { + return unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": apiVersion, + "kind": kind, + "metadata": map[string]interface{}{ + "namespace": namespace, + "name": name, + }, + "spec": spec, + }, + } +} + // MockContainer creates a container object func MockContainer(name string) corev1.Container { c := corev1.Container{ @@ -45,166 +60,159 @@ func MockNakedPod() corev1.Pod { } } -// MockDeploy creates a Deployment object. -func MockDeploy() appsv1.Deployment { - p := MockPod() - d := appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, +// MockController creates a mock controller and pod +func MockController(apiVersion, kind, namespace, name string, spec interface{}, podSpec corev1.PodSpec) (unstructured.Unstructured, corev1.Pod) { + d := newUnstructured(apiVersion, kind, namespace, name, spec) + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-12345", + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: apiVersion, + Kind: kind, + Name: name, + }}, }, + Spec: podSpec, } - return d + return d, pod } -// MockStatefulSet creates a StatefulSet object. -func MockStatefulSet() appsv1.StatefulSet { +// MockControllerWithNormalSpec mocks a controller with podspec at spec.template.spec +func MockControllerWithNormalSpec(apiVersion, kind, namespace, name string) (unstructured.Unstructured, corev1.Pod) { p := MockPod() - s := appsv1.StatefulSet{ - Spec: appsv1.StatefulSetSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, + b, err := json.Marshal(p.Spec) + if err != nil { + panic(err) + } + pSpec := map[string]interface{}{} + err = json.Unmarshal(b, &pSpec) + if err != nil { + panic(err) + } + spec := map[string]interface{}{ + "template": map[string]interface{}{ + "spec": pSpec, }, } - return s + return MockController(apiVersion, kind, namespace, name, spec, p.Spec) +} + +// MockDeploy creates a Deployment object. +func MockDeploy(namespace, name string) (unstructured.Unstructured, corev1.Pod) { + return MockControllerWithNormalSpec("apps/v1", "Deployment", namespace, name) +} + +// MockStatefulSet creates a StatefulSet object. +func MockStatefulSet(namespace, name string) (unstructured.Unstructured, corev1.Pod) { + return MockControllerWithNormalSpec("apps/v1", "StatefulSet", namespace, name) } // MockDaemonSet creates a DaemonSet object. -func MockDaemonSet() appsv1.DaemonSet { - p := MockPod() - return appsv1.DaemonSet{ - Spec: appsv1.DaemonSetSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, - }, - } +func MockDaemonSet(namespace, name string) (unstructured.Unstructured, corev1.Pod) { + return MockControllerWithNormalSpec("apps/v1", "DaemonSet", namespace, name) } // MockJob creates a Job object. -func MockJob() batchv1.Job { - p := MockPod() - return batchv1.Job{ - Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, - }, - } +func MockJob(namespace, name string) (unstructured.Unstructured, corev1.Pod) { + return MockControllerWithNormalSpec("batch/v1", "Job", namespace, name) } // MockCronJob creates a CronJob object. -func MockCronJob() batchv1beta1.CronJob { +func MockCronJob(namespace, name string) (unstructured.Unstructured, corev1.Pod) { p := MockPod() - return batchv1beta1.CronJob{ - Spec: batchv1beta1.CronJobSpec{ - JobTemplate: batchv1beta1.JobTemplateSpec{ - Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, + b, err := json.Marshal(p.Spec) + if err != nil { + panic(err) + } + pSpec := map[string]interface{}{} + err = json.Unmarshal(b, &pSpec) + if err != nil { + panic(err) + } + spec := map[string]interface{}{ + "job_template": map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": pSpec, }, }, }, } + return MockController("batch/v1beta1", "CronJob", namespace, name, spec, p.Spec) } // MockReplicationController creates a ReplicationController object. -func MockReplicationController() corev1.ReplicationController { - p := MockPod() - return corev1.ReplicationController{ - Spec: corev1.ReplicationControllerSpec{ - Template: &corev1.PodTemplateSpec{Spec: p.Spec}, - }, - } +func MockReplicationController(namespace, name string) (unstructured.Unstructured, corev1.Pod) { + return MockControllerWithNormalSpec("core/v1", "ReplicationController", namespace, name) } // SetupTestAPI creates a test kube API struct. -func SetupTestAPI() (kubernetes.Interface, dynamic.Interface) { +func SetupTestAPI(objects ...runtime.Object) (kubernetes.Interface, dynamic.Interface) { scheme := runtime.NewScheme() - - return fake.NewSimpleClientset(), dynamicFake.NewSimpleDynamicClient(scheme) -} - -// SetupAddControllers creates mock controllers and adds them to the test clientset. -func SetupAddControllers(ctx context.Context, k kubernetes.Interface, namespace string) kubernetes.Interface { - d1 := MockDeploy() - if _, err := k.AppsV1().Deployments(namespace).Create(ctx, &d1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - s1 := MockStatefulSet() - if _, err := k.AppsV1().StatefulSets(namespace).Create(ctx, &s1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - ds1 := MockDaemonSet() - if _, err := k.AppsV1().DaemonSets(namespace).Create(ctx, &ds1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - j1 := MockJob() - if _, err := k.BatchV1().Jobs(namespace).Create(ctx, &j1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - cj1 := MockCronJob() - if _, err := k.BatchV1beta1().CronJobs(namespace).Create(ctx, &cj1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - rc1 := MockReplicationController() - if _, err := k.CoreV1().ReplicationControllers(namespace).Create(ctx, &rc1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - p1 := MockNakedPod() - if _, err := k.CoreV1().Pods(namespace).Create(ctx, &p1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - return k -} - -// SetupAddExtraControllerVersions creates mock controllers and adds them to the test clientset. -func SetupAddExtraControllerVersions(ctx context.Context, k kubernetes.Interface, namespace string) kubernetes.Interface { - p := MockPod() - - dv1b1 := appsv1beta1.Deployment{ - Spec: appsv1beta1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, + appsv1.AddToScheme(scheme) + corev1.AddToScheme(scheme) + fake.AddToScheme(scheme) + dynamicClient := dynamicFake.NewSimpleDynamicClient(scheme, objects...) + k := fake.NewSimpleClientset(objects...) + k.Resources = []*metav1.APIResourceList{ + { + GroupVersion: corev1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Name: "pods", Namespaced: true, Kind: "Pod"}, + {Name: "replicationcontrollers", Namespaced: true, Kind: "ReplicationController"}, + }, }, - } - if _, err := k.AppsV1beta1().Deployments(namespace).Create(ctx, &dv1b1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - dv1b2 := appsv1beta2.Deployment{ - Spec: appsv1beta2.DeploymentSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, + { + GroupVersion: appsv1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Name: "deployments", Namespaced: true, Kind: "Deployment"}, + {Name: "daemonsets", Namespaced: true, Kind: "DaemonSet"}, + {Name: "statefulsets", Namespaced: true, Kind: "StatefulSet"}, + }, }, - } - if _, err := k.AppsV1beta2().Deployments(namespace).Create(ctx, &dv1b2, metav1.CreateOptions{}); err != nil { - panic(err) - } - - ssv1b1 := appsv1beta1.StatefulSet{ - Spec: appsv1beta1.StatefulSetSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, + { + GroupVersion: batchv1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Name: "jobs", Namespaced: true, Kind: "Job"}, + }, }, - } - if _, err := k.AppsV1beta1().StatefulSets(namespace).Create(ctx, &ssv1b1, metav1.CreateOptions{}); err != nil { - panic(err) - } - - ssv1b2 := appsv1beta2.StatefulSet{ - Spec: appsv1beta2.StatefulSetSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, + { + GroupVersion: batchv1beta1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Name: "cronjobs", Namespaced: true, Kind: "CronJob"}, + }, }, - } - if _, err := k.AppsV1beta2().StatefulSets(namespace).Create(ctx, &ssv1b2, metav1.CreateOptions{}); err != nil { - panic(err) - } - - dsv1b2 := appsv1beta2.DaemonSet{ - Spec: appsv1beta2.DaemonSetSpec{ - Template: corev1.PodTemplateSpec{Spec: p.Spec}, + { + GroupVersion: appsv1beta2.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Name: "deployments", Namespaced: true, Kind: "Deployment"}, + {Name: "deployments/scale", Namespaced: true, Kind: "Scale", Group: "apps", Version: "v1beta2"}, + }, + }, + { + GroupVersion: appsv1beta1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Name: "statefulsets", Namespaced: true, Kind: "StatefulSet"}, + {Name: "statefulsets/scale", Namespaced: true, Kind: "Scale", Group: "apps", Version: "v1beta1"}, + }, }, } - if _, err := k.AppsV1beta2().DaemonSets(namespace).Create(ctx, &dsv1b2, metav1.CreateOptions{}); err != nil { - panic(err) + return k, dynamicClient +} + +// GetMockControllers returns mocked controllers for 5 major controller types +func GetMockControllers(namespace string) []runtime.Object { + deploy, deployPod := MockDeploy(namespace, "deploy") + statefulset, statefulsetPod := MockStatefulSet(namespace, "statefulset") + daemonset, daemonsetPod := MockDaemonSet(namespace, "daemonset") + job, jobPod := MockJob(namespace, "job") + cronjob, cronjobPod := MockCronJob(namespace, "cronjob") + return []runtime.Object{ + &deploy, &deployPod, + &daemonset, &daemonsetPod, + &statefulset, &statefulsetPod, + &cronjob, &cronjobPod, + &job, &jobPod, } - return k }