diff --git a/chart/kubeapps/README.md b/chart/kubeapps/README.md index 62ca268375d..265aa137c79 100644 --- a/chart/kubeapps/README.md +++ b/chart/kubeapps/README.md @@ -485,13 +485,13 @@ Once you have installed Kubeapps follow the [Getting Started Guide](https://gith | `kubeappsapis.containerSecurityContext.runAsUser` | Set KubeappsAPIs container's Security Context runAsUser | `1001` | | `kubeappsapis.containerSecurityContext.runAsNonRoot` | Set KubeappsAPIs container's Security Context runAsNonRoot | `true` | | `kubeappsapis.livenessProbe.enabled` | Enable livenessProbe | `true` | -| `kubeappsapis.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `60` | +| `kubeappsapis.livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `10` | | `kubeappsapis.livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | | `kubeappsapis.livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `5` | | `kubeappsapis.livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `6` | | `kubeappsapis.livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | | `kubeappsapis.readinessProbe.enabled` | Enable readinessProbe | `true` | -| `kubeappsapis.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `0` | +| `kubeappsapis.readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `5` | | `kubeappsapis.readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` | | `kubeappsapis.readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `5` | | `kubeappsapis.readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `6` | @@ -1034,4 +1034,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. diff --git a/chart/kubeapps/templates/apprepository/apprepositories.yaml b/chart/kubeapps/templates/apprepository/apprepositories.yaml index e12718f458d..75f64e8b3f8 100644 --- a/chart/kubeapps/templates/apprepository/apprepositories.yaml +++ b/chart/kubeapps/templates/apprepository/apprepositories.yaml @@ -26,12 +26,14 @@ spec: - {{ . }} {{- end }} {{- end }} - {{- if or $.Values.apprepository.containerSecurityContext.enabled $.Values.apprepository.initialReposProxy.enabled .nodeSelector }} + {{- if or $.Values.apprepository.podSecurityContext.enabled $.Values.apprepository.containerSecurityContext.enabled $.Values.apprepository.initialReposProxy.enabled .nodeSelector .tolerations}} syncJobPodTemplate: spec: - {{- if $.Values.apprepository.initialReposProxy.enabled }} + {{- if or $.Values.apprepository.initialReposProxy.enabled $.Values.apprepository.containerSecurityContext.enabled }} containers: - - env: + - + {{- if $.Values.apprepository.initialReposProxy.enabled }} + env: - name: https_proxy value: {{ $.Values.apprepository.initialReposProxy.httpsProxy }} - name: http_proxy @@ -40,8 +42,11 @@ spec: value: {{ $.Values.apprepository.initialReposProxy.noProxy }} {{- end }} {{- if $.Values.apprepository.containerSecurityContext.enabled }} - securityContext: - runAsUser: {{ $.Values.apprepository.containerSecurityContext.runAsUser }} + securityContext: {{- omit $.Values.apprepository.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- end }} + {{- if $.Values.apprepository.podSecurityContext.enabled }} + securityContext: {{- omit $.Values.apprepository.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} {{- if .nodeSelector }} nodeSelector: {{- toYaml .nodeSelector | nindent 8 }} diff --git a/chart/kubeapps/templates/apprepository/deployment.yaml b/chart/kubeapps/templates/apprepository/deployment.yaml index 02a668da752..1395ae97d4f 100644 --- a/chart/kubeapps/templates/apprepository/deployment.yaml +++ b/chart/kubeapps/templates/apprepository/deployment.yaml @@ -121,6 +121,12 @@ spec: - --custom-labels={{ (print $key "=" $value) | quote }} {{- end }} {{- end }} + {{- if.Values.apprepository.containerSecurityContext.enabled }} + - --default-container-security-context={{ toJson .Values.apprepository.containerSecurityContext | squote }} + {{- end }} + {{- if.Values.apprepository.podSecurityContext.enabled }} + - --default-pod-security-context={{ toJson .Values.apprepository.podSecurityContext | squote }} + {{- end }} {{- range .Values.apprepository.extraFlags }} - {{ . }} {{- end }} diff --git a/chart/kubeapps/templates/kubeappsapis/deployment.yaml b/chart/kubeapps/templates/kubeappsapis/deployment.yaml index f8eccbc7a95..fa262684607 100644 --- a/chart/kubeapps/templates/kubeappsapis/deployment.yaml +++ b/chart/kubeapps/templates/kubeappsapis/deployment.yaml @@ -189,7 +189,6 @@ spec: livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.kubeappsapis.livenessProbe "enabled") "context" $) | nindent 12 }} exec: command: ["grpc_health_probe", "-addr=:{{ .Values.kubeappsapis.containerPorts.http }}"] - initialDelaySeconds: 10 {{- end }} {{- if .Values.kubeappsapis.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.kubeappsapis.customReadinessProbe "context" $) | nindent 12 }} @@ -197,7 +196,6 @@ spec: readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.kubeappsapis.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: ["grpc_health_probe", "-addr=:{{ .Values.kubeappsapis.containerPorts.http }}"] - initialDelaySeconds: 5 {{- end }} {{- if .Values.kubeappsapis.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.kubeappsapis.customStartupProbe "context" $) | nindent 12 }} diff --git a/chart/kubeapps/values.yaml b/chart/kubeapps/values.yaml index a96535561c2..a6ec91938ff 100644 --- a/chart/kubeapps/values.yaml +++ b/chart/kubeapps/values.yaml @@ -292,7 +292,7 @@ frontend: podSecurityContext: enabled: true fsGroup: 1001 - ## Configure Container Security Context (only main container) + ## Configure Container Security Context for NGINX ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param frontend.containerSecurityContext.enabled Enabled NGINX containers' Security Context ## @param frontend.containerSecurityContext.runAsUser Set NGINX container's Security Context runAsUser @@ -643,7 +643,7 @@ dashboard: podSecurityContext: enabled: true fsGroup: 1001 - ## Configure Container Security Context (only main container) + ## Configure Container Security Context for Dashboard ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param dashboard.containerSecurityContext.enabled Enabled Dashboard containers' Security Context ## @param dashboard.containerSecurityContext.runAsUser Set Dashboard container's Security Context runAsUser @@ -990,7 +990,7 @@ apprepository: podSecurityContext: enabled: true fsGroup: 1001 - ## Configure Container Security Context (only main container) + ## Configure Container Security Context for App Repository jobs ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param apprepository.containerSecurityContext.enabled Enabled AppRepository Controller containers' Security Context ## @param apprepository.containerSecurityContext.runAsUser Set AppRepository Controller container's Security Context runAsUser @@ -1657,7 +1657,7 @@ kubeappsapis: podSecurityContext: enabled: true fsGroup: 1001 - ## Configure Container Security Context (only main container) + ## Configure Container Security Context for Kubeapps APIs ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param kubeappsapis.containerSecurityContext.enabled Enabled KubeappsAPIs containers' Security Context ## @param kubeappsapis.containerSecurityContext.runAsUser Set KubeappsAPIs container's Security Context runAsUser @@ -1680,7 +1680,7 @@ kubeappsapis: ## livenessProbe: enabled: true - initialDelaySeconds: 60 + initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 6 @@ -1694,7 +1694,7 @@ kubeappsapis: ## readinessProbe: enabled: true - initialDelaySeconds: 0 + initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 6 diff --git a/cmd/apprepository-controller/cmd/root.go b/cmd/apprepository-controller/cmd/root.go index b7c58db8849..bc46a6cdf11 100644 --- a/cmd/apprepository-controller/cmd/root.go +++ b/cmd/apprepository-controller/cmd/root.go @@ -85,13 +85,12 @@ func setFlags(c *cobra.Command) { c.Flags().StringVar(&serveOpts.DBSecretKey, "database-secret-key", "postgresql-root-password", "Kubernetes secret key used for database credentials") c.Flags().StringVar(&serveOpts.UserAgentComment, "user-agent-comment", "", "UserAgent comment used during outbound requests") c.Flags().StringVar(&serveOpts.Crontab, "crontab", "*/10 * * * *", "CronTab to specify schedule") - // TTLSecondsAfterFinished specifies the number of seconds a sync job should live after finishing. - // The support for this is currently beta in K8s (v1.21), older versions require a feature gate being set to enable it. - // See https://kubernetes.io/docs/concepts/workloads/controllers/job/#clean-up-finished-jobs-automatically - c.Flags().StringVar(&serveOpts.TTLSecondsAfterFinished, "ttl-lifetime-afterfinished-job", "3600", "Lifetime limit after which the resource Jobs are deleted expressed in seconds by default is 3600 (1h) ") + c.Flags().StringVar(&serveOpts.TTLSecondsAfterFinished, "ttl-lifetime-afterfinished-job", "3600", "Lifetime limit after which the resource Jobs are deleted expressed in seconds by default is 3600 (1h)") c.Flags().StringSliceVar(&serveOpts.CustomAnnotations, "custom-annotations", []string{""}, "optional annotations to be passed to the generated CronJobs, Jobs and Pods objects. For example: my/annotation=foo") c.Flags().StringSliceVar(&serveOpts.CustomLabels, "custom-labels", []string{""}, "optional labels to be passed to the generated CronJobs, Jobs and Pods objects. For example: my/label=foo") c.Flags().BoolVar(&serveOpts.V1Beta1CronJobs, "v1-beta1-cron-jobs", false, "Defaults to false and so using the v1 cronjobs.") + c.Flags().StringVar(&serveOpts.DefaultPodSecContext, "default-pod-security-context", "", "Default Pod Security Context to use for the cleanup jobs and sync jobs (unless overridden by the CRD)") + c.Flags().StringVar(&serveOpts.DefaultContainerSecContext, "default-container-security-context", "", "Default Container Security Context to use for the cleanup jobs and sync jobs (unless overridden by the CRD)") } // initConfig reads in config file and ENV variables if set. diff --git a/cmd/apprepository-controller/cmd/root_test.go b/cmd/apprepository-controller/cmd/root_test.go index ceac39d1986..622d4db0f1d 100644 --- a/cmd/apprepository-controller/cmd/root_test.go +++ b/cmd/apprepository-controller/cmd/root_test.go @@ -25,26 +25,30 @@ func TestParseFlagsCorrect(t *testing.T) { "no arguments returns default flag values", []string{}, server.Config{ - Kubeconfig: "", - APIServerURL: "", - RepoSyncImage: "docker.io/kubeapps/asset-syncer:latest", - RepoSyncImagePullSecrets: nil, - RepoSyncCommand: "/chart-repo", - KubeappsNamespace: "kubeapps", - GlobalPackagingNamespace: "kubeapps", - ReposPerNamespace: true, - DBURL: "localhost", - DBUser: "root", - DBName: "charts", - DBSecretName: "kubeapps-db", - DBSecretKey: "postgresql-root-password", - UserAgentComment: "", - Crontab: "*/10 * * * *", - TTLSecondsAfterFinished: "3600", - CustomAnnotations: []string{""}, - CustomLabels: []string{""}, - ParsedCustomAnnotations: map[string]string{}, - ParsedCustomLabels: map[string]string{}, + Kubeconfig: "", + APIServerURL: "", + RepoSyncImage: "docker.io/kubeapps/asset-syncer:latest", + RepoSyncImagePullSecrets: nil, + RepoSyncCommand: "/chart-repo", + KubeappsNamespace: "kubeapps", + GlobalPackagingNamespace: "kubeapps", + ReposPerNamespace: true, + DBURL: "localhost", + DBUser: "root", + DBName: "charts", + DBSecretName: "kubeapps-db", + DBSecretKey: "postgresql-root-password", + UserAgentComment: "", + Crontab: "*/10 * * * *", + TTLSecondsAfterFinished: "3600", + CustomAnnotations: []string{""}, + CustomLabels: []string{""}, + ParsedCustomAnnotations: map[string]string{}, + ParsedCustomLabels: map[string]string{}, + ImagePullSecretsRefs: nil, + V1Beta1CronJobs: false, + DefaultPodSecContext: "", + DefaultContainerSecContext: "", }, true, }, @@ -55,27 +59,30 @@ func TestParseFlagsCorrect(t *testing.T) { "--repo-sync-image-pullsecrets= s3", }, server.Config{ - Kubeconfig: "", - APIServerURL: "", - RepoSyncImage: "docker.io/kubeapps/asset-syncer:latest", - RepoSyncImagePullSecrets: []string{"s1", " s2", " s3"}, - ImagePullSecretsRefs: []v1.LocalObjectReference{{Name: "s1"}, {Name: " s2"}, {Name: " s3"}}, - RepoSyncCommand: "/chart-repo", - KubeappsNamespace: "kubeapps", - GlobalPackagingNamespace: "kubeapps", - ReposPerNamespace: true, - DBURL: "localhost", - DBUser: "root", - DBName: "charts", - DBSecretName: "kubeapps-db", - DBSecretKey: "postgresql-root-password", - UserAgentComment: "", - Crontab: "*/10 * * * *", - TTLSecondsAfterFinished: "3600", - CustomAnnotations: []string{""}, - CustomLabels: []string{""}, - ParsedCustomAnnotations: map[string]string{}, - ParsedCustomLabels: map[string]string{}, + Kubeconfig: "", + APIServerURL: "", + RepoSyncImage: "docker.io/kubeapps/asset-syncer:latest", + RepoSyncImagePullSecrets: []string{"s1", " s2", " s3"}, + ImagePullSecretsRefs: []v1.LocalObjectReference{{Name: "s1"}, {Name: " s2"}, {Name: " s3"}}, + RepoSyncCommand: "/chart-repo", + KubeappsNamespace: "kubeapps", + GlobalPackagingNamespace: "kubeapps", + ReposPerNamespace: true, + DBURL: "localhost", + DBUser: "root", + DBName: "charts", + DBSecretName: "kubeapps-db", + DBSecretKey: "postgresql-root-password", + UserAgentComment: "", + Crontab: "*/10 * * * *", + TTLSecondsAfterFinished: "3600", + CustomAnnotations: []string{""}, + CustomLabels: []string{""}, + ParsedCustomAnnotations: map[string]string{}, + ParsedCustomLabels: map[string]string{}, + V1Beta1CronJobs: false, + DefaultPodSecContext: "", + DefaultContainerSecContext: "", }, true, }, @@ -101,29 +108,36 @@ func TestParseFlagsCorrect(t *testing.T) { "--custom-annotations", "foo13=bar13,foo13x=bar13x", "--custom-annotations", "extra13=extra13", "--custom-labels", "foo14=bar14,foo14x=bar14x", + "--ttl-lifetime-afterfinished-job", "1200", + "--v1-beta1-cron-jobs", "true", + "--default-pod-security-context", "{'foo':'bar'}", + "--default-container-security-context", "{'foo':'bar'}", }, server.Config{ - Kubeconfig: "foo01", - APIServerURL: "foo02", - RepoSyncImage: "foo03", - RepoSyncImagePullSecrets: []string{"s1", "s2", "s3"}, - ImagePullSecretsRefs: []v1.LocalObjectReference{{Name: "s1"}, {Name: "s2"}, {Name: "s3"}}, - RepoSyncCommand: "foo04", - KubeappsNamespace: "foo05", - GlobalPackagingNamespace: "kubeapps-repos-global", - ReposPerNamespace: false, - DBURL: "foo06", - DBUser: "foo07", - DBName: "foo08", - DBSecretName: "foo09", - DBSecretKey: "foo10", - UserAgentComment: "foo11", - Crontab: "foo12", - TTLSecondsAfterFinished: "3600", - CustomAnnotations: []string{"foo13=bar13", "foo13x=bar13x", "extra13=extra13"}, - CustomLabels: []string{"foo14=bar14", "foo14x=bar14x"}, - ParsedCustomAnnotations: map[string]string{"foo13": "bar13", "foo13x": "bar13x", "extra13": "extra13"}, - ParsedCustomLabels: map[string]string{"foo14": "bar14", "foo14x": "bar14x"}, + Kubeconfig: "foo01", + APIServerURL: "foo02", + RepoSyncImage: "foo03", + RepoSyncImagePullSecrets: []string{"s1", "s2", "s3"}, + ImagePullSecretsRefs: []v1.LocalObjectReference{{Name: "s1"}, {Name: "s2"}, {Name: "s3"}}, + RepoSyncCommand: "foo04", + KubeappsNamespace: "foo05", + GlobalPackagingNamespace: "kubeapps-repos-global", + ReposPerNamespace: false, + DBURL: "foo06", + DBUser: "foo07", + DBName: "foo08", + DBSecretName: "foo09", + DBSecretKey: "foo10", + UserAgentComment: "foo11", + Crontab: "foo12", + TTLSecondsAfterFinished: "1200", + CustomAnnotations: []string{"foo13=bar13", "foo13x=bar13x", "extra13=extra13"}, + CustomLabels: []string{"foo14=bar14", "foo14x=bar14x"}, + ParsedCustomAnnotations: map[string]string{"foo13": "bar13", "foo13x": "bar13x", "extra13": "extra13"}, + ParsedCustomLabels: map[string]string{"foo14": "bar14", "foo14x": "bar14x"}, + V1Beta1CronJobs: true, + DefaultPodSecContext: "{'foo':'bar'}", + DefaultContainerSecContext: "{'foo':'bar'}", }, true, }, diff --git a/cmd/apprepository-controller/server/controller.go b/cmd/apprepository-controller/server/controller.go index c421cc061ef..0d5e1174c44 100644 --- a/cmd/apprepository-controller/server/controller.go +++ b/cmd/apprepository-controller/server/controller.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "dario.cat/mergo" "github.com/adhocore/gronx" apprepov1alpha1 "github.com/vmware-tanzu/kubeapps/cmd/apprepository-controller/pkg/apis/apprepository/v1alpha1" clientset "github.com/vmware-tanzu/kubeapps/cmd/apprepository-controller/pkg/client/clientset/versioned" @@ -311,7 +312,12 @@ func (c *Controller) syncHandler(key string) error { // TODO: Workaround until the sync jobs are moved to the repoNamespace (#1647) // Delete the cronjob in the Kubeapps namespace to avoid re-syncing the repository - err = c.kubeclientset.BatchV1().CronJobs(c.conf.KubeappsNamespace).Delete(context.TODO(), cronJobName(namespace, name, false), metav1.DeleteOptions{}) + if c.conf.V1Beta1CronJobs { + err = c.kubeclientset.BatchV1beta1().CronJobs(c.conf.KubeappsNamespace).Delete(context.TODO(), cronJobName(namespace, name, false), metav1.DeleteOptions{}) + } else { + err = c.kubeclientset.BatchV1().CronJobs(c.conf.KubeappsNamespace).Delete(context.TODO(), cronJobName(namespace, name, false), metav1.DeleteOptions{}) + } + if err != nil && !errors.IsNotFound(err) { log.Errorf("Unable to delete sync cronjob: %v", err) return err @@ -624,6 +630,18 @@ func syncJobSpec(apprepo *apprepov1alpha1.AppRepository, config Config) batchv1. MountPath: "/usr/local/share/ca-certificates", }) } + + // Fetch the global pod and container security context + podSecContext, err := unmarshallPodSecurityContext(config.DefaultPodSecContext) + if err != nil { + log.Errorf("Unable to unmarshall pod security context configuration %q: %v", config.DefaultPodSecContext, err) + } + + containerSecContext, err := unmarshallContainerSecurityContext(config.DefaultContainerSecContext) + if err != nil { + log.Errorf("Unable to unmarshall container security context configuration %q: %v", config.DefaultContainerSecContext, err) + } + // Get the predefined pod spec for the apprepo definition if exists podTemplateSpec := apprepo.Spec.SyncJobPodTemplate // Add labels @@ -655,6 +673,35 @@ func syncJobSpec(apprepo *apprepov1alpha1.AppRepository, config Config) batchv1. // Add volumes podTemplateSpec.Spec.Volumes = append(podTemplateSpec.Spec.Volumes, volumes...) + // if there is a default security context, try to use it + if podSecContext != nil { + // if the ApRepo CRD does not define any, just use the default, + if podTemplateSpec.Spec.SecurityContext == nil { + podTemplateSpec.Spec.SecurityContext = podSecContext + } else { + // otherwise, try to merge it, allowing the override by the AppRepo CRD's ones. + err = mergo.Merge(podTemplateSpec.Spec.SecurityContext, podSecContext) + if err != nil { + log.Errorf("Unable to merge pod security context %v: %v", podSecContext, err) + } + } + } + + if containerSecContext != nil { + for i := range podTemplateSpec.Spec.Containers { + // if the ApRepo CRD does not define any, just use the default, + if podTemplateSpec.Spec.Containers[i].SecurityContext == nil { + podTemplateSpec.Spec.Containers[i].SecurityContext = containerSecContext + } else { + // otherwise, try to merge it, allowing the override by the AppRepo CRD's ones. + err = mergo.Merge(podTemplateSpec.Spec.Containers[i].SecurityContext, containerSecContext) + if err != nil { + log.Errorf("Unable to merge container security context %v: %v", containerSecContext, err) + } + } + } + } + return batchv1.JobSpec{ TTLSecondsAfterFinished: ttlLifetimeJobs(config), Template: podTemplateSpec, @@ -677,7 +724,7 @@ func newCleanupJob(kubeappsNamespace, repoNamespace, name string, config Config) // cleanupJobSpec returns a batchv1.JobSpec for running the chart-repo delete job func cleanupJobSpec(namespace, name string, config Config) batchv1.JobSpec { - return batchv1.JobSpec{ + cleanupJobSpec := batchv1.JobSpec{ TTLSecondsAfterFinished: ttlLifetimeJobs(config), Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ @@ -707,6 +754,17 @@ func cleanupJobSpec(namespace, name string, config Config) batchv1.JobSpec { }, }, } + + // Add the global pod and container security context as we don't have any AppRepo CR from which we can read + if podSecContext, err := unmarshallPodSecurityContext(config.DefaultPodSecContext); err == nil && podSecContext != nil { + cleanupJobSpec.Template.Spec.SecurityContext = podSecContext + } + + if containerSecContext, err := unmarshallContainerSecurityContext(config.DefaultContainerSecContext); err == nil && containerSecContext != nil { + cleanupJobSpec.Template.Spec.Containers[0].SecurityContext = containerSecContext + } + + return cleanupJobSpec } // jobLabels returns the labels for the job and cronjob resources @@ -887,3 +945,31 @@ func dbFlags(config Config) []string { "--database-name=" + config.DBName, } } + +func unmarshallPodSecurityContext(str string) (*corev1.PodSecurityContext, error) { + if str == "" { + return nil, nil + } + + var secContext *corev1.PodSecurityContext + err := json.Unmarshal([]byte(str), &secContext) + if err != nil { + return nil, err + } + + return secContext, err +} + +func unmarshallContainerSecurityContext(str string) (*corev1.SecurityContext, error) { + if str == "" { + return nil, nil + } + + var secContext *corev1.SecurityContext + err := json.Unmarshal([]byte(str), &secContext) + if err != nil { + return nil, err + } + + return secContext, err +} diff --git a/cmd/apprepository-controller/server/controller_test.go b/cmd/apprepository-controller/server/controller_test.go index e2b5722575a..0c1792cd2e2 100644 --- a/cmd/apprepository-controller/server/controller_test.go +++ b/cmd/apprepository-controller/server/controller_test.go @@ -21,16 +21,18 @@ var defaultTTL = int32(3600) func Test_newCronJob(t *testing.T) { tests := []struct { - name string - crontab string - userAgentComment string - apprepo *apprepov1alpha1.AppRepository - expected batchv1.CronJob + name string + crontab string + userAgentComment string + defaultSecContext []string + apprepo *apprepov1alpha1.AppRepository + expected batchv1.CronJob }{ { "my-charts", "*/10 * * * *", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -122,6 +124,7 @@ func Test_newCronJob(t *testing.T) { "my-charts with long names", "*/10 * * * *", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -204,6 +207,7 @@ func Test_newCronJob(t *testing.T) { "my-charts with auth, userAgent and crontab configuration", "*/20 * * * *", "kubeapps/v2.3", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -305,6 +309,7 @@ func Test_newCronJob(t *testing.T) { "a cronjob for an app repo in another namespace references the repo secret in kubeapps", "*/20 * * * *", "kubeapps/v2.3", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -397,6 +402,7 @@ func Test_newCronJob(t *testing.T) { "my-charts with custom (good) interval", "*/10 * * * *", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -489,6 +495,7 @@ func Test_newCronJob(t *testing.T) { "my-charts with custom (good) crontab", "*/10 * * * *", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -584,9 +591,11 @@ func Test_newCronJob(t *testing.T) { config := makeDefaultConfig() config.Crontab = tt.crontab config.UserAgentComment = tt.userAgentComment + config.DefaultPodSecContext = tt.defaultSecContext[0] + config.DefaultContainerSecContext = tt.defaultSecContext[1] result := newCronJob(tt.apprepo, config) - if got, want := tt.expected, *result; !cmp.Equal(want, got) { + if got, want := *result, tt.expected; !cmp.Equal(want, got) { t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(want, got)) } }) @@ -599,9 +608,11 @@ func Test_newCronJob(t *testing.T) { config.V1Beta1CronJobs = true config.Crontab = tt.crontab config.UserAgentComment = tt.userAgentComment + config.DefaultPodSecContext = tt.defaultSecContext[0] + config.DefaultContainerSecContext = tt.defaultSecContext[1] result := newCronJob(tt.apprepo, config) - if got, want := tt.expected, *result; !cmp.Equal(want, got) { + if got, want := *result, tt.expected; !cmp.Equal(want, got) { t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(want, got)) } }) @@ -610,14 +621,193 @@ func Test_newCronJob(t *testing.T) { func Test_newSyncJob(t *testing.T) { tests := []struct { - name string - userAgentComment string - apprepo *apprepov1alpha1.AppRepository - expected batchv1.Job + name string + userAgentComment string + defaultSecContext []string + apprepo *apprepov1alpha1.AppRepository + expected batchv1.Job }{ { "my-charts", "", + []string{"", ""}, + &apprepov1alpha1.AppRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: "AppRepository", + APIVersion: "kubeapps.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-charts", + Namespace: "kubeapps", + Labels: map[string]string{ + "name": "my-charts", + "created-by": "kubeapps", + }, + }, + Spec: apprepov1alpha1.AppRepositorySpec{ + Type: "helm", + URL: "https://charts.acme.com/my-charts", + }, + }, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "apprepo-kubeapps-sync-my-charts-", + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef( + &apprepov1alpha1.AppRepository{ObjectMeta: metav1.ObjectMeta{Name: "my-charts"}}, + schema.GroupVersionKind{ + Group: apprepov1alpha1.SchemeGroupVersion.Group, + Version: apprepov1alpha1.SchemeGroupVersion.Version, + Kind: "AppRepository", + }, + ), + }, + Annotations: map[string]string{}, + Labels: map[string]string{}, + }, + Spec: batchv1.JobSpec{ + TTLSecondsAfterFinished: &defaultTTL, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + LabelRepoName: "my-charts", + LabelRepoNamespace: "kubeapps", + }, + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{ + { + Name: "sync", + Image: repoSyncImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/chart-repo"}, + Args: []string{ + "sync", + "--database-url=postgresql.kubeapps", + "--database-user=admin", + "--database-name=assets", + "--global-repos-namespace=kubeapps-global", + "--namespace=kubeapps", + "my-charts", + "https://charts.acme.com/my-charts", + "helm", + }, + Env: []corev1.EnvVar{ + { + Name: "DB_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "postgresql"}, Key: "postgresql-root-password"}}, + }, + }, + VolumeMounts: nil, + }, + }, + Volumes: nil, + }, + }, + }, + }, + }, + { + "my-charts with default sec policy", + "", + []string{`{"enabled":true,"fsGroup":10001}`, `{"allowPrivilegeEscalation":false,"enabled":true,"readOnlyRootFilesystem":true,"runAsGroup":10001,"runAsNonRoot":true,"runAsUser":1001}`}, + &apprepov1alpha1.AppRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: "AppRepository", + APIVersion: "kubeapps.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-charts", + Namespace: "kubeapps", + Labels: map[string]string{ + "name": "my-charts", + "created-by": "kubeapps", + }, + }, + Spec: apprepov1alpha1.AppRepositorySpec{ + Type: "helm", + URL: "https://charts.acme.com/my-charts", + }, + }, + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "apprepo-kubeapps-sync-my-charts-", + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef( + &apprepov1alpha1.AppRepository{ObjectMeta: metav1.ObjectMeta{Name: "my-charts"}}, + schema.GroupVersionKind{ + Group: apprepov1alpha1.SchemeGroupVersion.Group, + Version: apprepov1alpha1.SchemeGroupVersion.Version, + Kind: "AppRepository", + }, + ), + }, + Annotations: map[string]string{}, + Labels: map[string]string{}, + }, + Spec: batchv1.JobSpec{ + TTLSecondsAfterFinished: &defaultTTL, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + LabelRepoName: "my-charts", + LabelRepoNamespace: "kubeapps", + }, + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: func(i int64) *int64 { return &i }(10001), + }, + + Containers: []corev1.Container{ + { + Name: "sync", + Image: repoSyncImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/chart-repo"}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: func(i bool) *bool { return &i }(false), + ReadOnlyRootFilesystem: func(i bool) *bool { return &i }(true), + RunAsGroup: func(i int64) *int64 { return &i }(10001), + RunAsNonRoot: func(i bool) *bool { return &i }(true), + RunAsUser: func(i int64) *int64 { return &i }(1001), + }, + Args: []string{ + "sync", + "--database-url=postgresql.kubeapps", + "--database-user=admin", + "--database-name=assets", + "--global-repos-namespace=kubeapps-global", + "--namespace=kubeapps", + "my-charts", + "https://charts.acme.com/my-charts", + "helm", + }, + Env: []corev1.EnvVar{ + { + Name: "DB_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "postgresql"}, Key: "postgresql-root-password"}}, + }, + }, + VolumeMounts: nil, + }, + }, + Volumes: nil, + }, + }, + }, + }, + }, + { + "my-charts with default + crd sec policy", + "", + []string{`{"enabled":true,"fsGroup":10001}`, `{"allowPrivilegeEscalation":false,"enabled":true,"readOnlyRootFilesystem":true,"runAsGroup":10001,"runAsNonRoot":true,"runAsUser":1001}`}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -634,6 +824,18 @@ func Test_newSyncJob(t *testing.T) { Spec: apprepov1alpha1.AppRepositorySpec{ Type: "helm", URL: "https://charts.acme.com/my-charts", + SyncJobPodTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: func(i int64) *int64 { return &i }(1234), + }, + Containers: []corev1.Container{ + {SecurityContext: &corev1.SecurityContext{ + Privileged: func(i bool) *bool { return &i }(true), + }}, + }, + }, + }, }, }, batchv1.Job{ @@ -664,12 +866,24 @@ func Test_newSyncJob(t *testing.T) { }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: func(i int64) *int64 { return &i }(1234), + }, + Containers: []corev1.Container{ { Name: "sync", Image: repoSyncImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/chart-repo"}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: func(i bool) *bool { return &i }(false), + ReadOnlyRootFilesystem: func(i bool) *bool { return &i }(true), + RunAsGroup: func(i int64) *int64 { return &i }(10001), + RunAsNonRoot: func(i bool) *bool { return &i }(true), + RunAsUser: func(i int64) *int64 { return &i }(1001), + Privileged: func(i bool) *bool { return &i }(true), + }, Args: []string{ "sync", "--database-url=postgresql.kubeapps", @@ -700,6 +914,7 @@ func Test_newSyncJob(t *testing.T) { { "my-charts with long names", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -772,6 +987,7 @@ func Test_newSyncJob(t *testing.T) { { "an app repository in another namespace results in jobs without owner references", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -844,6 +1060,7 @@ func Test_newSyncJob(t *testing.T) { { "my-charts with auth and userAgent comment", "kubeapps/v2.3", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -936,6 +1153,7 @@ func Test_newSyncJob(t *testing.T) { { "my-charts with a customCA", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -1037,6 +1255,7 @@ func Test_newSyncJob(t *testing.T) { { "my-charts with a customCA and auth header", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -1146,6 +1365,7 @@ func Test_newSyncJob(t *testing.T) { { "my-charts linked to docker registry creds", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -1236,6 +1456,7 @@ func Test_newSyncJob(t *testing.T) { { "my-charts with a custom pod template", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -1341,6 +1562,7 @@ func Test_newSyncJob(t *testing.T) { { "OCI registry with repositories", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -1426,6 +1648,7 @@ func Test_newSyncJob(t *testing.T) { { "Skip TLS verification", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -1513,6 +1736,7 @@ func Test_newSyncJob(t *testing.T) { { "Paas credentials", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -1600,6 +1824,7 @@ func Test_newSyncJob(t *testing.T) { { "Repository with filters", "", + []string{"", ""}, &apprepov1alpha1.AppRepository{ TypeMeta: metav1.TypeMeta{ Kind: "AppRepository", @@ -1690,9 +1915,11 @@ func Test_newSyncJob(t *testing.T) { t.Run(tt.name, func(t *testing.T) { config := makeDefaultConfig() config.UserAgentComment = tt.userAgentComment + config.DefaultPodSecContext = tt.defaultSecContext[0] + config.DefaultContainerSecContext = tt.defaultSecContext[1] result := newSyncJob(tt.apprepo, config) - if got, want := tt.expected, *result; !cmp.Equal(want, got) { + if got, want := *result, tt.expected; !cmp.Equal(want, got) { t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(want, got)) } }) @@ -1701,17 +1928,68 @@ func Test_newSyncJob(t *testing.T) { func Test_newCleanupJob(t *testing.T) { tests := []struct { - name string - kubeappsNamespace string - repoName string - repoNamespace string - expected batchv1.Job + name string + kubeappsNamespace string + repoName string + repoNamespace string + podSecContext string + containerSecContext string + expected batchv1.Job }{ { "my-charts with", "kubeapps", "my-charts", "kubeapps", + "", + "", + batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "apprepo-kubeapps-cleanup-my-charts-", + Namespace: "kubeapps", + Annotations: map[string]string{}, + Labels: map[string]string{}, + }, + Spec: batchv1.JobSpec{ + TTLSecondsAfterFinished: &defaultTTL, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "delete", + Image: repoSyncImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/chart-repo"}, + Args: []string{ + "delete", + "my-charts", + "--namespace=kubeapps", + "--database-url=postgresql.kubeapps", + "--database-user=admin", + "--database-name=assets", + }, + Env: []corev1.EnvVar{ + { + Name: "DB_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "postgresql"}, Key: "postgresql-root-password"}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + "my-charts with security context", + "kubeapps", + "my-charts", + "kubeapps", + `{"enabled":true,"fsGroup":10001}`, + `{"allowPrivilegeEscalation":false,"enabled":true,"readOnlyRootFilesystem":true,"runAsGroup":10001,"runAsNonRoot":true,"runAsUser":1001}`, batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "apprepo-kubeapps-cleanup-my-charts-", @@ -1724,12 +2002,22 @@ func Test_newCleanupJob(t *testing.T) { Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: func(i int64) *int64 { return &i }(10001), + }, Containers: []corev1.Container{ { Name: "delete", Image: repoSyncImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/chart-repo"}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: func(i bool) *bool { return &i }(false), + ReadOnlyRootFilesystem: func(i bool) *bool { return &i }(true), + RunAsGroup: func(i int64) *int64 { return &i }(10001), + RunAsNonRoot: func(i bool) *bool { return &i }(true), + RunAsUser: func(i int64) *int64 { return &i }(1001), + }, Args: []string{ "delete", "my-charts", @@ -1757,6 +2045,8 @@ func Test_newCleanupJob(t *testing.T) { "kubeapps", "a-really-long-long-long-long-but-valid-name-under-63-characters", "a-really-long-long-long-but-valid-namespace-under-63-characters", + "", + "", batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "apprepo-a-real-1762006330-cleanup-a-real-1687295243-", @@ -1801,8 +2091,12 @@ func Test_newCleanupJob(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := newCleanupJob(tt.kubeappsNamespace, tt.repoNamespace, tt.repoName, makeDefaultConfig()) - if got, want := tt.expected, *result; !cmp.Equal(want, got) { + config := makeDefaultConfig() + config.DefaultContainerSecContext = tt.containerSecContext + config.DefaultPodSecContext = tt.podSecContext + + result := newCleanupJob(tt.kubeappsNamespace, tt.repoNamespace, tt.repoName, config) + if got, want := *result, tt.expected; !cmp.Equal(want, got) { t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(want, got)) } }) @@ -2160,25 +2454,29 @@ func TestIntervalToCron(t *testing.T) { func makeDefaultConfig() Config { return Config{ - Kubeconfig: "", - APIServerURL: "", - RepoSyncImage: repoSyncImage, - RepoSyncImagePullSecrets: []string{}, - RepoSyncCommand: "/chart-repo", - KubeappsNamespace: "kubeapps", - GlobalPackagingNamespace: "kubeapps-global", - ReposPerNamespace: true, - DBURL: "postgresql.kubeapps", - DBUser: "admin", - DBName: "assets", - DBSecretName: "postgresql", - DBSecretKey: "postgresql-root-password", - UserAgentComment: "", - TTLSecondsAfterFinished: "3600", - Crontab: "*/10 * * * *", - CustomAnnotations: []string{}, - CustomLabels: []string{}, - ParsedCustomLabels: map[string]string{}, - ParsedCustomAnnotations: map[string]string{}, + Kubeconfig: "", + APIServerURL: "", + RepoSyncImage: repoSyncImage, + RepoSyncImagePullSecrets: []string{"", ""}, + RepoSyncCommand: "/chart-repo", + KubeappsNamespace: "kubeapps", + GlobalPackagingNamespace: "kubeapps-global", + ReposPerNamespace: true, + DBURL: "postgresql.kubeapps", + DBUser: "admin", + DBName: "assets", + DBSecretName: "postgresql", + DBSecretKey: "postgresql-root-password", + UserAgentComment: "", + TTLSecondsAfterFinished: "3600", + Crontab: "*/10 * * * *", + CustomAnnotations: []string{"", ""}, + CustomLabels: []string{"", ""}, + ParsedCustomLabels: map[string]string{}, + ParsedCustomAnnotations: map[string]string{}, + ImagePullSecretsRefs: nil, + V1Beta1CronJobs: false, + DefaultPodSecContext: "", + DefaultContainerSecContext: "", } } diff --git a/cmd/apprepository-controller/server/server.go b/cmd/apprepository-controller/server/server.go index 9c2cc6d2162..ca814125701 100644 --- a/cmd/apprepository-controller/server/server.go +++ b/cmd/apprepository-controller/server/server.go @@ -16,28 +16,30 @@ import ( ) type Config struct { - APIServerURL string - Kubeconfig string - RepoSyncImage string - RepoSyncImagePullSecrets []string - ImagePullSecretsRefs []corev1.LocalObjectReference - RepoSyncCommand string - KubeappsNamespace string - GlobalPackagingNamespace string - DBURL string - DBUser string - DBName string - DBSecretName string - DBSecretKey string - UserAgentComment string - Crontab string - TTLSecondsAfterFinished string - ReposPerNamespace bool - CustomAnnotations []string - CustomLabels []string - ParsedCustomAnnotations map[string]string - ParsedCustomLabels map[string]string - V1Beta1CronJobs bool + APIServerURL string + Kubeconfig string + RepoSyncImage string + RepoSyncImagePullSecrets []string + ImagePullSecretsRefs []corev1.LocalObjectReference + RepoSyncCommand string + KubeappsNamespace string + GlobalPackagingNamespace string + DBURL string + DBUser string + DBName string + DBSecretName string + DBSecretKey string + UserAgentComment string + Crontab string + TTLSecondsAfterFinished string + ReposPerNamespace bool + CustomAnnotations []string + CustomLabels []string + ParsedCustomAnnotations map[string]string + ParsedCustomLabels map[string]string + V1Beta1CronJobs bool + DefaultPodSecContext string + DefaultContainerSecContext string } func Serve(serveOpts Config) error { diff --git a/go.mod b/go.mod index 71b38d4965a..918667d74a8 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ module github.com/vmware-tanzu/kubeapps go 1.20 require ( + dario.cat/mergo v1.0.0 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/adhocore/gronx v1.6.5 diff --git a/go.sum b/go.sum index 7db6f98e936..e6ba018de65 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=