From 81300f98ea3381859f4bc7b779902388d8b8ecf7 Mon Sep 17 00:00:00 2001 From: jiangkaihua Date: Fri, 26 Aug 2022 15:28:06 +0800 Subject: [PATCH] Update CA insert method in webhooks. Signed-off-by: jiangkaihua --- cmd/webhook-manager/app/options/options.go | 7 +++-- cmd/webhook-manager/app/server.go | 13 ++++++--- cmd/webhook-manager/app/util.go | 26 +++++++---------- .../chart/volcano/templates/admission.yaml | 3 +- .../chart/volcano/templates/webhooks.yaml | 16 +++++------ installer/helm/chart/volcano/values.yaml | 15 +++++----- installer/volcano-development.yaml | 1 + pkg/webhooks/router/admission.go | 28 +++++++++++++++++-- 8 files changed, 67 insertions(+), 42 deletions(-) diff --git a/cmd/webhook-manager/app/options/options.go b/cmd/webhook-manager/app/options/options.go index b0ece916283..d7466978a93 100644 --- a/cmd/webhook-manager/app/options/options.go +++ b/cmd/webhook-manager/app/options/options.go @@ -29,10 +29,11 @@ const ( defaultSchedulerName = "volcano" defaultQPS = 50.0 defaultBurst = 100 - defaultEnabledAdmission = "/jobs/mutate,/jobs/validate,/podgroups/mutate,/pods/validate,/pods/mutate,/queues/mutate,/queues/validate" defaultIgnoredNamespaces = "volcano-system,kube-system" ) +var defaultEnabledAdmission = map[string]string{"/jobs/mutate": "true", "/jobs/validate": "true", "/podgroups/mutate": "true", "/pods/validate": "true", "/pods/mutate": "true", "/queues/mutate": "true", "/queues/validate": "true"} + // Config admission-controller server config. type Config struct { KubeClientOptions kube.ClientOptions @@ -50,7 +51,7 @@ type Config struct { SchedulerNames []string WebhookURL string ConfigPath string - EnabledAdmission string + EnabledAdmission map[string]string IgnoredNamespaces string } @@ -79,7 +80,7 @@ func (c *Config) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&c.WebhookNamespace, "webhook-namespace", "", "The namespace of this webhook") fs.StringVar(&c.WebhookName, "webhook-service-name", "", "The name of this webhook") fs.StringVar(&c.WebhookURL, "webhook-url", "", "The url of this webhook") - fs.StringVar(&c.EnabledAdmission, "enabled-admission", defaultEnabledAdmission, "enabled admission webhooks, if this parameter is modified, make sure corresponding webhook configurations are the same.") + fs.StringToStringVar(&c.EnabledAdmission, "enabled-admission", defaultEnabledAdmission, "enabled admission webhooks, if this parameter is modified, make sure corresponding webhook configurations are the same.") fs.StringArrayVar(&c.SchedulerNames, "scheduler-name", []string{defaultSchedulerName}, "Volcano will handle pods whose .spec.SchedulerName is same as scheduler-name") fs.StringVar(&c.ConfigPath, "admission-conf", "", "The configmap file of this webhook") fs.StringVar(&c.IgnoredNamespaces, "ignored-namespaces", defaultIgnoredNamespaces, "Comma-separated list of namespaces to be ignored by admission webhooks") diff --git a/cmd/webhook-manager/app/server.go b/cmd/webhook-manager/app/server.go index 7527e1fdc40..7a2e92dd0a2 100644 --- a/cmd/webhook-manager/app/server.go +++ b/cmd/webhook-manager/app/server.go @@ -67,7 +67,7 @@ func Run(config *options.Config) error { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: commonutil.GenerateComponentName(config.SchedulerNames)}) - router.ForEachAdmission(config, func(service *router.AdmissionService) { + if err := router.ForEachAdmission(config, func(service *router.AdmissionService) error { if service.Config != nil { service.Config.VolcanoClient = vClient service.Config.KubeClient = kubeClient @@ -78,11 +78,16 @@ func Run(config *options.Config) error { klog.V(3).Infof("Registered '%s' as webhook.", service.Path) http.HandleFunc(service.Path, service.Handler) - }) - if err = addCaCertForWebhook(kubeClient, config.CaCertData); err != nil { - return fmt.Errorf("failed to add caCert for webhook %v", err) + klog.V(3).Infof("Add CaCert for webhook <%s>", service.Path) + if err = addCaCertForWebhook(kubeClient, service, config.CaCertData); err != nil { + return fmt.Errorf("failed to add caCert for webhook %v", err) + } + return nil + }); err != nil { + return err } + klog.V(3).Infof("Successfully added caCert for all webhooks") webhookServeError := make(chan struct{}) diff --git a/cmd/webhook-manager/app/util.go b/cmd/webhook-manager/app/util.go index a8361d3558f..72664680801 100644 --- a/cmd/webhook-manager/app/util.go +++ b/cmd/webhook-manager/app/util.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "strings" "time" v1 "k8s.io/api/admissionregistration/v1" @@ -34,24 +35,15 @@ import ( "volcano.sh/apis/pkg/client/clientset/versioned" "volcano.sh/volcano/cmd/webhook-manager/app/options" + "volcano.sh/volcano/pkg/webhooks/router" ) -var ( - validatingWebhooksName = []string{ - "volcano-admission-service-jobs-validate", - "volcano-admission-service-pods-validate", - "volcano-admission-service-queues-validate", - } - mutatingWebhooksName = []string{ - "volcano-admission-service-pods-mutate", - "volcano-admission-service-queues-mutate", - "volcano-admission-service-podgroups-mutate", - "volcano-admission-service-jobs-mutate", - } -) +const volcanoAdmissionPrefix = "volcano-admission-service" -func addCaCertForWebhook(kubeClient *kubernetes.Clientset, caBundle []byte) error { - for _, mutatingWebhookName := range mutatingWebhooksName { +func addCaCertForWebhook(kubeClient *kubernetes.Clientset, service *router.AdmissionService, caBundle []byte) error { + if service.MutatingConfig != nil { + // update MutatingWebhookConfigurations + var mutatingWebhookName = volcanoAdmissionPrefix + strings.ReplaceAll(service.Path, "/", "-") var mutatingWebhook *v1.MutatingWebhookConfiguration webhookChanged := false if err := wait.Poll(time.Second, 5*time.Minute, func() (done bool, err error) { @@ -82,7 +74,9 @@ func addCaCertForWebhook(kubeClient *kubernetes.Clientset, caBundle []byte) erro } } - for _, validatingWebhookName := range validatingWebhooksName { + if service.ValidatingConfig != nil { + // update ValidatingWebhookConfigurations + var validatingWebhookName = volcanoAdmissionPrefix + strings.ReplaceAll(service.Path, "/", "-") var validatingWebhook *v1.ValidatingWebhookConfiguration webhookChanged := false if err := wait.Poll(time.Second, 5*time.Minute, func() (done bool, err error) { diff --git a/installer/helm/chart/volcano/templates/admission.yaml b/installer/helm/chart/volcano/templates/admission.yaml index 50256f61d69..784aed49a1e 100644 --- a/installer/helm/chart/volcano/templates/admission.yaml +++ b/installer/helm/chart/volcano/templates/admission.yaml @@ -84,6 +84,7 @@ spec: {{- end }} containers: - args: + - --enabled-admission={{ .Values.custom.enabled_admissions }} - --tls-cert-file=/admission.local.config/certificates/tls.crt - --tls-private-key-file=/admission.local.config/certificates/tls.key - --ca-cert-file=/admission.local.config/certificates/ca.crt @@ -154,4 +155,4 @@ spec: imagePullPolicy: IfNotPresent command: ["./gen-admission-secret.sh", "--service", "{{ .Release.Name }}-admission-service", "--namespace", "{{ .Release.Namespace }}", "--secret", "{{.Values.basic.admission_secret_name}}"] -{{- end }} \ No newline at end of file +{{- end }} diff --git a/installer/helm/chart/volcano/templates/webhooks.yaml b/installer/helm/chart/volcano/templates/webhooks.yaml index 1a292115e1a..63c0b33a89f 100644 --- a/installer/helm/chart/volcano/templates/webhooks.yaml +++ b/installer/helm/chart/volcano/templates/webhooks.yaml @@ -1,6 +1,6 @@ {{- if .Values.custom.admission_enable }} -{{- if .Values.custom.pods_mutatingwebhook_enable }} +{{- if .Values.custom.enabled_admissions.pods_mutate_enable }} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: @@ -42,7 +42,7 @@ webhooks: --- -{{- if .Values.custom.queues_mutatingwebhook_enable }} +{{- if .Values.custom.enabled_admissions.queues_mutate_enable }} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: @@ -84,7 +84,7 @@ webhooks: --- -{{- if .Values.custom.podgroups_mutatingwebhook_enable }} +{{- if .Values.custom.enabled_admissions.podgroups_mutate_enable }} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: @@ -126,7 +126,7 @@ webhooks: --- -{{- if .Values.custom.jobs_mutatingwebhook_enable }} +{{- if .Values.custom.enabled_admissions.jobs_mutate_enable }} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: @@ -168,7 +168,7 @@ webhooks: --- -{{- if .Values.custom.jobs_validatingwebhook_enable }} +{{- if .Values.custom.enabled_admissions.jobs_validate_enable }} apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -210,7 +210,7 @@ webhooks: --- -{{- if .Values.custom.pods_validatingwebhook_enable }} +{{- if .Values.custom.enabled_admissions.pods_validate_enable }} apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -251,7 +251,7 @@ webhooks: --- -{{- if .Values.custom.queues_validatingwebhook_enable }} +{{- if .Values.custom.enabled_admissions.queues_validate_enable }} apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -291,4 +291,4 @@ webhooks: sideEffects: NoneOnDryRun timeoutSeconds: 10 {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/installer/helm/chart/volcano/values.yaml b/installer/helm/chart/volcano/values.yaml index 8bdb067a538..c2d4a7fa51c 100644 --- a/installer/helm/chart/volcano/values.yaml +++ b/installer/helm/chart/volcano/values.yaml @@ -13,10 +13,11 @@ custom: admission_enable: true controller_enable: true scheduler_enable: true - pods_mutatingwebhook_enable: true - queues_mutatingwebhook_enable: true - podgroups_mutatingwebhook_enable: true - jobs_mutatingwebhook_enable: true - jobs_validatingwebhook_enable: true - pods_validatingwebhook_enable: true - queues_validatingwebhook_enable: true \ No newline at end of file + enabled_admissions: + - pods_mutate_enable: true + - queues_mutate_enable: true + - podgroups_mutate_enable: true + - jobs_mutate_enable: true + - jobs_validate_enable: true + - pods_validate_enable: true + - queues_validate_enable: true diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index aac32554967..94dc0c65513 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -134,6 +134,7 @@ spec: priorityClassName: system-cluster-critical containers: - args: + - --enabled-admission=map[jobs_mutate_enable:true jobs_validate_enable:true podgroups_mutate_enable:true pods_mutate_enable:true pods_validate_enable:true queues_mutate_enable:true queues_validate_enable:true] - --tls-cert-file=/admission.local.config/certificates/tls.crt - --tls-private-key-file=/admission.local.config/certificates/tls.key - --ca-cert-file=/admission.local.config/certificates/ca.crt diff --git a/pkg/webhooks/router/admission.go b/pkg/webhooks/router/admission.go index 03426cdfe0a..6df842a4190 100644 --- a/pkg/webhooks/router/admission.go +++ b/pkg/webhooks/router/admission.go @@ -19,9 +19,12 @@ package router import ( "fmt" "net/http" + "strconv" "strings" "sync" + "k8s.io/klog" + "volcano.sh/volcano/cmd/webhook-manager/app/options" ) @@ -48,11 +51,30 @@ func RegisterAdmission(service *AdmissionService) error { return nil } -func ForEachAdmission(config *options.Config, handler func(*AdmissionService)) { - admissions := strings.Split(strings.TrimSpace(config.EnabledAdmission), ",") +func ForEachAdmission(config *options.Config, handler func(*AdmissionService) error) error { + admissions := getEnabledAdmissionFromConfig(config.EnabledAdmission) + klog.V(3).Infof("Enabled admissions are: %v, registered map are: %v", admissions, admissionMap) for _, admission := range admissions { if service, found := admissionMap[admission]; found { - handler(service) + if err := handler(service); err != nil { + return err + } + } + } + return nil +} + +func getEnabledAdmissionFromConfig(enabledAdmissions map[string]string) []string { + var admissions = make([]string, 0) + for key, value := range enabledAdmissions { + enabled, err := strconv.ParseBool(value) + if err != nil { + klog.Warningf("invalid config for arg enabled-key: <%s:%s>, error: %s", key, value, err.Error()) + continue + } + if enabled { + admissions = append(admissions, "/"+strings.ReplaceAll(strings.TrimSuffix(key, "_enable"), "_", "/")) } } + return admissions }