Update CA insert method in webhooks.

Signed-off-by: jiangkaihua <jiangkaihua1@huawei.com>

cmd/webhook-manager/app/options/options.go

@@ -29,10 +29,11 @@ const (
 	defaultSchedulerName     = "volcano"
 	defaultQPS               = 50.0
 	defaultBurst             = 100
-	defaultEnabledAdmission  = "/jobs/mutate,/jobs/validate,/podgroups/mutate,/pods/validate,/pods/mutate,/queues/mutate,/queues/validate"
 	defaultIgnoredNamespaces = "volcano-system,kube-system"
 )
 
+var defaultEnabledAdmission = map[string]string{"/jobs/mutate": "true", "/jobs/validate": "true", "/podgroups/mutate": "true", "/pods/validate": "true", "/pods/mutate": "true", "/queues/mutate": "true", "/queues/validate": "true"}
+
 // Config admission-controller server config.
 type Config struct {
 	KubeClientOptions kube.ClientOptions
@@ -50,7 +51,7 @@ type Config struct {
 	SchedulerNames    []string
 	WebhookURL        string
 	ConfigPath        string
-	EnabledAdmission  string
+	EnabledAdmission  map[string]string
 	IgnoredNamespaces string
 }
 
@@ -79,7 +80,7 @@ func (c *Config) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&c.WebhookNamespace, "webhook-namespace", "", "The namespace of this webhook")
 	fs.StringVar(&c.WebhookName, "webhook-service-name", "", "The name of this webhook")
 	fs.StringVar(&c.WebhookURL, "webhook-url", "", "The url of this webhook")
-	fs.StringVar(&c.EnabledAdmission, "enabled-admission", defaultEnabledAdmission, "enabled admission webhooks, if this parameter is modified, make sure corresponding webhook configurations are the same.")
+	fs.StringToStringVar(&c.EnabledAdmission, "enabled-admission", defaultEnabledAdmission, "enabled admission webhooks, if this parameter is modified, make sure corresponding webhook configurations are the same.")
 	fs.StringArrayVar(&c.SchedulerNames, "scheduler-name", []string{defaultSchedulerName}, "Volcano will handle pods whose .spec.SchedulerName is same as scheduler-name")
 	fs.StringVar(&c.ConfigPath, "admission-conf", "", "The configmap file of this webhook")
 	fs.StringVar(&c.IgnoredNamespaces, "ignored-namespaces", defaultIgnoredNamespaces, "Comma-separated list of namespaces to be ignored by admission webhooks")

cmd/webhook-manager/app/server.go

@@ -67,7 +67,7 @@ func Run(config *options.Config) error {
 	broadcaster := record.NewBroadcaster()
 	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
 	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: commonutil.GenerateComponentName(config.SchedulerNames)})
-	router.ForEachAdmission(config, func(service *router.AdmissionService) {
+	if err := router.ForEachAdmission(config, func(service *router.AdmissionService) error {
 		if service.Config != nil {
 			service.Config.VolcanoClient = vClient
 			service.Config.KubeClient = kubeClient
@@ -78,11 +78,16 @@ func Run(config *options.Config) error {
 
 		klog.V(3).Infof("Registered '%s' as webhook.", service.Path)
 		http.HandleFunc(service.Path, service.Handler)
-	})
 
-	if err = addCaCertForWebhook(kubeClient, config.CaCertData); err != nil {
-		return fmt.Errorf("failed to add caCert for webhook %v", err)
+		klog.V(3).Infof("Add CaCert for webhook <%s>", service.Path)
+		if err = addCaCertForWebhook(kubeClient, service, config.CaCertData); err != nil {
+			return fmt.Errorf("failed to add caCert for webhook %v", err)
+		}
+		return nil
+	}); err != nil {
+		return err
 	}
+
 	klog.V(3).Infof("Successfully added caCert for all webhooks")
 
 	webhookServeError := make(chan struct{})

cmd/webhook-manager/app/util.go

@@ -22,6 +22,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"strings"
 	"time"
 
 	v1 "k8s.io/api/admissionregistration/v1"
@@ -34,24 +35,15 @@ import (
 
 	"volcano.sh/apis/pkg/client/clientset/versioned"
 	"volcano.sh/volcano/cmd/webhook-manager/app/options"
+	"volcano.sh/volcano/pkg/webhooks/router"
 )
 
-var (
-	validatingWebhooksName = []string{
-		"volcano-admission-service-jobs-validate",
-		"volcano-admission-service-pods-validate",
-		"volcano-admission-service-queues-validate",
-	}
-	mutatingWebhooksName = []string{
-		"volcano-admission-service-pods-mutate",
-		"volcano-admission-service-queues-mutate",
-		"volcano-admission-service-podgroups-mutate",
-		"volcano-admission-service-jobs-mutate",
-	}
-)
+const volcanoAdmissionPrefix = "volcano-admission-service"
 
-func addCaCertForWebhook(kubeClient *kubernetes.Clientset, caBundle []byte) error {
-	for _, mutatingWebhookName := range mutatingWebhooksName {
+func addCaCertForWebhook(kubeClient *kubernetes.Clientset, service *router.AdmissionService, caBundle []byte) error {
+	if service.MutatingConfig != nil {
+		// update MutatingWebhookConfigurations
+		var mutatingWebhookName = volcanoAdmissionPrefix + strings.ReplaceAll(service.Path, "/", "-")
 		var mutatingWebhook *v1.MutatingWebhookConfiguration
 		webhookChanged := false
 		if err := wait.Poll(time.Second, 5*time.Minute, func() (done bool, err error) {
@@ -82,7 +74,9 @@ func addCaCertForWebhook(kubeClient *kubernetes.Clientset, caBundle []byte) erro
 		}
 	}
 
-	for _, validatingWebhookName := range validatingWebhooksName {
+	if service.ValidatingConfig != nil {
+		// update ValidatingWebhookConfigurations
+		var validatingWebhookName = volcanoAdmissionPrefix + strings.ReplaceAll(service.Path, "/", "-")
 		var validatingWebhook *v1.ValidatingWebhookConfiguration
 		webhookChanged := false
 		if err := wait.Poll(time.Second, 5*time.Minute, func() (done bool, err error) {

installer/helm/chart/volcano/templates/admission.yaml

@@ -84,6 +84,7 @@ spec:
       {{- end }}
       containers:
         - args:
+            - --enabled-admission={{ .Values.custom.enabled_admissions }}
             - --tls-cert-file=/admission.local.config/certificates/tls.crt
             - --tls-private-key-file=/admission.local.config/certificates/tls.key
             - --ca-cert-file=/admission.local.config/certificates/ca.crt
@@ -154,4 +155,4 @@ spec:
           imagePullPolicy: IfNotPresent
           command: ["./gen-admission-secret.sh", "--service", "{{ .Release.Name }}-admission-service", "--namespace",
                     "{{ .Release.Namespace }}", "--secret", "{{.Values.basic.admission_secret_name}}"]
-{{- end }}
+{{- end }}

installer/helm/chart/volcano/templates/webhooks.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.custom.admission_enable }}
 
-{{- if .Values.custom.pods_mutatingwebhook_enable }}
+{{- if .Values.custom.enabled_admissions.pods_mutate_enable }}
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
@@ -42,7 +42,7 @@ webhooks:
 
 ---
 
-{{- if .Values.custom.queues_mutatingwebhook_enable }}
+{{- if .Values.custom.enabled_admissions.queues_mutate_enable }}
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
@@ -84,7 +84,7 @@ webhooks:
 
 ---
 
-{{- if .Values.custom.podgroups_mutatingwebhook_enable }}
+{{- if .Values.custom.enabled_admissions.podgroups_mutate_enable }}
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
@@ -126,7 +126,7 @@ webhooks:
 
 ---
 
-{{- if .Values.custom.jobs_mutatingwebhook_enable }}
+{{- if .Values.custom.enabled_admissions.jobs_mutate_enable }}
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
@@ -168,7 +168,7 @@ webhooks:
 
 ---
 
-{{- if .Values.custom.jobs_validatingwebhook_enable }}
+{{- if .Values.custom.enabled_admissions.jobs_validate_enable }}
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
@@ -210,7 +210,7 @@ webhooks:
 
 ---
 
-{{- if .Values.custom.pods_validatingwebhook_enable }}
+{{- if .Values.custom.enabled_admissions.pods_validate_enable }}
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
@@ -251,7 +251,7 @@ webhooks:
 
 ---
 
-{{- if .Values.custom.queues_validatingwebhook_enable }}
+{{- if .Values.custom.enabled_admissions.queues_validate_enable }}
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
@@ -291,4 +291,4 @@ webhooks:
     sideEffects: NoneOnDryRun
     timeoutSeconds: 10
 {{- end }}
-{{- end }}
+{{- end }}

installer/helm/chart/volcano/values.yaml

@@ -13,10 +13,11 @@ custom:
   admission_enable: true
   controller_enable: true
   scheduler_enable: true
-  pods_mutatingwebhook_enable: true
-  queues_mutatingwebhook_enable: true
-  podgroups_mutatingwebhook_enable: true
-  jobs_mutatingwebhook_enable: true
-  jobs_validatingwebhook_enable: true
-  pods_validatingwebhook_enable: true
-  queues_validatingwebhook_enable: true
+  enabled_admissions:
+    - pods_mutate_enable: true
+    - queues_mutate_enable: true
+    - podgroups_mutate_enable: true
+    - jobs_mutate_enable: true
+    - jobs_validate_enable: true
+    - pods_validate_enable: true
+    - queues_validate_enable: true

installer/volcano-development.yaml

@@ -134,6 +134,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - args:
+            - --enabled-admission=map[jobs_mutate_enable:true jobs_validate_enable:true podgroups_mutate_enable:true pods_mutate_enable:true pods_validate_enable:true queues_mutate_enable:true queues_validate_enable:true]
             - --tls-cert-file=/admission.local.config/certificates/tls.crt
             - --tls-private-key-file=/admission.local.config/certificates/tls.key
             - --ca-cert-file=/admission.local.config/certificates/ca.crt

pkg/webhooks/router/admission.go

@@ -19,9 +19,12 @@ package router
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 
+	"k8s.io/klog"
+
 	"volcano.sh/volcano/cmd/webhook-manager/app/options"
 )
 
@@ -48,11 +51,30 @@ func RegisterAdmission(service *AdmissionService) error {
 	return nil
 }
 
-func ForEachAdmission(config *options.Config, handler func(*AdmissionService)) {
-	admissions := strings.Split(strings.TrimSpace(config.EnabledAdmission), ",")
+func ForEachAdmission(config *options.Config, handler func(*AdmissionService) error) error {
+	admissions := getEnabledAdmissionFromConfig(config.EnabledAdmission)
+	klog.V(3).Infof("Enabled admissions are: %v, registered map are: %v", admissions, admissionMap)
 	for _, admission := range admissions {
 		if service, found := admissionMap[admission]; found {
-			handler(service)
+			if err := handler(service); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func getEnabledAdmissionFromConfig(enabledAdmissions map[string]string) []string {
+	var admissions = make([]string, 0)
+	for key, value := range enabledAdmissions {
+		enabled, err := strconv.ParseBool(value)
+		if err != nil {
+			klog.Warningf("invalid config for arg enabled-key: <%s:%s>, error: %s", key, value, err.Error())
+			continue
+		}
+		if enabled {
+			admissions = append(admissions, "/"+strings.ReplaceAll(strings.TrimSuffix(key, "_enable"), "_", "/"))
 		}
 	}
+	return admissions
 }