Merge pull request #2737 from jiamin13579/affinity

pod admission  mutate patch affinity

docs/design/multi-scheduler.md

@@ -21,7 +21,8 @@ The configmap defines some resource groups, each resource group contains
   - Annotation field
 - The pod's data volcano needs to patch, volcano support to patch the fields and the fields are optional and not mandatory. User can set them according the application scenario.
   - Tolerations 
-  - NodeSelector 
+  - Affinity
+  - NodeSelector
   - SchedulerName
 
 If the object field is not setted, it is filled with a default as the following:
@@ -67,6 +68,14 @@ data:
       schedulerName: volcano                      # the annotation key is fixed and is "volcano.sh/resource-group", The corresponding value is the resourceGroup field
       labels:
         volcano.sh/nodetype: gpu
+    - resourceGroup: fixed                        # if the object is unsetted, default is:  the key is annotation,
+      schedulerName: volcano                      # the annotation key is fixed and is "volcano.sh/resource-group", The corresponding value is the resourceGroup field
+      object:
+        key: annotation
+        value:
+        - "volcano.sh/resource-group-job-role: master"
+      # set the affinity for patching, the format is a json string.
+      affinity: "{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"matchExpressions\":[{\"key\":\"volcano.sh/nodetype\",\"operator\":\"In\",\"values\":[\"fixed\"]}]}]}}}"
 ````
 
 ### The pod mutate process 
@@ -205,6 +214,72 @@ schedulerName: volcano
 The pod in job-A is scheduled to node1.
 The pod in job-B job is scheduled to node2.
 ````
+
+### case 3 
+
+Here is a cluster as the following:
+
+|node|label|
+|----|-----|
+|node1| volcano.sh/nodetype: fixed|
+|node2| none| none|
+
+|volcano job | annotation|
+|----|----|
+|job-A|volcano.sh/resource-group-job-role: master| 
+|job-B|none|
+
+1. Edit volcano-admission-configmap
+````
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: volcano-admission-configmap
+  namespace: volcano-system
+data:
+  volcano-admission.conf: |
+    resourceGroups:
+    - resourceGroup: fixed                        # if the object is unsetted, default is:  the key is annotation,
+      schedulerName: volcano                      # the annotation key is fixed and is "volcano.sh/resource-group", The corresponding value is the resourceGroup field
+      object:
+        key: annotation
+        value:
+        - "volcano.sh/resource-group-job-role: master"
+      # set the affinity for patching, the format is a json string.
+      affinity: "{\"nodeAffinity\":{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"matchExpressions\":[{\"key\":\"volcano.sh/nodetype",\"operator\":\"In\",\"values\":[\"fixed\"]}]}]}}}"
+````
+2. Submit job-A and job-B
+
+3. Check the Pod information 
+
+````
+job-A:
+....
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: volcano.sh/nodetype
+          operator: In
+          values:
+          - fixed
+...
+schedulerName: volcano
+....
+
+job-B:
+....
+...
+schedulerName: volcano
+....
+ ````
+4. Check the result of the pod's scheduling
+````
+The pod in job-A is scheduled to node1.
+The pod in job-B job is scheduled to node1/node2.
+````
+
 ## NOTE
 
 Enable this feature may modify pod information and affect resource utilization.

pkg/webhooks/admission/pods/mutate/mutate_pod.go

@@ -122,6 +122,11 @@ func createPatch(pod *v1.Pod) ([]byte, error) {
 			patch = append(patch, *patchLabel)
 		}
 
+		patchAffinity := patchAffinity(pod, resourceGroup)
+		if patchAffinity != nil {
+			patch = append(patch, *patchAffinity)
+		}
+
 		patchToleration := patchTaintToleration(pod, resourceGroup)
 		if patchToleration != nil {
 			patch = append(patch, *patchToleration)
@@ -156,6 +161,28 @@ func patchLabels(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) *patchOper
 	return &patchOperation{Op: "add", Path: "/spec/nodeSelector", Value: nodeSelector}
 }
 
+// patchAffinity patch affinity
+func patchAffinity(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) *patchOperation {
+	if resGroupConfig.Affinity == "" {
+		return nil
+	}
+
+	if pod.Spec.Affinity != nil {
+		klog.V(5).Infof("pod affinity exist: %s", pod.Name)
+		return nil
+	}
+
+	var affinity v1.Affinity
+	err := json.Unmarshal([]byte(resGroupConfig.Affinity), &affinity)
+	if err != nil {
+		fmt.Println("Failed to unmarshal JSON:", err)
+		klog.V(3).Infof("Failed to unmarshal JSON: %s", err)
+		return nil
+	}
+
+	return &patchOperation{Op: "add", Path: "/spec/affinity", Value: affinity}
+}
+
 // patchTaintToleration patch taint toleration
 func patchTaintToleration(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) *patchOperation {
 	if len(resGroupConfig.Tolerations) == 0 {

pkg/webhooks/admission/pods/mutate/mutate_pod_test.go

@@ -28,6 +28,9 @@ import (
 )
 
 func TestMutatePods(t *testing.T) {
+	affinityJsonStr := `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/os","operator":"In","values":["linux"]}]}]}}}`
+	var affinity v1.Affinity
+	json.Unmarshal([]byte(affinityJsonStr), &affinity)
 
 	admissionConfigData := &webconfig.AdmissionConfiguration{
 		ResGroupsConfig: []webconfig.ResGroupConfig{
@@ -48,6 +51,7 @@ func TestMutatePods(t *testing.T) {
 						Effect:   v1.TaintEffectNoSchedule,
 					},
 				},
+				Affinity: affinityJsonStr,
 				Labels: map[string]string{
 					"volcano.sh/nodetype": "management",
 				},
@@ -105,6 +109,11 @@ func TestMutatePods(t *testing.T) {
 						"volcano.sh/nodetype": "management",
 					},
 				},
+				{
+					Op:    "add",
+					Path:  "/spec/affinity",
+					Value: affinity,
+				},
 				{
 					Op:   "add",
 					Path: "/spec/tolerations",

pkg/webhooks/config/config.go

@@ -42,6 +42,7 @@ type ResGroupConfig struct {
 	SchedulerName string            `yaml:"schedulerName"`
 	Tolerations   []v1.Toleration   `yaml:"tolerations"`
 	Labels        map[string]string `yaml:"labels"`
+	Affinity      string            `yaml:"affinity"`
 }
 
 // AdmissionConfiguration defines the configuration of admission.