Merge pull request #820 from pivotal/build-pod-node-selection

Make build pod nodeSelector, Tolerations, and Affinity configurable

api/openapi-spec/swagger.json

@@ -4879,7 +4879,7 @@
       ],
       "properties": {
         "build": {
-          "$ref": "#/definitions/kpack.core.v1alpha1.ImageBuild"
+          "$ref": "#/definitions/kpack.build.v1alpha1.ImageBuild"
         },
         "builder": {
           "$ref": "#/definitions/io.k8s.api.core.v1.ObjectReference"
@@ -5203,6 +5203,9 @@
         "source"
       ],
       "properties": {
+        "affinity": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.Affinity"
+        },
         "bindings": {
           "type": "array",
           "items": {
@@ -5229,6 +5232,12 @@
         "lastBuild": {
           "$ref": "#/definitions/kpack.build.v1alpha2.LastBuild"
         },
+        "nodeSelector": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        },
         "notary": {
           "$ref": "#/definitions/kpack.core.v1alpha1.NotaryConfig"
         },
@@ -5238,6 +5247,12 @@
         "resources": {
           "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
         },
+        "runtimeClassName": {
+          "type": "string"
+        },
+        "schedulerName": {
+          "type": "string"
+        },
         "serviceAccount": {
           "type": "string"
         },
@@ -5250,6 +5265,13 @@
             "type": "string"
           },
           "x-kubernetes-list-type": ""
+        },
+        "tolerations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/io.k8s.api.core.v1.Toleration"
+          },
+          "x-kubernetes-list-type": ""
         }
       }
     },
@@ -5761,6 +5783,50 @@
         }
       }
     },
+    "kpack.build.v1alpha2.ImageBuild": {
+      "type": "object",
+      "properties": {
+        "affinity": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.Affinity"
+        },
+        "bindings": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/kpack.core.v1alpha1.Binding"
+          },
+          "x-kubernetes-list-type": ""
+        },
+        "env": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
+          },
+          "x-kubernetes-list-type": ""
+        },
+        "nodeSelector": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        },
+        "resources": {
+          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+        },
+        "runtimeClassName": {
+          "type": "string"
+        },
+        "schedulerName": {
+          "type": "string"
+        },
+        "tolerations": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/io.k8s.api.core.v1.Toleration"
+          },
+          "x-kubernetes-list-type": ""
+        }
+      }
+    },
     "kpack.build.v1alpha2.ImageBuilder": {
       "type": "object",
       "required": [
@@ -5833,7 +5899,7 @@
       ],
       "properties": {
         "build": {
-          "$ref": "#/definitions/kpack.core.v1alpha1.ImageBuild"
+          "$ref": "#/definitions/kpack.build.v1alpha2.ImageBuild"
         },
         "builder": {
           "$ref": "#/definitions/io.k8s.api.core.v1.ObjectReference"
@@ -6262,28 +6328,6 @@
         }
       }
     },
-    "kpack.core.v1alpha1.ImageBuild": {
-      "type": "object",
-      "properties": {
-        "bindings": {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/kpack.core.v1alpha1.Binding"
-          },
-          "x-kubernetes-list-type": ""
-        },
-        "env": {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar"
-          },
-          "x-kubernetes-list-type": ""
-        },
-        "resources": {
-          "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements"
-        }
-      }
-    },
     "kpack.core.v1alpha1.NotaryConfig": {
       "type": "object",
       "properties": {

docs/build.md

@@ -37,6 +37,22 @@ spec:
     limits:
       cpu: "0.5"
       memory: "256M"
+  tolerations:
+    - key: "key1"
+      operator: "Exists"
+      effect: "NoSchedule"
+  nodeSelector:
+    disktype: ssd
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: kubernetes.io/e2e-az-name
+                operator: In
+                values:
+                  - e2e-az1
+                  - e2e-az2
 ```
 
 - `tags`: A list of docker tags to build. At least one tag is required.
@@ -50,6 +66,9 @@ spec:
 - `env`: Optional list of build time environment variables.
 - `projectDescriptorPath`: Path to the [project descriptor file](https://buildpacks.io/docs/reference/config/project-descriptor/) relative to source root dir or `subPath` if set. If unset, kpack will look for `project.toml` at the root dir or `subPath` if set.
 - `resources`: Optional configurable resource limits on `CPU` and `memory`.
+- `tolerations`: Optional configurable pod spec tolerations
+- `nodeSelector`: Optional configurable pod spec nodeSelector
+- `affinity`: Optional configurabl pod spec affinity
 
 > Note: All fields on a build are immutable. Instead of updating a build, create a new one.
 

docs/image.md

@@ -92,7 +92,7 @@ The `source` field is a composition of a source code location and a `subpath`. I
 
 ### <a id='build-config'></a>Build Configuration
 
-The `build` field on the `image` resource can be used to configure env variables required during the build process and to configure resource limits on `CPU` and `memory`.
+The `build` field on the `image` resource can be used to configure env variables required during the build process, to configure resource limits on `CPU` and `memory`, and to configure pod tolerations, node selector, and affinity.
 
 ```yaml
 build:
@@ -106,6 +106,22 @@ build:
       limits:
         cpu: "0.5"
         memory: "256M"
+  tolerations:
+    - key: "key1"
+      operator: "Exists"
+      effect: "NoSchedule"
+  nodeSelector:
+    disktype: ssd
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: kubernetes.io/e2e-az-name
+                operator: In
+                values:
+                  - e2e-az1
+                  - e2e-az2
 ```
 
 See the kubernetes documentation on [setting environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) and [resource limits and requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) for more information.

pkg/apis/build/v1alpha1/image_builds_test.go

@@ -204,7 +204,7 @@ func testImageBuilds(t *testing.T, when spec.G, it spec.S) {
 		})
 
 		it("adds the env vars to the build spec", func() {
-			image.Spec.Build = &corev1alpha1.ImageBuild{
+			image.Spec.Build = &ImageBuild{
 				Env: []corev1.EnvVar{
 					{Name: "keyA", Value: "new"},
 				},
@@ -228,7 +228,7 @@ func testImageBuilds(t *testing.T, when spec.G, it spec.S) {
 		})
 
 		it("adds build resources", func() {
-			image.Spec.Build = &corev1alpha1.ImageBuild{
+			image.Spec.Build = &ImageBuild{
 				Resources: corev1.ResourceRequirements{
 					Limits: corev1.ResourceList{
 						corev1.ResourceCPU:    resource.MustParse("2"),

pkg/apis/build/v1alpha1/image_types.go

@@ -52,10 +52,18 @@ type ImageSpec struct {
 	FailedBuildHistoryLimit  *int64                            `json:"failedBuildHistoryLimit,omitempty"`
 	SuccessBuildHistoryLimit *int64                            `json:"successBuildHistoryLimit,omitempty"`
 	ImageTaggingStrategy     corev1alpha1.ImageTaggingStrategy `json:"imageTaggingStrategy,omitempty"`
-	Build                    *corev1alpha1.ImageBuild          `json:"build,omitempty"`
+	Build                    *ImageBuild                       `json:"build,omitempty"`
 	Notary                   *corev1alpha1.NotaryConfig        `json:"notary,omitempty"`
 }
 
+type ImageBuild struct {
+	// +listType
+	Bindings corev1alpha1.Bindings `json:"bindings,omitempty"`
+	// +listType
+	Env       []corev1.EnvVar             `json:"env,omitempty"`
+	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+}
+
 // +k8s:openapi-gen=true
 type ImageStatus struct {
 	corev1alpha1.Status        `json:",inline"`

pkg/apis/build/v1alpha1/image_validation.go

@@ -119,6 +119,14 @@ func (is *ImageSpec) validateCacheSize(ctx context.Context) *apis.FieldError {
 	return nil
 }
 
+func (ib *ImageBuild) Validate(ctx context.Context) *apis.FieldError {
+	if ib == nil {
+		return nil
+	}
+
+	return ib.Bindings.Validate(ctx).ViaField("bindings")
+}
+
 func validateBuilder(builder v1.ObjectReference) *apis.FieldError {
 	if builder.Name == "" {
 		return apis.ErrMissingField("name")

pkg/apis/build/v1alpha1/image_validation_test.go

@@ -44,7 +44,7 @@ func testImageValidation(t *testing.T, when spec.G, it spec.S) {
 			FailedBuildHistoryLimit:  &limit,
 			SuccessBuildHistoryLimit: &limit,
 			ImageTaggingStrategy:     corev1alpha1.None,
-			Build: &corev1alpha1.ImageBuild{
+			Build: &ImageBuild{
 				Env: []corev1.EnvVar{
 					{
 						Name:  "keyA",

pkg/apis/build/v1alpha2/build_pod.go

@@ -21,6 +21,7 @@ const (
 	BuildLabel                   = "kpack.io/build"
 	DOCKERSecretAnnotationPrefix = "kpack.io/docker"
 	GITSecretAnnotationPrefix    = "kpack.io/git"
+	k8sOSLabel                   = "kubernetes.io/os"
 
 	cacheDirName              = "cache-dir"
 	layersDirName             = "layers-dir"
@@ -132,7 +133,7 @@ var (
 
 type stepModifier func(corev1.Container) corev1.Container
 
-func (b *Build) BuildPod(images BuildPodImages, secrets []corev1.Secret, taints []corev1.Taint, config BuildPodBuilderConfig) (*corev1.Pod, error) {
+func (b *Build) BuildPod(images BuildPodImages, secrets []corev1.Secret, config BuildPodBuilderConfig) (*corev1.Pod, error) {
 	platformAPI, err := config.highestSupportedPlatformAPI(b)
 	if err != nil {
 		return nil, err
@@ -445,10 +446,11 @@ func (b *Build) BuildPod(images BuildPodImages, secrets []corev1.Secret, taints
 				)
 			}),
 			ServiceAccountName: b.Spec.ServiceAccount,
-			NodeSelector: map[string]string{
-				"kubernetes.io/os": config.OS,
-			},
-			Tolerations: tolerations(taints),
+			NodeSelector:       b.nodeSelector(config.OS),
+			Tolerations:        b.Spec.Tolerations,
+			Affinity:           b.Spec.Affinity,
+			RuntimeClassName:   b.Spec.RuntimeClassName,
+			SchedulerName:      b.Spec.SchedulerName,
 			Volumes: append(append(
 				append(secretVolumes, b.cacheVolume(config.OS)...),
 				corev1.Volume{
@@ -588,9 +590,11 @@ func (b *Build) rebasePod(secrets []corev1.Secret, images BuildPodImages, config
 		},
 		Spec: corev1.PodSpec{
 			ServiceAccountName: b.Spec.ServiceAccount,
-			NodeSelector: map[string]string{
-				"kubernetes.io/os": "linux",
-			},
+			NodeSelector:       b.nodeSelector("linux"),
+			Tolerations:        b.Spec.Tolerations,
+			Affinity:           b.Spec.Affinity,
+			RuntimeClassName:   b.Spec.RuntimeClassName,
+			SchedulerName:      b.Spec.SchedulerName,
 			Volumes: append(
 				secretVolumes,
 				corev1.Volume{
@@ -811,19 +815,9 @@ func (bc *BuildPodBuilderConfig) highestSupportedPlatformAPI(b *Build) (*semver.
 	return nil, errors.Errorf("unsupported builder platform API versions: %s", strings.Join(bc.PlatformAPIs, ","))
 }
 
-func tolerations(taints []corev1.Taint) []corev1.Toleration {
-	t := make([]corev1.Toleration, 0, len(taints))
-
-	for _, taint := range taints {
-		t = append(t, corev1.Toleration{
-			Key:      taint.Key,
-			Operator: corev1.TolerationOpEqual,
-			Value:    taint.Value,
-			Effect:   taint.Effect,
-		})
-	}
-
-	return t
+func (b Build) nodeSelector(os string) map[string]string {
+	b.Spec.NodeSelector[k8sOSLabel] = os
+	return b.Spec.NodeSelector
 }
 
 func builderSecretVolume(bbs corev1alpha1.BuildBuilderSpec) corev1.Volume {