Add support for defining pod and container overrides via attribute

pkg/constants/attributes.go

@@ -103,4 +103,42 @@ const (
 	// MergedContributionsAttribute is applied as an attribute onto a component to list the components from the unflattened
 	// DevWorkspace that have been merged into the current component. The contributions are listed in a comma-separated list.
 	MergedContributionsAttribute = "controller.devfile.io/merged-contributions"
+
+	// PodOverridesAttribute is an attribute applied to a container component or in global attributes to specify overrides
+	// for the pod spec used in the main workspace deployment. The format of the field is the same as the Kubernetes
+	// PodSpec API. Overrides are applied over the default pod template spec used via strategic merge patch.
+	//
+	// If this attribute is used multiple times, all overrides are applied in the order they are defined in the DevWorkspace,
+	// with later values overriding previous ones. Overrides defined in the top-level attributes field are applied last and
+	// override any overrides from container components.
+	//
+	// Example:
+	//   kind: DevWorkspace
+	//   apiVersion: workspace.devfile.io/v1alpha2
+	//   spec:
+	//     template:
+	//       attributes:
+	//         pod-overrides:
+	//           metadata:
+	//             annotations:
+	//               io.openshift.userns: "true"
+	//               io.kubernetes.cri-o.userns-mode: "auto:size=65536;map-to-root=true"  # <-- user namespace
+	//               openshift.io/scc: container-build
+	//           spec:
+	//             runtimeClassName: kata
+	//             schedulerName: stork
+	PodOverridesAttribute = "pod-overrides"
+
+	// ContainerOverridesAttribute is an attribute applied to a container component to specify arbitrary fields in that
+	// container. This attribute should only be used to set fields that are not configurable in the container component
+	// itself. Any values specified in the overrides attribute overwrite fields on the container.
+	//
+	// Example:
+	//   components:
+	//     - name: go
+	//       attributes:
+	//         container-overrides: {"resources":{"limits":{"nvidia.com/gpu": "1"}}}
+	//       container:
+	//         image: ...
+	ContainerOverridesAttribute = "container-overrides"
 )

pkg/library/container/container.go

@@ -17,16 +17,18 @@
 // components
 //
 // TODO:
-// - Devfile API spec is unclear on how mountSources should be handled -- mountPath is assumed to be /projects
-//   and volume name is assumed to be "projects"
-//   see issues:
-//     - https://github.com/devfile/api/issues/290
-//     - https://github.com/devfile/api/issues/291
+//   - Devfile API spec is unclear on how mountSources should be handled -- mountPath is assumed to be /projects
+//     and volume name is assumed to be "projects"
+//     see issues:
+//   - https://github.com/devfile/api/issues/290
+//   - https://github.com/devfile/api/issues/291
 package container
 
 import (
 	"fmt"
 
+	"github.com/devfile/devworkspace-operator/pkg/library/overrides"
+
 	dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 	"github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
 	"github.com/devfile/devworkspace-operator/pkg/library/flatten"
@@ -49,7 +51,7 @@ func GetKubeContainersFromDevfile(workspace *dw.DevWorkspaceTemplateSpec, pullPo
 	}
 	podAdditions := &v1alpha1.PodAdditions{}
 
-	initContainers, mainComponents, err := lifecycle.GetInitContainers(workspace.DevWorkspaceTemplateSpecContent)
+	initComponents, mainComponents, err := lifecycle.GetInitContainers(workspace.DevWorkspaceTemplateSpecContent)
 	if err != nil {
 		return nil, err
 	}
@@ -63,19 +65,33 @@ func GetKubeContainersFromDevfile(workspace *dw.DevWorkspaceTemplateSpec, pullPo
 			return nil, err
 		}
 		handleMountSources(k8sContainer, component.Container, workspace.Projects)
+		if overrides.NeedsContainerOverride(&component) {
+			patchedContainer, err := overrides.ApplyContainerOverrides(&component, k8sContainer)
+			if err != nil {
+				return nil, err
+			}
+			k8sContainer = patchedContainer
+		}
 		podAdditions.Containers = append(podAdditions.Containers, *k8sContainer)
 	}
 
 	if err := lifecycle.AddPostStartLifecycleHooks(workspace, podAdditions.Containers); err != nil {
 		return nil, err
 	}
 
-	for _, container := range initContainers {
-		k8sContainer, err := convertContainerToK8s(container, pullPolicy)
+	for _, initComponent := range initComponents {
+		k8sContainer, err := convertContainerToK8s(initComponent, pullPolicy)
 		if err != nil {
 			return nil, err
 		}
-		handleMountSources(k8sContainer, container.Container, workspace.Projects)
+		handleMountSources(k8sContainer, initComponent.Container, workspace.Projects)
+		if overrides.NeedsContainerOverride(&initComponent) {
+			patchedContainer, err := overrides.ApplyContainerOverrides(&initComponent, k8sContainer)
+			if err != nil {
+				return nil, err
+			}
+			k8sContainer = patchedContainer
+		}
 		podAdditions.InitContainers = append(podAdditions.InitContainers, *k8sContainer)
 	}
 

pkg/library/overrides/containers.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2019-2022 Red Hat, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package overrides
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+
+	dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	"github.com/devfile/devworkspace-operator/pkg/constants"
+)
+
+func NeedsContainerOverride(component *dw.Component) bool {
+	return component.Container != nil && component.Attributes.Exists(constants.ContainerOverridesAttribute)
+}
+
+func ApplyContainerOverrides(component *dw.Component, container *corev1.Container) (*corev1.Container, error) {
+	override := &corev1.Container{}
+	if err := component.Attributes.GetInto(constants.ContainerOverridesAttribute, override); err != nil {
+		return nil, fmt.Errorf("failed to parse %s attribute on component %s: %w", constants.ContainerOverridesAttribute, component.Name, err)
+	}
+	override = restrictContainerOverride(override)
+
+	overrideBytes, err := json.Marshal(override)
+	if err != nil {
+		return nil, fmt.Errorf("error applying container overrides: %w", err)
+	}
+
+	originalBytes, err := json.Marshal(container)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal container to yaml: %w", err)
+	}
+
+	patchedBytes, err := strategicpatch.StrategicMergePatch(originalBytes, overrideBytes, &corev1.Container{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to apply container overrides: %w", err)
+	}
+
+	patched := &corev1.Container{}
+	if err := json.Unmarshal(patchedBytes, patched); err != nil {
+		return nil, fmt.Errorf("error applying container overrides: %w", err)
+	}
+	// Applying the patch will overwrite the container's name as corev1.Container.Name
+	// does not have the omitempty json tag.
+	patched.Name = container.Name
+	return patched, nil
+}
+
+// restrictContainerOverride unsets fields on a container that should not be
+// considered for container overrides. These fields are generally available to
+// set as fields on the container component itself.
+func restrictContainerOverride(override *corev1.Container) *corev1.Container {
+	result := override.DeepCopy()
+	result.Name = ""
+	result.Image = ""
+	result.Command = nil
+	result.Args = nil
+	result.Ports = nil
+	result.VolumeMounts = nil
+	result.Env = nil
+
+	return result
+}

pkg/library/overrides/containers_test.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2019-2022 Red Hat, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package overrides
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/yaml"
+)
+
+func TestApplyContainerOverrides(t *testing.T) {
+	tests := loadAllContainerTestCasesOrPanic(t, "testdata/container-overrides")
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%s (%s)", tt.Name, tt.originalFilename), func(t *testing.T) {
+			outContainer, err := ApplyContainerOverrides(tt.Input.Component, tt.Input.Container)
+			if tt.Output.ErrRegexp != nil && assert.Error(t, err) {
+				assert.Regexp(t, *tt.Output.ErrRegexp, err.Error(), "Error message should match")
+			} else {
+				if !assert.NoError(t, err, "Should not return error") {
+					return
+				}
+				assert.Truef(t, cmp.Equal(tt.Output.Container, outContainer),
+					"Container should match expected output:\n%s",
+					cmp.Diff(tt.Output.Container, outContainer))
+			}
+		})
+	}
+}
+
+type containerTestCase struct {
+	Name             string               `json:"name,omitempty"`
+	Input            *containerTestInput  `json:"input,omitempty"`
+	Output           *containerTestOutput `json:"output,omitempty"`
+	originalFilename string
+}
+
+type containerTestInput struct {
+	Component *dw.Component     `json:"component,omitempty"`
+	Container *corev1.Container `json:"container,omitempty"`
+}
+
+type containerTestOutput struct {
+	Container *corev1.Container `json:"container,omitempty"`
+	ErrRegexp *string           `json:"errRegexp,omitempty"`
+}
+
+func loadAllContainerTestCasesOrPanic(t *testing.T, fromDir string) []containerTestCase {
+	files, err := os.ReadDir(fromDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var tests []containerTestCase
+	for _, file := range files {
+		if file.IsDir() {
+			tests = append(tests, loadAllContainerTestCasesOrPanic(t, filepath.Join(fromDir, file.Name()))...)
+		} else {
+			tests = append(tests, loadContainerTestCaseOrPanic(t, filepath.Join(fromDir, file.Name())))
+		}
+	}
+	return tests
+}
+
+func loadContainerTestCaseOrPanic(t *testing.T, testPath string) containerTestCase {
+	bytes, err := os.ReadFile(testPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var test containerTestCase
+	if err := yaml.Unmarshal(bytes, &test); err != nil {
+		t.Fatal(err)
+	}
+	test.originalFilename = testPath
+	return test
+}

pkg/library/overrides/pods.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2019-2022 Red Hat, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package overrides
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+
+	"github.com/devfile/devworkspace-operator/pkg/common"
+	"github.com/devfile/devworkspace-operator/pkg/constants"
+)
+
+// NeedsPodOverrides returns whether the current DevWorkspace defines pod overrides via an attribute
+// attribute.
+func NeedsPodOverrides(workspace *common.DevWorkspaceWithConfig) bool {
+	if workspace.Spec.Template.Attributes.Exists(constants.PodOverridesAttribute) {
+		return true
+	}
+	for _, component := range workspace.Spec.Template.Components {
+		if component.Attributes.Exists(constants.PodOverridesAttribute) {
+			return true
+		}
+	}
+	return false
+}
+
+func ApplyPodOverrides(workspace *common.DevWorkspaceWithConfig, deployment *appsv1.Deployment) (*appsv1.Deployment, error) {
+	overrides, err := getPodOverrides(workspace)
+	if err != nil {
+		return nil, err
+	}
+
+	patched := deployment.DeepCopy()
+	// Workaround: the definition for corev1.PodSpec does not make containers optional, so even a nil list
+	// will be interpreted as "delete all containers" as the serialized patch will include "containers": null.
+	// To avoid this, save the original containers and reset them at the end.
+	originalContainers := patched.Spec.Template.Spec.Containers
+	patchedTemplateBytes, err := json.Marshal(patched.Spec.Template)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal deployment to yaml: %w", err)
+	}
+	for _, override := range overrides {
+		patchBytes, err := json.Marshal(override)
+		if err != nil {
+			return nil, fmt.Errorf("error applying pod overrides: %w", err)
+		}
+
+		patchedTemplateBytes, err = strategicpatch.StrategicMergePatch(patchedTemplateBytes, patchBytes, &corev1.PodTemplateSpec{})
+		if err != nil {
+			return nil, fmt.Errorf("error applying pod overrides: %w", err)
+		}
+	}
+
+	patchedPodSpecTemplate := corev1.PodTemplateSpec{}
+	if err := json.Unmarshal(patchedTemplateBytes, &patchedPodSpecTemplate); err != nil {
+		return nil, fmt.Errorf("error applying pod overrides: %w", err)
+	}
+	patched.Spec.Template = patchedPodSpecTemplate
+	patched.Spec.Template.Spec.Containers = originalContainers
+	return patched, nil
+}
+
+// getPodOverrides returns PodTemplateSpecOverrides for every instance of the pod overrides attribute
+// present in the DevWorkspace. The order of elements is
+// 1. Pod overrides defined on Container components, in the order they appear in the DevWorkspace
+// 2. Pod overrides defined in the global attributes field (.spec.template.attributes)
+func getPodOverrides(workspace *common.DevWorkspaceWithConfig) ([]corev1.PodTemplateSpec, error) {
+	var allOverrides []corev1.PodTemplateSpec
+
+	for _, component := range workspace.Spec.Template.Components {
+		if component.Attributes.Exists(constants.PodOverridesAttribute) {
+			override := corev1.PodTemplateSpec{}
+			err := component.Attributes.GetInto(constants.PodOverridesAttribute, &override)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse %s attribute on component %s: %w", constants.PodOverridesAttribute, component.Name, err)
+			}
+			// Do not allow overriding containers
+			override.Spec.Containers = nil
+			override.Spec.InitContainers = nil
+			override.Spec.Volumes = nil
+			allOverrides = append(allOverrides, override)
+		}
+	}
+	if workspace.Spec.Template.Attributes.Exists(constants.PodOverridesAttribute) {
+		override := corev1.PodTemplateSpec{}
+		err := workspace.Spec.Template.Attributes.GetInto(constants.PodOverridesAttribute, &override)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %s attribute for workspace: %w", constants.PodOverridesAttribute, err)
+		}
+		// Do not allow overriding containers or volumes
+		override.Spec.Containers = nil
+		override.Spec.InitContainers = nil
+		override.Spec.Volumes = nil
+		allOverrides = append(allOverrides, override)
+	}
+	return allOverrides, nil
+}