diff --git a/api/v1/testkube.yaml b/api/v1/testkube.yaml index 50079b8db7..f060a0e820 100644 --- a/api/v1/testkube.yaml +++ b/api/v1/testkube.yaml @@ -8678,6 +8678,10 @@ components: type: string readinessProbe: $ref: "#/components/schemas/Probe" + pvcs: + type: object + additionalProperties: + $ref: "#/components/schemas/TestWorkflowPvcConfig" TestWorkflowServiceSpec: type: object @@ -8797,6 +8801,10 @@ components: $ref: "#/components/schemas/TestWorkflowEvent" execution: $ref: "#/components/schemas/TestWorkflowTagSchema" + pvcs: + type: object + additionalProperties: + $ref: "#/components/schemas/TestWorkflowPvcConfig" TestWorkflowTemplateSpec: type: object @@ -8835,6 +8843,10 @@ components: $ref: "#/components/schemas/TestWorkflowEvent" execution: $ref: "#/components/schemas/TestWorkflowTagSchema" + pvcs: + type: object + additionalProperties: + $ref: "#/components/schemas/TestWorkflowPvcConfig" TestWorkflowStepControl: type: object @@ -10588,6 +10600,67 @@ components: must be defined type: boolean + TestWorkflowPvcConfig: + type: object + properties: + accessModes: + description: 'Access mode for claim storage. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes' + type: array + items: + type: string + volumeMode: + description: 'Volume mode indicates the consumption of the volume as either a filesystem or block device. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#volume-mode' + $ref: "#/components/schemas/BoxedString" + resources: + description: 'Resources required for pvc' + $ref: "#/components/schemas/TestWorkflowResources" + storageClassName: + description: 'Storage class name specifies the name of a StorageClass. More info: https://kubernetes.io/docs/concepts/storage/storage-classes/' + $ref: "#/components/schemas/BoxedString" + volumeName: + description: 'Volume name is used to identify the volume' + type: string + selector: + description: Only the volumes whose labels match the selector can be bound to the claim + $ref: "#/components/schemas/LabelSelector" + dataSource: + description: 'Data source field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim)' + $ref: "#/components/schemas/TypedLocalObjectReference" + dataSourceRef: + description: 'Data source reference specifies the object from which to populate the volume with data, if a non-empty volume is desired' + $ref: "#/components/schemas/TypedObjectReference" + volumeAttributesClassName: + description: 'Volume attributes class name may be used to set the VolumeAttributesClass used by this claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass' + $ref: "#/components/schemas/BoxedString" + + TypedLocalObjectReference: + description: TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace + type: object + properties: + apiGroup: + description: api group is the group for the resource being referenced + $ref: "#/components/schemas/BoxedString" + kind: + description: kind is the type of resource being referenced + type: string + name: + description: name is the name of resource being referenced + type: string + + TypedObjectReference: + description: TypedObjectReference contains enough information to let you locate the typed referenced object inside the specified namespace + type: object + allOf: + - $ref: "#/components/schemas/TypedLocalObjectReference" + properties: + namespace: + description: Namespace is the namespace of resource being referenced + $ref: "#/components/schemas/BoxedString" + # # Errors # diff --git a/cmd/tcl/testworkflow-toolkit/commands/parallel.go b/cmd/tcl/testworkflow-toolkit/commands/parallel.go index 75d6f233df..d06f0bd0ed 100644 --- a/cmd/tcl/testworkflow-toolkit/commands/parallel.go +++ b/cmd/tcl/testworkflow-toolkit/commands/parallel.go @@ -205,6 +205,7 @@ func NewParallelCmd() *cobra.Command { testworkflowconfig.CreateResourceMachine(&cfg.Resource), testworkflowconfig.CreateWorkerMachine(&cfg.Worker), baseMachine, + testworkflowconfig.CreatePvcMachine(cfg.Execution.PvcNames), params.MachineAt(index), ) diff --git a/cmd/tcl/testworkflow-toolkit/commands/services.go b/cmd/tcl/testworkflow-toolkit/commands/services.go index b32f3231fb..96784dd001 100644 --- a/cmd/tcl/testworkflow-toolkit/commands/services.go +++ b/cmd/tcl/testworkflow-toolkit/commands/services.go @@ -156,6 +156,7 @@ func NewServicesCmd() *cobra.Command { Steps: []testworkflowsv1.Step{ {StepOperations: testworkflowsv1.StepOperations{Run: common.Ptr(svcSpec.StepRun)}}, }, + Pvcs: svcSpec.Pvcs, } spec.Steps[0].Run.ContainerConfig = testworkflowsv1.ContainerConfig{} @@ -260,6 +261,7 @@ func NewServicesCmd() *cobra.Command { testworkflowconfig.CreateResourceMachine(&cfg.Resource), testworkflowconfig.CreateWorkerMachine(&cfg.Worker), baseMachine, + testworkflowconfig.CreatePvcMachine(cfg.Execution.PvcNames), params.MachineAt(index), ) diff --git a/go.mod b/go.mod index f2e215d06b..c4e5b664e7 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/keygen-sh/jsonapi-go v1.2.1 github.com/keygen-sh/keygen-go/v3 v3.2.0 github.com/kubepug/kubepug v1.7.1 - github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a + github.com/kubeshop/testkube-operator v1.17.55-0.20241219071935-0330347ee160 github.com/minio/minio-go/v7 v7.0.66 github.com/montanaflynn/stats v0.7.1 github.com/moogar0880/problems v0.1.1 diff --git a/go.sum b/go.sum index 56f08fbd15..c05bb6f2fb 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubepug/kubepug v1.7.1 h1:LKhfSxS8Y5mXs50v+3Lpyec+cogErDLcV7CMUuiaisw= github.com/kubepug/kubepug v1.7.1/go.mod h1:lv+HxD0oTFL7ZWjj0u6HKhMbbTIId3eG7aWIW0gyF8g= -github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a h1:xget2cwwqOL+K2Op9FPbMgfzj9lSVJAzZ9p48yxuFrE= -github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk= +github.com/kubeshop/testkube-operator v1.17.55-0.20241219071935-0330347ee160 h1:Bocn/Cgx/XLEsqvU3IdK5U/cFDJDFD3tES/1KS2wLkY= +github.com/kubeshop/testkube-operator v1.17.55-0.20241219071935-0330347ee160/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4= diff --git a/pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go b/pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go index 192eeadb60..57ea947f73 100644 --- a/pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go +++ b/pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go @@ -41,5 +41,6 @@ type TestWorkflowIndependentServiceSpec struct { // matrix of parameters to spawn instances Matrix map[string]interface{} `json:"matrix,omitempty"` // parameters that should be distributed across sharded instances - Shards map[string]interface{} `json:"shards,omitempty"` + Shards map[string]interface{} `json:"shards,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go b/pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go index ddb97dc54c..a68339239e 100644 --- a/pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go +++ b/pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go @@ -53,4 +53,5 @@ type TestWorkflowIndependentStepParallel struct { After []TestWorkflowIndependentStep `json:"after,omitempty"` Events []TestWorkflowEvent `json:"events,omitempty"` Execution *TestWorkflowTagSchema `json:"execution,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_pvc_config.go b/pkg/api/v1/testkube/model_test_workflow_pvc_config.go new file mode 100644 index 0000000000..a7a03cd126 --- /dev/null +++ b/pkg/api/v1/testkube/model_test_workflow_pvc_config.go @@ -0,0 +1,24 @@ +/* + * Testkube API + * + * Testkube provides a Kubernetes-native framework for test definition, execution and results + * + * API version: 1.0.0 + * Contact: testkube@kubeshop.io + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ +package testkube + +type TestWorkflowPvcConfig struct { + // Access mode for claim storage. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes + AccessModes []string `json:"accessModes,omitempty"` + VolumeMode *BoxedString `json:"volumeMode,omitempty"` + Resources *TestWorkflowResources `json:"resources,omitempty"` + StorageClassName *BoxedString `json:"storageClassName,omitempty"` + // Volume name is used to identify the volume + VolumeName string `json:"volumeName,omitempty"` + Selector *LabelSelector `json:"selector,omitempty"` + DataSource *TypedLocalObjectReference `json:"dataSource,omitempty"` + DataSourceRef *TypedObjectReference `json:"dataSourceRef,omitempty"` + VolumeAttributesClassName *BoxedString `json:"volumeAttributesClassName,omitempty"` +} diff --git a/pkg/api/v1/testkube/model_test_workflow_service_spec.go b/pkg/api/v1/testkube/model_test_workflow_service_spec.go index ad915dd162..4df57b081e 100644 --- a/pkg/api/v1/testkube/model_test_workflow_service_spec.go +++ b/pkg/api/v1/testkube/model_test_workflow_service_spec.go @@ -42,5 +42,6 @@ type TestWorkflowServiceSpec struct { // matrix of parameters to spawn instances Matrix map[string]interface{} `json:"matrix,omitempty"` // parameters that should be distributed across sharded instances - Shards map[string]interface{} `json:"shards,omitempty"` + Shards map[string]interface{} `json:"shards,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_spec.go b/pkg/api/v1/testkube/model_test_workflow_spec.go index 87a6c0e7ee..f8a69366d4 100644 --- a/pkg/api/v1/testkube/model_test_workflow_spec.go +++ b/pkg/api/v1/testkube/model_test_workflow_spec.go @@ -23,4 +23,5 @@ type TestWorkflowSpec struct { After []TestWorkflowStep `json:"after,omitempty"` Events []TestWorkflowEvent `json:"events,omitempty"` Execution *TestWorkflowTagSchema `json:"execution,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_step_parallel.go b/pkg/api/v1/testkube/model_test_workflow_step_parallel.go index 7c8a9ffcca..d11b2e1d85 100644 --- a/pkg/api/v1/testkube/model_test_workflow_step_parallel.go +++ b/pkg/api/v1/testkube/model_test_workflow_step_parallel.go @@ -55,4 +55,5 @@ type TestWorkflowStepParallel struct { After []TestWorkflowStep `json:"after,omitempty"` Events []TestWorkflowEvent `json:"events,omitempty"` Execution *TestWorkflowTagSchema `json:"execution,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_template_spec.go b/pkg/api/v1/testkube/model_test_workflow_template_spec.go index 17d16455a7..c1b9c34c40 100644 --- a/pkg/api/v1/testkube/model_test_workflow_template_spec.go +++ b/pkg/api/v1/testkube/model_test_workflow_template_spec.go @@ -22,4 +22,5 @@ type TestWorkflowTemplateSpec struct { After []TestWorkflowIndependentStep `json:"after,omitempty"` Events []TestWorkflowEvent `json:"events,omitempty"` Execution *TestWorkflowTagSchema `json:"execution,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_typed_local_object_reference.go b/pkg/api/v1/testkube/model_typed_local_object_reference.go new file mode 100644 index 0000000000..4701fd64b4 --- /dev/null +++ b/pkg/api/v1/testkube/model_typed_local_object_reference.go @@ -0,0 +1,19 @@ +/* + * Testkube API + * + * Testkube provides a Kubernetes-native framework for test definition, execution and results + * + * API version: 1.0.0 + * Contact: testkube@kubeshop.io + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ +package testkube + +// TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace +type TypedLocalObjectReference struct { + ApiGroup *BoxedString `json:"apiGroup,omitempty"` + // kind is the type of resource being referenced + Kind string `json:"kind,omitempty"` + // name is the name of resource being referenced + Name string `json:"name,omitempty"` +} diff --git a/pkg/api/v1/testkube/model_typed_object_reference.go b/pkg/api/v1/testkube/model_typed_object_reference.go new file mode 100644 index 0000000000..18617f8d0b --- /dev/null +++ b/pkg/api/v1/testkube/model_typed_object_reference.go @@ -0,0 +1,20 @@ +/* + * Testkube API + * + * Testkube provides a Kubernetes-native framework for test definition, execution and results + * + * API version: 1.0.0 + * Contact: testkube@kubeshop.io + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ +package testkube + +// TypedObjectReference contains enough information to let you locate the typed referenced object inside the specified namespace +type TypedObjectReference struct { + ApiGroup *BoxedString `json:"apiGroup,omitempty"` + // kind is the type of resource being referenced + Kind string `json:"kind,omitempty"` + // name is the name of resource being referenced + Name string `json:"name,omitempty"` + Namespace *BoxedString `json:"namespace,omitempty"` +} diff --git a/pkg/mapper/testworkflows/kube_openapi.go b/pkg/mapper/testworkflows/kube_openapi.go index 9076451da8..cb1ac92b34 100644 --- a/pkg/mapper/testworkflows/kube_openapi.go +++ b/pkg/mapper/testworkflows/kube_openapi.go @@ -915,6 +915,7 @@ func MapStepParallelKubeToAPI(v testworkflowsv1.StepParallel) testkube.TestWorkf Run: common.MapPtr(v.Run, MapStepRunKubeToAPI), Execute: common.MapPtr(v.Execute, MapStepExecuteKubeToAPI), Artifacts: common.MapPtr(v.Artifacts, MapStepArtifactsKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -948,6 +949,7 @@ func MapIndependentStepParallelKubeToAPI(v testworkflowsv1.IndependentStepParall Run: common.MapPtr(v.Run, MapStepRunKubeToAPI), Execute: common.MapPtr(v.Execute, MapStepExecuteKubeToAPI), Artifacts: common.MapPtr(v.Artifacts, MapStepArtifactsKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1028,6 +1030,7 @@ func MapIndependentServiceSpecKubeToAPI(v testworkflowsv1.IndependentServiceSpec Logs: MapStringToBoxedString(v.Logs), RestartPolicy: string(v.RestartPolicy), ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1057,6 +1060,7 @@ func MapServiceSpecKubeToAPI(v testworkflowsv1.ServiceSpec) testkube.TestWorkflo Logs: MapStringToBoxedString(v.Logs), RestartPolicy: string(v.RestartPolicy), ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1134,6 +1138,7 @@ func MapSpecKubeToAPI(v testworkflowsv1.TestWorkflowSpec) testkube.TestWorkflowS After: common.MapSlice(v.After, MapStepKubeToAPI), Events: common.MapSlice(v.Events, MapEventKubeToAPI), Execution: common.MapPtr(v.Execution, MapTestWorkflowTagSchemaKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1150,6 +1155,7 @@ func MapTemplateSpecKubeToAPI(v testworkflowsv1.TestWorkflowTemplateSpec) testku After: common.MapSlice(v.After, MapIndependentStepKubeToAPI), Events: common.MapSlice(v.Events, MapEventKubeToAPI), Execution: common.MapPtr(v.Execution, MapTestWorkflowTagSchemaKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1206,3 +1212,67 @@ func MapTestWorkflowTagSchemaKubeToAPI(v testworkflowsv1.TestWorkflowTagSchema) Tags: v.Tags, } } + +func MapTypeLocalObjectReferenceKubeToAPI(v corev1.TypedLocalObjectReference) testkube.TypedLocalObjectReference { + return testkube.TypedLocalObjectReference{ + ApiGroup: MapStringToBoxedString(v.APIGroup), + Kind: v.Kind, + Name: v.Name, + } +} + +func MapTypeObjectReferenceKubeToAPI(v corev1.TypedObjectReference) testkube.TypedObjectReference { + return testkube.TypedObjectReference{ + ApiGroup: MapStringToBoxedString(v.APIGroup), + Kind: v.Kind, + Name: v.Name, + Namespace: MapStringToBoxedString(v.Namespace), + } +} + +func MapVolumeResourceRequirementsKubeToAPI(v corev1.VolumeResourceRequirements) *testkube.TestWorkflowResources { + return &testkube.TestWorkflowResources{ + Limits: MapResourcesListKubeCoreToAPI(v.Limits), + Requests: MapResourcesListKubeCoreToAPI(v.Requests), + } +} + +func MapResourcesListKubeCoreToAPI(v corev1.ResourceList) *testkube.TestWorkflowResourcesList { + if len(v) == 0 { + return nil + } + + res := &testkube.TestWorkflowResourcesList{} + if q, ok := v[corev1.ResourceCPU]; ok { + res.Cpu = q.String() + } + + if q, ok := v[corev1.ResourceMemory]; ok { + res.Memory = q.String() + } + + if q, ok := v[corev1.ResourceStorage]; ok { + res.Storage = q.String() + } + + if q, ok := v[corev1.ResourceEphemeralStorage]; ok { + res.EphemeralStorage = q.String() + } + + return res +} + +func MapPvcConfigKubeToAPI(v corev1.PersistentVolumeClaimSpec) testkube.TestWorkflowPvcConfig { + return testkube.TestWorkflowPvcConfig{ + AccessModes: common.MapSlice(v.AccessModes, + func(v corev1.PersistentVolumeAccessMode) string { return (string)(v) }), + VolumeMode: MapStringToBoxedString((*string)(v.VolumeMode)), + Resources: MapVolumeResourceRequirementsKubeToAPI(v.Resources), + StorageClassName: MapStringToBoxedString(v.StorageClassName), + VolumeName: v.VolumeName, + Selector: common.MapPtr(v.Selector, MapSelectorToAPI), + DataSource: common.MapPtr(v.DataSource, MapTypeLocalObjectReferenceKubeToAPI), + DataSourceRef: common.MapPtr(v.DataSourceRef, MapTypeObjectReferenceKubeToAPI), + VolumeAttributesClassName: MapStringToBoxedString(v.VolumeAttributesClassName), + } +} diff --git a/pkg/mapper/testworkflows/openapi_kube.go b/pkg/mapper/testworkflows/openapi_kube.go index 86b9bf7ede..9523414b2d 100644 --- a/pkg/mapper/testworkflows/openapi_kube.go +++ b/pkg/mapper/testworkflows/openapi_kube.go @@ -951,6 +951,7 @@ func MapStepParallelAPIToKube(v testkube.TestWorkflowStepParallel) testworkflows Setup: common.MapSlice(v.Setup, MapStepAPIToKube), Steps: common.MapSlice(v.Steps, MapStepAPIToKube), After: common.MapSlice(v.After, MapStepAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), }, StepControl: testworkflowsv1.StepControl{ Paused: v.Paused, @@ -995,6 +996,7 @@ func MapIndependentStepParallelAPIToKube(v testkube.TestWorkflowIndependentStepP Setup: common.MapSlice(v.Setup, MapIndependentStepAPIToKube), Steps: common.MapSlice(v.Steps, MapIndependentStepAPIToKube), After: common.MapSlice(v.After, MapIndependentStepAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), }, StepControl: testworkflowsv1.StepControl{ Paused: v.Paused, @@ -1098,6 +1100,7 @@ func MapIndependentServiceSpecAPIToKube(v testkube.TestWorkflowIndependentServic Logs: MapBoxedStringToString(v.Logs), RestartPolicy: testworkflowsv1.ServiceRestartPolicy(v.RestartPolicy), ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), } } @@ -1134,6 +1137,7 @@ func MapServiceSpecAPIToKube(v testkube.TestWorkflowServiceSpec) testworkflowsv1 Logs: MapBoxedStringToString(v.Logs), RestartPolicy: testworkflowsv1.ServiceRestartPolicy(v.RestartPolicy), ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), }, } } @@ -1234,6 +1238,7 @@ func MapSpecAPIToKube(v testkube.TestWorkflowSpec) testworkflowsv1.TestWorkflowS Setup: common.MapSlice(v.Setup, MapStepAPIToKube), Steps: common.MapSlice(v.Steps, MapStepAPIToKube), After: common.MapSlice(v.After, MapStepAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), } } @@ -1253,6 +1258,7 @@ func MapTemplateSpecAPIToKube(v testkube.TestWorkflowTemplateSpec) testworkflows Setup: common.MapSlice(v.Setup, MapIndependentStepAPIToKube), Steps: common.MapSlice(v.Steps, MapIndependentStepAPIToKube), After: common.MapSlice(v.After, MapIndependentStepAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), } } @@ -1485,3 +1491,71 @@ func MapTestWorkflowTagSchemaAPIToKube(v testkube.TestWorkflowTagSchema) testwor Tags: v.Tags, } } + +func MapTypeLocalObjectReferenceAPIToKube(v testkube.TypedLocalObjectReference) corev1.TypedLocalObjectReference { + return corev1.TypedLocalObjectReference{ + APIGroup: MapBoxedStringToString(v.ApiGroup), + Kind: v.Kind, + Name: v.Name, + } +} + +func MapTypeObjectReferenceAPIToKube(v testkube.TypedObjectReference) corev1.TypedObjectReference { + return corev1.TypedObjectReference{ + APIGroup: MapBoxedStringToString(v.ApiGroup), + Kind: v.Kind, + Name: v.Name, + Namespace: MapBoxedStringToString(v.Namespace), + } +} + +func MapVolumeResourceRequirementsAPIToKube(v *testkube.TestWorkflowResources) corev1.VolumeResourceRequirements { + if v == nil { + return corev1.VolumeResourceRequirements{} + } + + return corev1.VolumeResourceRequirements{ + Limits: MapResourcesListAPIToKubeCore(v.Limits), + Requests: MapResourcesListAPIToKubeCore(v.Requests), + } +} + +func MapResourcesListAPIToKubeCore(v *testkube.TestWorkflowResourcesList) corev1.ResourceList { + if v == nil { + return nil + } + + res := make(map[corev1.ResourceName]resource.Quantity) + if v.Cpu != "" { + res[corev1.ResourceCPU], _ = resource.ParseQuantity(v.Cpu) + } + + if v.Memory != "" { + res[corev1.ResourceMemory], _ = resource.ParseQuantity(v.Memory) + } + + if v.Storage != "" { + res[corev1.ResourceStorage], _ = resource.ParseQuantity(v.Storage) + } + + if v.EphemeralStorage != "" { + res[corev1.ResourceEphemeralStorage], _ = resource.ParseQuantity(v.EphemeralStorage) + } + + return res +} + +func MapPvcConfigAPIToKube(v testkube.TestWorkflowPvcConfig) corev1.PersistentVolumeClaimSpec { + return corev1.PersistentVolumeClaimSpec{ + AccessModes: common.MapSlice(v.AccessModes, + func(v string) corev1.PersistentVolumeAccessMode { return (corev1.PersistentVolumeAccessMode)(v) }), + VolumeMode: (*corev1.PersistentVolumeMode)(MapBoxedStringToString(v.VolumeMode)), + Resources: MapVolumeResourceRequirementsAPIToKube(v.Resources), + StorageClassName: MapBoxedStringToString(v.StorageClassName), + VolumeName: v.VolumeName, + Selector: common.MapPtr(v.Selector, MapLabelSelectorAPIToKube), + DataSource: common.MapPtr(v.DataSource, MapTypeLocalObjectReferenceAPIToKube), + DataSourceRef: common.MapPtr(v.DataSourceRef, MapTypeObjectReferenceAPIToKube), + VolumeAttributesClassName: MapBoxedStringToString(v.VolumeAttributesClassName), + } +} diff --git a/pkg/testworkflows/executionworker/controller/cleanup.go b/pkg/testworkflows/executionworker/controller/cleanup.go index 1a6faa7d8b..a70d7194ee 100644 --- a/pkg/testworkflows/executionworker/controller/cleanup.go +++ b/pkg/testworkflows/executionworker/controller/cleanup.go @@ -57,6 +57,17 @@ func cleanupJobs(labelName string) func(ctx context.Context, clientSet kubernete } } +func cleanupPvcs(labelName string) func(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error { + return func(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error { + return clientSet.CoreV1().PersistentVolumeClaims(namespace).DeleteCollection(ctx, metav1.DeleteOptions{ + GracePeriodSeconds: common.Ptr(int64(0)), + PropagationPolicy: common.Ptr(metav1.DeletePropagationBackground), + }, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelName, id), + }) + } +} + func Cleanup(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error { var errs []error var errsMu sync.Mutex @@ -70,6 +81,8 @@ func Cleanup(ctx context.Context, clientSet kubernetes.Interface, namespace, id cleanupConfigMaps(constants.ResourceIdLabelName), cleanupSecrets(constants.RootResourceIdLabelName), cleanupSecrets(constants.ResourceIdLabelName), + cleanupPvcs(constants.RootResourceIdLabelName), + cleanupPvcs(constants.ResourceIdLabelName), } wg.Add(len(ops)) for _, op := range ops { @@ -96,6 +109,7 @@ func CleanupGroup(ctx context.Context, clientSet kubernetes.Interface, namespace cleanupPods(constants.GroupIdLabelName), cleanupConfigMaps(constants.GroupIdLabelName), cleanupSecrets(constants.GroupIdLabelName), + cleanupPvcs(constants.GroupIdLabelName), } wg.Add(len(ops)) for _, op := range ops { diff --git a/pkg/testworkflows/testworkflowconfig/config.go b/pkg/testworkflows/testworkflowconfig/config.go index 8f41f3c8cf..c3efe42bb9 100644 --- a/pkg/testworkflows/testworkflowconfig/config.go +++ b/pkg/testworkflows/testworkflowconfig/config.go @@ -22,6 +22,7 @@ type ExecutionConfig struct { OrganizationId string `json:"o,omitempty"` EnvironmentId string `json:"e,omitempty"` ParentIds string `json:"p,omitempty"` + PvcNames map[string]string `json:"c,omitempty"` } type WorkflowConfig struct { diff --git a/pkg/testworkflows/testworkflowconfig/expressions.go b/pkg/testworkflows/testworkflowconfig/expressions.go index 3cd5cd330b..b5d85df25c 100644 --- a/pkg/testworkflows/testworkflowconfig/expressions.go +++ b/pkg/testworkflows/testworkflowconfig/expressions.go @@ -1,6 +1,8 @@ package testworkflowconfig -import "github.com/kubeshop/testkube/pkg/expressions" +import ( + "github.com/kubeshop/testkube/pkg/expressions" +) func CreateExecutionMachine(cfg *ExecutionConfig) expressions.Machine { return expressions.NewMachine(). @@ -77,3 +79,12 @@ func CreateWorkerMachine(cfg *WorkerConfig) expressions.Machine { }) return expressions.CombinedMachines(machine) } + +func CreatePvcMachine(pvcNames map[string]string) expressions.Machine { + pvcMap := make(map[string]string) + for name, pvcName := range pvcNames { + pvcMap[name+".name"] = pvcName + } + + return expressions.NewMachine().RegisterStringMap("pvcs", pvcMap) +} diff --git a/pkg/testworkflows/testworkflowprocessor/bundle.go b/pkg/testworkflows/testworkflowprocessor/bundle.go index 84bfe8a694..fb49754391 100644 --- a/pkg/testworkflows/testworkflowprocessor/bundle.go +++ b/pkg/testworkflows/testworkflowprocessor/bundle.go @@ -27,6 +27,7 @@ type BundleOptions struct { type Bundle struct { Secrets []corev1.Secret ConfigMaps []corev1.ConfigMap + Pvcs []corev1.PersistentVolumeClaim Job batchv1.Job Signature []stage.Signature FullSignature []stage.Signature @@ -50,6 +51,9 @@ func (b *Bundle) SetGroupId(groupId string) { for i := range b.Secrets { AnnotateGroupId(&b.Secrets[i], groupId) } + for i := range b.Pvcs { + AnnotateGroupId(&b.Pvcs[i], groupId) + } } func (b *Bundle) Deploy(ctx context.Context, clientSet kubernetes.Interface, namespace string) (err error) { @@ -68,6 +72,13 @@ func (b *Bundle) Deploy(ctx context.Context, clientSet kubernetes.Interface, nam return errors.Wrap(err, "failed to deploy config maps") } } + for _, item := range b.Pvcs { + _, err = clientSet.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, &item, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "failed to deploy pvcs") + } + } + _, err = clientSet.BatchV1().Jobs(namespace).Create(ctx, &b.Job, metav1.CreateOptions{}) return errors.Wrap(err, "failed to deploy job") } diff --git a/pkg/testworkflows/testworkflowprocessor/intermediate.go b/pkg/testworkflows/testworkflowprocessor/intermediate.go index b92b268338..2432bfa98a 100644 --- a/pkg/testworkflows/testworkflowprocessor/intermediate.go +++ b/pkg/testworkflows/testworkflowprocessor/intermediate.go @@ -4,6 +4,7 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" @@ -21,9 +22,11 @@ type Intermediate interface { ConfigMaps() []corev1.ConfigMap Secrets() []corev1.Secret Volumes() []corev1.Volume + Pvcs() map[string]corev1.PersistentVolumeClaim AppendJobConfig(cfg *testworkflowsv1.JobConfig) Intermediate AppendPodConfig(cfg *testworkflowsv1.PodConfig) Intermediate + AppendPvcs(cfg map[string]corev1.PersistentVolumeClaimSpec) Intermediate AddConfigMap(configMap corev1.ConfigMap) Intermediate AddSecret(secret corev1.Secret) Intermediate @@ -47,8 +50,9 @@ type intermediate struct { Job testworkflowsv1.JobConfig `expr:"include"` // Actual Kubernetes resources to use - Secs []corev1.Secret `expr:"force"` - Cfgs []corev1.ConfigMap `expr:"force"` + Secs []corev1.Secret `expr:"force"` + Cfgs []corev1.ConfigMap `expr:"force"` + Ps map[string]corev1.PersistentVolumeClaim `expr:"force"` // Storing files Files ConfigMapFiles `expr:"include"` @@ -60,7 +64,9 @@ func NewIntermediate() Intermediate { RefCounter: ref, Root: stage.NewGroupStage("", true), Container: stage.NewContainer(), - Files: NewConfigMapFiles(fmt.Sprintf("{{resource.id}}-%s", ref.NextRef()), nil)} + Files: NewConfigMapFiles(fmt.Sprintf("{{resource.id}}-%s", ref.NextRef()), nil), + Ps: make(map[string]corev1.PersistentVolumeClaim), + } } func (s *intermediate) ContainerDefaults() stage.Container { @@ -87,6 +93,10 @@ func (s *intermediate) Volumes() []corev1.Volume { return append(s.Pod.Volumes, s.Files.Volumes()...) } +func (s *intermediate) Pvcs() map[string]corev1.PersistentVolumeClaim { + return s.Ps +} + func (s *intermediate) AppendJobConfig(cfg *testworkflowsv1.JobConfig) Intermediate { s.Job = *testworkflowresolver.MergeJobConfig(&s.Job, cfg) return s @@ -97,6 +107,18 @@ func (s *intermediate) AppendPodConfig(cfg *testworkflowsv1.PodConfig) Intermedi return s } +func (s *intermediate) AppendPvcs(cfg map[string]corev1.PersistentVolumeClaimSpec) Intermediate { + for name, spec := range cfg { + s.Ps[name] = corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("{{resource.root}}-%s", s.NextRef()), + }, + Spec: spec, + } + } + return s +} + func (s *intermediate) AddVolume(volume corev1.Volume) Intermediate { s.Pod.Volumes = append(s.Pod.Volumes, volume) return s diff --git a/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go b/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go index 119db5cbdb..a5627ed0d5 100644 --- a/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go +++ b/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go @@ -150,6 +150,20 @@ func (mr *MockIntermediateMockRecorder) AppendPodConfig(arg0 interface{}) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendPodConfig", reflect.TypeOf((*MockIntermediate)(nil).AppendPodConfig), arg0) } +// AppendPvc mocks base method. +func (m *MockIntermediate) AppendPvc(arg0 map[string]v10.PersistentVolumeClaimSpec) Intermediate { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendPvc", arg0) + ret0, _ := ret[0].(Intermediate) + return ret0 +} + +// AppendPvc indicates an expected call of AppendPvc. +func (mr *MockIntermediateMockRecorder) AppendPvc(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendPvc", reflect.TypeOf((*MockIntermediate)(nil).AppendPvc), arg0) +} + // ConfigMaps mocks base method. func (m *MockIntermediate) ConfigMaps() []v10.ConfigMap { m.ctrl.T.Helper() @@ -220,6 +234,20 @@ func (mr *MockIntermediateMockRecorder) PodConfig() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PodConfig", reflect.TypeOf((*MockIntermediate)(nil).PodConfig)) } +// Pvcs mocks base method. +func (m *MockIntermediate) Pvcs() []v10.PersistentVolumeClaim { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pvcs") + ret0, _ := ret[0].([]v10.PersistentVolumeClaim) + return ret0 +} + +// Pvcs indicates an expected call of Pvcs. +func (mr *MockIntermediateMockRecorder) Pvcs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pvcs", reflect.TypeOf((*MockIntermediate)(nil).Pvcs)) +} + // Secrets mocks base method. func (m *MockIntermediate) Secrets() []v10.Secret { m.ctrl.T.Helper() diff --git a/pkg/testworkflows/testworkflowprocessor/processor.go b/pkg/testworkflows/testworkflowprocessor/processor.go index 1a0d460a69..049738434e 100644 --- a/pkg/testworkflows/testworkflowprocessor/processor.go +++ b/pkg/testworkflows/testworkflowprocessor/processor.go @@ -106,7 +106,8 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo // Initialize intermediate layer layer := NewIntermediate(). AppendPodConfig(workflow.Spec.Pod). - AppendJobConfig(workflow.Spec.Job) + AppendJobConfig(workflow.Spec.Job). + AppendPvcs(workflow.Spec.Pvcs) layer.ContainerDefaults(). ApplyCR(constants.DefaultContainerConfig.DeepCopy()). AppendVolumeMounts(layer.AddEmptyDirVolume(nil, constants.DefaultInternalPath)). @@ -117,7 +118,9 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo machines = append(machines, createSecretMachine(mapEnv), testworkflowconfig.CreateWorkerMachine(&options.Config.Worker), - testworkflowconfig.CreateResourceMachine(&options.Config.Resource)) + testworkflowconfig.CreateResourceMachine(&options.Config.Resource), + testworkflowconfig.CreatePvcMachine(common.MapMap(layer.Pvcs(), func(v corev1.PersistentVolumeClaim) string { return v.Name })), + ) // Fetch resource root and resource ID if options.Config.Resource.Id == "" { @@ -162,6 +165,27 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo } } + // Finalize Pvcs + pvcs := make([]corev1.PersistentVolumeClaim, 0) + mapPvc := make(map[string]string) + for name, spec := range layer.Pvcs() { + AnnotateControlledBy(&spec, options.Config.Resource.RootId, options.Config.Resource.Id) + err = expressions.FinalizeForce(&spec, machines...) + if err != nil { + return nil, errors.Wrap(err, "finalizing Pvc") + } + + data := struct{ value string }{value: name} + err = expressions.FinalizeForce(&data, machines...) + if err != nil { + return nil, errors.Wrap(err, "finalizing name") + } + + mapPvc[data.value] = spec.Name + pvcs = append(pvcs, spec) + } + options.Config.Execution.PvcNames = common.MergeMaps(options.Config.Execution.PvcNames, mapPvc) + // Finalize Secrets secrets := append(layer.Secrets(), options.Secrets...) for i := range secrets { @@ -420,6 +444,7 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo bundle = &Bundle{ ConfigMaps: configMaps, Secrets: secrets, + Pvcs: pvcs, Job: jobSpec, Signature: sig, FullSignature: fullSig, diff --git a/pkg/testworkflows/testworkflowresolver/apply.go b/pkg/testworkflows/testworkflowresolver/apply.go index a3afdc9ea8..07ee151165 100644 --- a/pkg/testworkflows/testworkflowresolver/apply.go +++ b/pkg/testworkflows/testworkflowresolver/apply.go @@ -63,6 +63,7 @@ func injectTemplateToSpec(spec *testworkflowsv1.TestWorkflowSpec, template testw spec.Job = MergeJobConfig(template.Spec.Job, spec.Job) spec.Events = append(template.Spec.Events, spec.Events...) spec.Execution = MergeExecution(template.Spec.Execution, spec.Execution) + spec.Pvcs = MergeMap(template.Spec.Pvcs, spec.Pvcs) // Apply basic configuration spec.Content = MergeContent(template.Spec.Content, spec.Content) @@ -125,6 +126,7 @@ func InjectServiceTemplate(svc *testworkflowsv1.ServiceSpec, template testworkfl svc.Pod = MergePodConfig(template.Spec.Pod, svc.Pod) svc.Content = MergeContent(template.Spec.Content, svc.Content) svc.ContainerConfig = *MergeContainerConfig(template.Spec.Container, &svc.ContainerConfig) + svc.Pvcs = MergeMap(template.Spec.Pvcs, svc.Pvcs) return nil } diff --git a/pkg/testworkflows/testworkflowresolver/merge.go b/pkg/testworkflows/testworkflowresolver/merge.go index 1d503f1b84..cdf602ca43 100644 --- a/pkg/testworkflows/testworkflowresolver/merge.go +++ b/pkg/testworkflows/testworkflowresolver/merge.go @@ -373,6 +373,7 @@ func ConvertIndependentStepParallelToStepParallel(step testworkflowsv1.Independe Setup: common.MapSlice(step.TestWorkflowTemplateSpec.Setup, ConvertIndependentStepToStep), Steps: common.MapSlice(step.TestWorkflowTemplateSpec.Steps, ConvertIndependentStepToStep), After: common.MapSlice(step.TestWorkflowTemplateSpec.After, ConvertIndependentStepToStep), + Pvcs: step.TestWorkflowTemplateSpec.Pvcs, }, StepControl: step.StepControl, StepOperations: step.StepOperations,