From 2c5ade1b9a4f3f0ae0d98649faeaeb1233c6be6e Mon Sep 17 00:00:00 2001 From: Alessandro Cattapan <31108344+alemax22@users.noreply.github.com> Date: Wed, 10 Jul 2024 07:47:25 +0200 Subject: [PATCH] Added pod affinity support (#741) * Added affinity to pod of clustering service * Added affinity to pod of hz deployment * Update the documentation * Added pod affinity to tiers of pega and updated documentation accordingly * Added pod affinity to install/upgrade jobs of pega and updated documentation accordingly * Added affinity to pegasearch deployment and updated documentation * Added pod affinity to constellation and updated documentation * Added pod affinity to constellation messaging service and updated documentation * Added pod affinity to srs service and updated documentation * Aligned all supplemental.tpl files * Removed useless comments in template definition and aligned comment in top part of _supplemental.tpl files * Added Test for affinity on pega tiers * Rearrange test code * Fixed conflicts * Aligned go files * remove obsolete logger (#746) * remove obsolete logger * update expected contents * Add support for custom dns policy setting of ndots value (#757) * support configuration of ndots * quote ndots to avoid json unmarshall error --------- Co-authored-by: Bachir Berrachedi Co-authored-by: Adam Talbot * Update chart versions to 3.21.0 * Bug 868785 : Replication factor value updated to 3. (#762) * BUG-868785 Update values.yaml with replication factor 3 Pega recommended replication factor is 3 --------- Co-authored-by: MadhuriArugula * US-612844: Added toleration configuration for workloads (#758) * US-612844: Added toleration configuration for workloads * US-612844: Updated values.yaml with toleration configuration * US-612844: Updated tolerations configurations in Readme --------- Co-authored-by: MadhuriArugula * US-614190: Stale action workflow to handle for github repo issues & PR's with no activity (#756) * US-614190: Stale action workflow to handle for github repo issues & pull requests with no activity for more than 60 days * fixed review comments * fixed review comments --------- Co-authored-by: rajuu * Fixed merge * Update chart versions to 3.22.0 * Removing conditional statements for LRW (#768) * removing conditional statements for LRW * Keeping indenting consistent --------- Co-authored-by: locoe * Update chart versions to 3.23.0 * BUG-870872 - Update SRS version to 1.31.2 (#760) Co-authored-by: MadhuriArugula * BUG-867426 : Hazelcast PDB corrected (#773) Co-authored-by: vermn1 * [US-624017] Adding JDBC Connection information (#777) * [US-624017] Adding JDBC Connection information * Edited wording following SME suggestion * Fixed merge * Fixed merge * Fixed merge * Fixed conglicts * Added Unit Test for Pegasearch deployment * Added unit test for affinity of install chart * Added Unit Tests for affinity in hazelcast and clustering service * Added Unit Test for affinity of SRS deployment * Added Unit Test of Affinity for constellation and constellation messaging charts --------- Co-authored-by: Kishor Kumar Vasantala Co-authored-by: Alessandro Cattapan Co-authored-by: Andrew Roskuski Co-authored-by: bachirgit Co-authored-by: Bachir Berrachedi Co-authored-by: Adam Talbot Co-authored-by: APegaDavis Co-authored-by: Narayana Raju <59383142+raju119@users.noreply.github.com> Co-authored-by: MadhuriArugula Co-authored-by: PEGA-NarasimhaRao-Meda <109585795+PEGA-NarasimhaRao-Meda@users.noreply.github.com> Co-authored-by: Uma Veera Co-authored-by: rajuu Co-authored-by: MadhuriArugula Co-authored-by: Evan1oconto Co-authored-by: locoe Co-authored-by: Evan1oconto Co-authored-by: Kinga Kowalska <120555574+kingakowalska1@users.noreply.github.com> Co-authored-by: vnihal72 <79415342+vnihal72@users.noreply.github.com> Co-authored-by: vermn1 --- .../charts/constellation-messaging/README.md | 1 + .../templates/_helpers.tpl | 6 + .../templates/messaging-deployment.yaml | 1 + .../charts/constellation/README.md | 1 + .../constellation/templates/_helpers.tpl | 7 + .../templates/clln-deployment.yaml | 1 + charts/backingservices/charts/srs/README.md | 2 +- .../charts/srs/templates/_helpers.tpl | 7 + .../srs/templates/srsservice_deployment.yaml | 1 + charts/pega/README.md | 23 ++ .../hazelcast/templates/_supplemental.tpl | 10 +- .../clustering-service-deployment.yaml | 1 + .../templates/pega-hz-deployment.yaml | 1 + .../templates/_pega-installer-job.tpl | 1 + .../installer/templates/_supplemental.tpl | 10 +- .../pegasearch/templates/_supplemental.tpl | 10 +- .../templates/pega-search-deployment.yaml | 1 + charts/pega/templates/_helpers.tpl | 2 +- charts/pega/templates/_pega-deployment.tpl | 1 + charts/pega/templates/_supplemental.tpl | 10 +- ...constellation-messaging-deployment_test.go | 35 +++ .../constellation-static-depoyment_test.go | 34 +++ .../backingservices/srs-deployment_test.go | 246 ++++++++++-------- .../clustering-service-deployment_test.go | 42 +++ .../src/test/pega/pega-hz-deployment_test.go | 56 +++- .../src/test/pega/pega-installer-job_test.go | 33 +++ .../test/pega/pega-search-deployment_test.go | 101 +++++-- .../test/pega/pega-tier-deployment_test.go | 43 +++ 28 files changed, 551 insertions(+), 136 deletions(-) create mode 100644 charts/backingservices/charts/constellation-messaging/templates/_helpers.tpl create mode 100644 terratest/src/test/backingservices/constellation-messaging-deployment_test.go create mode 100644 terratest/src/test/backingservices/constellation-static-depoyment_test.go diff --git a/charts/backingservices/charts/constellation-messaging/README.md b/charts/backingservices/charts/constellation-messaging/README.md index 492ecbf97..ded3b2f74 100644 --- a/charts/backingservices/charts/constellation-messaging/README.md +++ b/charts/backingservices/charts/constellation-messaging/README.md @@ -17,4 +17,5 @@ Complete information on the design of the service including architecture, scalab | `imagePullSecretNames` | List pre-existing secrets to be used for pulling docker images. | | `pegaMessagingPort` | Defines the port used by the Service. | | `pegaMessagingTargetPort` | Defines the port used by the Pod and Container. | +| `affinity` | Define pod affinity so that it is restricted to run on particular node(s), or to prefer to run on particular nodes. | | `ingress` | Allows optional configuration of a domain name, ingressClass, and annotations. An ingress will be provisioned if a domain name is supplied. Due to the diversity of network configurations, ingress vendors, and TLS requirements it may be necessary to define your ingress separately from this chart. diff --git a/charts/backingservices/charts/constellation-messaging/templates/_helpers.tpl b/charts/backingservices/charts/constellation-messaging/templates/_helpers.tpl new file mode 100644 index 000000000..8ba76dfa7 --- /dev/null +++ b/charts/backingservices/charts/constellation-messaging/templates/_helpers.tpl @@ -0,0 +1,6 @@ +{{- define "podAffinity" }} +{{- if .affinity }} +affinity: +{{- toYaml .affinity | nindent 2 }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/backingservices/charts/constellation-messaging/templates/messaging-deployment.yaml b/charts/backingservices/charts/constellation-messaging/templates/messaging-deployment.yaml index 8041926fa..d76b3064a 100644 --- a/charts/backingservices/charts/constellation-messaging/templates/messaging-deployment.yaml +++ b/charts/backingservices/charts/constellation-messaging/templates/messaging-deployment.yaml @@ -37,4 +37,5 @@ spec: - path=/c11n-messaging ports: - containerPort: {{ .Values.pegaMessagingTargetPort }} +{{- include "podAffinity" .Values | indent 6 }} {{ end }} diff --git a/charts/backingservices/charts/constellation/README.md b/charts/backingservices/charts/constellation/README.md index 0d94b16cc..d999dc80a 100644 --- a/charts/backingservices/charts/constellation/README.md +++ b/charts/backingservices/charts/constellation/README.md @@ -48,6 +48,7 @@ The values.yaml file provides configuration options to define the values for the | `ingressAnnotations` | Specify additional annotations to add to the ingress. | | `customerAssetVolumeClaimName` | Specify the volume claim name to be used for storing customer assets. | | `imagePullSecretNames` | Specify a list of existing ImagePullSecrets to be added to the Deployment. | +| `affinity` | Specify the pod affinity so that pods are restricted to run on particular node(s), or to prefer to run on particular nodes. | `docker`.`registry`.`url` | Specify the image registry url. | | `docker`.`registry`.`username` | Specify the username for the docker registry. | | `docker`.`registry`.`password` | Specify the password for the docker registry. | diff --git a/charts/backingservices/charts/constellation/templates/_helpers.tpl b/charts/backingservices/charts/constellation/templates/_helpers.tpl index 829116a44..91da6041d 100644 --- a/charts/backingservices/charts/constellation/templates/_helpers.tpl +++ b/charts/backingservices/charts/constellation/templates/_helpers.tpl @@ -8,3 +8,10 @@ {{- end }} {{- define "deploymentName" }}{{ $deploymentNamePrefix := "constellation" }}{{ if (.Values.deployment) }}{{ if (.Values.deployment.name) }}{{ $deploymentNamePrefix = .Values.deployment.name }}{{ end }}{{ end }}{{ $deploymentNamePrefix }}{{- end }} + +{{- define "podAffinity" }} +{{- if .affinity }} +affinity: +{{- toYaml .affinity | nindent 2 }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/backingservices/charts/constellation/templates/clln-deployment.yaml b/charts/backingservices/charts/constellation/templates/clln-deployment.yaml index f32e3ec59..ac27fe34b 100644 --- a/charts/backingservices/charts/constellation/templates/clln-deployment.yaml +++ b/charts/backingservices/charts/constellation/templates/clln-deployment.yaml @@ -69,4 +69,5 @@ spec: port: 3000 ports: - containerPort: 3000 +{{- include "podAffinity" .Values | indent 6 }} {{ end }} \ No newline at end of file diff --git a/charts/backingservices/charts/srs/README.md b/charts/backingservices/charts/srs/README.md index 6562cf24e..341dc8b3b 100644 --- a/charts/backingservices/charts/srs/README.md +++ b/charts/backingservices/charts/srs/README.md @@ -108,7 +108,7 @@ To deploy Pega Platform with the SRS backing service, the SRS helm chart require |-----------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `enabled` | Enable the Search and Reporting Service deployment as a backing service. Set this parameter to `true` to use SRS. | | `deploymentName` | Specify the name of your SRS cluster. Your deployment creates resources prefixed with this string. This is also the service name for the SRS. | -| `srsRuntime` | Use this section to define specific resource configuration options like image, replica count, cpu and memory resource settings in the SRS. The default minimum required number of replicas is 2, but as a best practice, deploy 3 replicas to maintain high availability. | +| `srsRuntime` | Use this section to define specific resource configuration options like image, replica count, pod affinity, cpu and memory resource settings in the SRS. The default minimum required number of replicas is 2, but as a best practice, deploy 3 replicas to maintain high availability. | | `busybox` | When provisioning an internally managed Elasticsearch cluster, you can customize the location and pull policy of the Alpine image used during the deployment process by specifying `busybox.image` and `busybox.imagePullPolicy`. | | `elasticsearch` | Define the elasticsearch cluster configurations. The [Elasticsearch](https://github.com/helm/charts/tree/master/stable/elasticsearch/values.yaml) chart defines the values for Elasticsearch provisioning in the SRS cluster. For internally provisioned Elasticsearch the default version is set to `7.17.9`. Set the `elasticsearch.imageTag` parameter in values.yaml to `7.16.3` to use this supported version in the SRS cluster. | | `k8sProvider` | Specify your Kubernetes provider name. Supported values are [`eks`, `aks`, `minikube`, `gke`, `openshift`, `pks`].. diff --git a/charts/backingservices/charts/srs/templates/_helpers.tpl b/charts/backingservices/charts/srs/templates/_helpers.tpl index b1336ccb3..55f6cc2ae 100644 --- a/charts/backingservices/charts/srs/templates/_helpers.tpl +++ b/charts/backingservices/charts/srs/templates/_helpers.tpl @@ -222,3 +222,10 @@ Network policy: `openshift-dns` for openshift cluster, `kube-dns | core-dns` for port: 8080 {{- end -}} {{- end -}} + +{{- define "podAffinity" }} +{{- if .affinity }} +affinity: +{{- toYaml .affinity | nindent 2 }} +{{- end }} +{{ end }} diff --git a/charts/backingservices/charts/srs/templates/srsservice_deployment.yaml b/charts/backingservices/charts/srs/templates/srsservice_deployment.yaml index 5bc225ca3..fb1470588 100644 --- a/charts/backingservices/charts/srs/templates/srsservice_deployment.yaml +++ b/charts/backingservices/charts/srs/templates/srsservice_deployment.yaml @@ -147,4 +147,5 @@ spec: name: srs-elastic-credentials key: password {{ end }} +{{- include "podAffinity" .Values.srsRuntime | indent 6 }} {{ end }} \ No newline at end of file diff --git a/charts/pega/README.md b/charts/pega/README.md index 7a0f3aba4..28933afb3 100644 --- a/charts/pega/README.md +++ b/charts/pega/README.md @@ -709,6 +709,26 @@ tier: : ``` +### Pod affinity + +You may optionally configure the pod affinity so that it is restricted to run on particular node(s), or to prefer to run on particular nodes. Pod affinity may be specified by using the `affinity` element for a given `tier`. See the official [Kubernetes Documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). + +Example: + +```yaml +tier: + - name: my-tier + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux +``` + ### Pega configuration files While Pega includes default configuration files in the Helm charts, the charts provide extension points to override the defaults with additional customizations. To change the configuration file, specify the replacement implementation to be injected into a ConfigMap. @@ -961,6 +981,7 @@ Parameter | Description | Default value `set_vm_max_map_count` | Elasticsearch uses a **mmapfs** directory by default to store its indices. The default operating system limits on mmap counts is likely to be too low, which may result in out of memory exceptions. An init container is provided to set the value correctly, but this action requires privileged access. If privileged access is not allowed in your environment, you may increase this setting manually by updating the `vm.max_map_count` setting in **/etc/sysctl.conf** according to the Elasticsearch documentation and can set this parameter to `false` to disable the init container. For more information, see the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html). | `true` `set_data_owner_on_startup` | Set to true to enable an init container that runs a chown command on the mapped volume at startup to reset the owner of the ES data to the current user. This is needed if a random user is used to run the pod, but also requires privileges to change the ownership of files. | `false` `podAnnotations` | Configurable annotations applied to all Elasticsearch pods. | {} +`affinity` | You may optionally configure the pod affinity so that it is restricted to run on particular node(s), or to prefer to run on particular nodes. | `""` Additional env settings supported by Elasticsearch may be specified in a `custom.env` block as shown in the example below. @@ -1064,6 +1085,7 @@ Parameter | Description | Default value `image` | Reference the `platform/installer` Docker image that you downloaded and pushed to your Docker registry that your deployment can access. | `YOUR_INSTALLER_IMAGE:TAG` `imagePullPolicy` | Specify when to pull an image. | `IfNotPresent` `adminPassword` | Specify a temporary, initial password to log into the Pega application. This will need to be changed at first login. The adminPassword value cannot start with "@". | `"ADMIN_PASSWORD"` +`affinity` | Configures policy to assign the pods to the nodes. See the official [Kubernetes Documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). | `""` `upgrade.upgradeType:` |Specify the type of process, applying a patch or upgrading. | See the next table for details. `upgrade.upgradeSteps:` |Specify the steps of a `custom` upgrade process that you want to complete. For `zero-downtime`, `out-of-place-rules`, `out-of-place-data`, or `in-place` upgrades, leave this parameter empty. |
    `enable_cluster_upgrade` `rules_migration` `rules_upgrade` `data_upgrade` `disable_cluster_upgrade`
`upgrade.targetRulesSchema:` |Specify the name of the schema you created the process creates for the new rules schema. | `""` @@ -1223,6 +1245,7 @@ Parameter | Description | Default value `hazelcast.username` | Configures the username to be used in a client-server Hazelcast model for authentication between the nodes in the Pega deployment and the nodes in the Hazelcast cluster. This parameter configures the username in Hazelcast cluster and your Pega nodes so authentication occurs automatically. | `""` `hazelcast.password` | Configures the password to be used in a client-server Hazelcast model for authentication between the nodes in the Pega deployment and the nodes in the Hazelcast cluster. This parameter configures the password credential in Hazelcast cluster and your Pega nodes so authentication occurs automatically. | `""` `hazelcast.external_secret_name` | If you configured a secret in an external secrets operator, enter the secret name. For details, see [this section](#optional-support-for-providing-credentialscertificates-using-external-secrets-operator). | `""` +`hazelcast.affinity` | Configures policy to assign the pods to the nodes. See the official [Kubernetes Documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). | `""` #### Example ```yaml diff --git a/charts/pega/charts/hazelcast/templates/_supplemental.tpl b/charts/pega/charts/hazelcast/templates/_supplemental.tpl index 20e01f3ef..01add432e 100644 --- a/charts/pega/charts/hazelcast/templates/_supplemental.tpl +++ b/charts/pega/charts/hazelcast/templates/_supplemental.tpl @@ -11,6 +11,7 @@ pega-db-secret-name pega-hz-secret-name deployDBSecret deployNonExtDBSecret +podAffinity secretResolver are copied from pega/templates/_helpers.tpl because helm lint requires charts to render standalone. See: https://github.com/helm/helm/issues/11260 for more details. */}} @@ -115,4 +116,11 @@ false name: {{ .extSecretName }} {{- end -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{- define "podAffinity" }} +{{- if .affinity }} +affinity: +{{- toYaml .affinity | nindent 2 }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/pega/charts/hazelcast/templates/clustering-service-deployment.yaml b/charts/pega/charts/hazelcast/templates/clustering-service-deployment.yaml index 789dfbeb3..5a593bdfc 100644 --- a/charts/pega/charts/hazelcast/templates/clustering-service-deployment.yaml +++ b/charts/pega/charts/hazelcast/templates/clustering-service-deployment.yaml @@ -73,4 +73,5 @@ spec: {{- include "hazelcastVolumeTemplate" . | indent 6 }} imagePullSecrets: {{- include "imagePullSecrets" . | indent 6 }} +{{- include "podAffinity" .Values | indent 6 }} {{ end }} diff --git a/charts/pega/charts/hazelcast/templates/pega-hz-deployment.yaml b/charts/pega/charts/hazelcast/templates/pega-hz-deployment.yaml index 3abb7e813..494c116ca 100644 --- a/charts/pega/charts/hazelcast/templates/pega-hz-deployment.yaml +++ b/charts/pega/charts/hazelcast/templates/pega-hz-deployment.yaml @@ -69,4 +69,5 @@ spec: {{- include "hazelcastVolumeTemplate" . | indent 6 }} imagePullSecrets: {{- include "imagePullSecrets" . | indent 6 }} +{{- include "podAffinity" .Values | indent 6 }} {{ end }} diff --git a/charts/pega/charts/installer/templates/_pega-installer-job.tpl b/charts/pega/charts/installer/templates/_pega-installer-job.tpl index 726d27e9d..9e025365f 100644 --- a/charts/pega/charts/installer/templates/_pega-installer-job.tpl +++ b/charts/pega/charts/installer/templates/_pega-installer-job.tpl @@ -182,5 +182,6 @@ spec: restartPolicy: Never imagePullSecrets: {{- include "imagePullSecrets" .root | indent 6 }} +{{- include "podAffinity" .root.Values | indent 6 }} --- {{- end -}} \ No newline at end of file diff --git a/charts/pega/charts/installer/templates/_supplemental.tpl b/charts/pega/charts/installer/templates/_supplemental.tpl index 20e01f3ef..01add432e 100644 --- a/charts/pega/charts/installer/templates/_supplemental.tpl +++ b/charts/pega/charts/installer/templates/_supplemental.tpl @@ -11,6 +11,7 @@ pega-db-secret-name pega-hz-secret-name deployDBSecret deployNonExtDBSecret +podAffinity secretResolver are copied from pega/templates/_helpers.tpl because helm lint requires charts to render standalone. See: https://github.com/helm/helm/issues/11260 for more details. */}} @@ -115,4 +116,11 @@ false name: {{ .extSecretName }} {{- end -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{- define "podAffinity" }} +{{- if .affinity }} +affinity: +{{- toYaml .affinity | nindent 2 }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/pega/charts/pegasearch/templates/_supplemental.tpl b/charts/pega/charts/pegasearch/templates/_supplemental.tpl index 20e01f3ef..01add432e 100644 --- a/charts/pega/charts/pegasearch/templates/_supplemental.tpl +++ b/charts/pega/charts/pegasearch/templates/_supplemental.tpl @@ -11,6 +11,7 @@ pega-db-secret-name pega-hz-secret-name deployDBSecret deployNonExtDBSecret +podAffinity secretResolver are copied from pega/templates/_helpers.tpl because helm lint requires charts to render standalone. See: https://github.com/helm/helm/issues/11260 for more details. */}} @@ -115,4 +116,11 @@ false name: {{ .extSecretName }} {{- end -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{- define "podAffinity" }} +{{- if .affinity }} +affinity: +{{- toYaml .affinity | nindent 2 }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/pega/charts/pegasearch/templates/pega-search-deployment.yaml b/charts/pega/charts/pegasearch/templates/pega-search-deployment.yaml index 389bff149..055f81a89 100644 --- a/charts/pega/charts/pegasearch/templates/pega-search-deployment.yaml +++ b/charts/pega/charts/pegasearch/templates/pega-search-deployment.yaml @@ -117,6 +117,7 @@ spec: mountPath: /usr/share/elasticsearch/data imagePullSecrets: {{- include "imagePullSecrets" . | indent 6 }} +{{- include "podAffinity" .Values | indent 6 }} volumeClaimTemplates: - metadata: name: esstorage diff --git a/charts/pega/templates/_helpers.tpl b/charts/pega/templates/_helpers.tpl index 5f3597921..74377ad14 100644 --- a/charts/pega/templates/_helpers.tpl +++ b/charts/pega/templates/_helpers.tpl @@ -530,4 +530,4 @@ servicePort: use-annotation - secret: name: {{ include "pega-diagnostic-secret-name" $}} -{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/pega/templates/_pega-deployment.tpl b/charts/pega/templates/_pega-deployment.tpl index 2266ce947..7be6b9ea4 100644 --- a/charts/pega/templates/_pega-deployment.tpl +++ b/charts/pega/templates/_pega-deployment.tpl @@ -337,6 +337,7 @@ spec: # If the image is in a protected registry, you must specify a secret to access it. imagePullSecrets: {{- include "imagePullSecrets" .root | indent 6 }} +{{- include "podAffinity" .node | indent 6 }} {{- if (.node.volumeClaimTemplate) }} volumeClaimTemplates: - metadata: diff --git a/charts/pega/templates/_supplemental.tpl b/charts/pega/templates/_supplemental.tpl index 20e01f3ef..01add432e 100644 --- a/charts/pega/templates/_supplemental.tpl +++ b/charts/pega/templates/_supplemental.tpl @@ -11,6 +11,7 @@ pega-db-secret-name pega-hz-secret-name deployDBSecret deployNonExtDBSecret +podAffinity secretResolver are copied from pega/templates/_helpers.tpl because helm lint requires charts to render standalone. See: https://github.com/helm/helm/issues/11260 for more details. */}} @@ -115,4 +116,11 @@ false name: {{ .extSecretName }} {{- end -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{- define "podAffinity" }} +{{- if .affinity }} +affinity: +{{- toYaml .affinity | nindent 2 }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/terratest/src/test/backingservices/constellation-messaging-deployment_test.go b/terratest/src/test/backingservices/constellation-messaging-deployment_test.go new file mode 100644 index 000000000..868a8dba6 --- /dev/null +++ b/terratest/src/test/backingservices/constellation-messaging-deployment_test.go @@ -0,0 +1,35 @@ +package backingservices + +import ( + "testing" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" +) + +func TestConstellationMessagingDeploymentWithAffinity(t *testing.T) { + + var affintiyBasePath = "constellation-messaging.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]." + + helmChartParser := NewHelmConfigParser( + NewHelmTestFromTemplate(t, helmChartRelativePath, map[string]string{ + "constellation-messaging.enabled": "true", + "constellation-messaging.name": "constellation-messaging", + affintiyBasePath + "key": "kubernetes.io/os", + affintiyBasePath + "operator": "In", + affintiyBasePath + "values[0]": "linux", + }, + []string{"charts/constellation-messaging/templates/messaging-deployment.yaml"}), + ) + + var cllnMessagingDeploymentObj appsv1.Deployment + helmChartParser.getResourceYAML(SearchResourceOption{ + Name: "constellation-messaging", + Kind: "Deployment", + }, &cllnMessagingDeploymentObj) + + deploymentAffinity := cllnMessagingDeploymentObj.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + require.Equal(t, "kubernetes.io/os", deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key) + require.Equal(t, "In", string(deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator)) + require.Equal(t, "linux", deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0]) +} diff --git a/terratest/src/test/backingservices/constellation-static-depoyment_test.go b/terratest/src/test/backingservices/constellation-static-depoyment_test.go new file mode 100644 index 000000000..bb8ee18ac --- /dev/null +++ b/terratest/src/test/backingservices/constellation-static-depoyment_test.go @@ -0,0 +1,34 @@ +package backingservices + +import ( + "testing" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" +) + +func TestConstellationStaticDeploymentWithAffinity(t *testing.T) { + + var affintiyBasePath = "constellation.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]." + + helmChartParser := NewHelmConfigParser( + NewHelmTestFromTemplate(t, helmChartRelativePath, map[string]string{ + "constellation.enabled": "true", + affintiyBasePath + "key": "kubernetes.io/os", + affintiyBasePath + "operator": "In", + affintiyBasePath + "values[0]": "linux", + }, + []string{"charts/constellation/templates/clln-deployment.yaml"}), + ) + + var cllnDeploymentObj appsv1.Deployment + helmChartParser.getResourceYAML(SearchResourceOption{ + Name: "constellation", + Kind: "Deployment", + }, &cllnDeploymentObj) + + deploymentAffinity := cllnDeploymentObj.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + require.Equal(t, "kubernetes.io/os", deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key) + require.Equal(t, "In", string(deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator)) + require.Equal(t, "linux", deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0]) +} diff --git a/terratest/src/test/backingservices/srs-deployment_test.go b/terratest/src/test/backingservices/srs-deployment_test.go index b7c046254..482068615 100644 --- a/terratest/src/test/backingservices/srs-deployment_test.go +++ b/terratest/src/test/backingservices/srs-deployment_test.go @@ -1,29 +1,30 @@ package backingservices import ( + "strings" + "testing" + "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" k8score "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" - "testing" - "strings" ) -func TestSRSDeployment(t *testing.T){ +func TestSRSDeployment(t *testing.T) { helmChartParser := NewHelmConfigParser( NewHelmTestFromTemplate(t, helmChartRelativePath, map[string]string{ - "srs.enabled": "true", - "srs.deploymentName": "test-srs", - "global.imageCredentials.registry": "docker-registry.io", - "srs.srsRuntime.replicaCount": "1", - "srs.srsRuntime.srsImage": "platform-services/search-n-reporting-service:latest", - "srs.srsRuntime.env.AuthEnabled": "false", - "srs.srsRuntime.env.OAuthPublicKeyURL": "", - "srs.srsStorage.tls.enabled": "true", + "srs.enabled": "true", + "srs.deploymentName": "test-srs", + "global.imageCredentials.registry": "docker-registry.io", + "srs.srsRuntime.replicaCount": "1", + "srs.srsRuntime.srsImage": "platform-services/search-n-reporting-service:latest", + "srs.srsRuntime.env.AuthEnabled": "false", + "srs.srsRuntime.env.OAuthPublicKeyURL": "", + "srs.srsStorage.tls.enabled": "true", "srs.srsStorage.basicAuthentication.enabled": "false", }, - []string{"charts/srs/templates/srsservice_deployment.yaml"}), + []string{"charts/srs/templates/srsservice_deployment.yaml"}), ) var srsDeploymentObj appsv1.Deployment @@ -40,7 +41,7 @@ func TestSRSDeployment(t *testing.T){ "false", "", false, - podResources{ "1300m", "2Gi", "650m", "2Gi"}, + podResources{"1300m", "2Gi", "650m", "2Gi"}, esDomain{ domain: "elasticsearch-master.default.svc", port: "9200", @@ -50,32 +51,32 @@ func TestSRSDeployment(t *testing.T){ }) } -func TestSRSDeploymentVariables(t *testing.T){ +func TestSRSDeploymentVariables(t *testing.T) { helmChartParser := NewHelmConfigParser( NewHelmTestFromTemplate(t, helmChartRelativePath, map[string]string{ - "srs.enabled": "true", - "srs.deploymentName": "test-srs-dev", - "global.imageCredentials.registry": "docker-registry.io", - "srs.srsRuntime.replicaCount": "3", - "srs.srsRuntime.srsImage": "platform-services/search-n-reporting-service:1.0.0", - "srs.srsRuntime.imagePullSecretNames": "{secret1, secret2}", - "srs.srsRuntime.env.AuthEnabled": "true", - "srs.srsRuntime.env.OAuthPublicKeyURL": "https://acme.authenticator.com/OAuthPublicKeyURL", - "srs.srsRuntime.resources.limits.cpu": "2", - "srs.srsRuntime.resources.limits.memory": "4Gi", - "srs.srsRuntime.resources.requests.cpu": "1", - "srs.srsRuntime.resources.requests.memory": "2Gi", - "srs.srsStorage.provisionInternalESCluster": "false", - "srs.srsStorage.tls.enabled": "false", - "srs.srsStorage.domain": "es-id.managed.cloudiest.io", - "srs.srsStorage.port": "443", - "srs.srsStorage.protocol": "https", - "srs.srsStorage.awsIAM.region": "us-east-1", - "srs.srsStorage.requireInternetAccess": "true", + "srs.enabled": "true", + "srs.deploymentName": "test-srs-dev", + "global.imageCredentials.registry": "docker-registry.io", + "srs.srsRuntime.replicaCount": "3", + "srs.srsRuntime.srsImage": "platform-services/search-n-reporting-service:1.0.0", + "srs.srsRuntime.imagePullSecretNames": "{secret1, secret2}", + "srs.srsRuntime.env.AuthEnabled": "true", + "srs.srsRuntime.env.OAuthPublicKeyURL": "https://acme.authenticator.com/OAuthPublicKeyURL", + "srs.srsRuntime.resources.limits.cpu": "2", + "srs.srsRuntime.resources.limits.memory": "4Gi", + "srs.srsRuntime.resources.requests.cpu": "1", + "srs.srsRuntime.resources.requests.memory": "2Gi", + "srs.srsStorage.provisionInternalESCluster": "false", + "srs.srsStorage.tls.enabled": "false", + "srs.srsStorage.domain": "es-id.managed.cloudiest.io", + "srs.srsStorage.port": "443", + "srs.srsStorage.protocol": "https", + "srs.srsStorage.awsIAM.region": "us-east-1", + "srs.srsStorage.requireInternetAccess": "true", "srs.srsStorage.basicAuthentication.enabled": "false", }, - []string{"charts/srs/templates/srsservice_deployment.yaml"}), + []string{"charts/srs/templates/srsservice_deployment.yaml"}), ) var srsDeploymentObj appsv1.Deployment @@ -97,36 +98,36 @@ func TestSRSDeploymentVariables(t *testing.T){ domain: "es-id.managed.cloudiest.io", port: "443", protocol: "https", - region: "us-east-1", + region: "us-east-1", }, true, }) } -func TestSRSDeploymentVariablesDefaultInternetEgress(t *testing.T){ +func TestSRSDeploymentVariablesDefaultInternetEgress(t *testing.T) { helmChartParser := NewHelmConfigParser( NewHelmTestFromTemplate(t, helmChartRelativePath, map[string]string{ - "srs.enabled": "true", - "srs.deploymentName": "test-srs-dev", - "global.imageCredentials.registry": "docker-registry.io", - "srs.srsRuntime.replicaCount": "3", - "srs.srsRuntime.srsImage": "platform-services/search-n-reporting-service:1.0.0", - "srs.srsRuntime.imagePullSecretNames": "{secret1, secret2}", - "srs.srsRuntime.env.AuthEnabled": "true", - "srs.srsRuntime.env.OAuthPublicKeyURL": "https://acme.authenticator.com/OAuthPublicKeyURL", - "srs.srsRuntime.resources.limits.cpu": "2", - "srs.srsRuntime.resources.limits.memory": "4Gi", - "srs.srsRuntime.resources.requests.cpu": "1", - "srs.srsRuntime.resources.requests.memory": "2Gi", - "srs.srsStorage.provisionInternalESCluster": "false", - "srs.srsStorage.domain": "es-id.managed.cloudiest.io", - "srs.srsStorage.port": "443", - "srs.srsStorage.protocol": "https", - "srs.srsStorage.tls.enabled": "false", + "srs.enabled": "true", + "srs.deploymentName": "test-srs-dev", + "global.imageCredentials.registry": "docker-registry.io", + "srs.srsRuntime.replicaCount": "3", + "srs.srsRuntime.srsImage": "platform-services/search-n-reporting-service:1.0.0", + "srs.srsRuntime.imagePullSecretNames": "{secret1, secret2}", + "srs.srsRuntime.env.AuthEnabled": "true", + "srs.srsRuntime.env.OAuthPublicKeyURL": "https://acme.authenticator.com/OAuthPublicKeyURL", + "srs.srsRuntime.resources.limits.cpu": "2", + "srs.srsRuntime.resources.limits.memory": "4Gi", + "srs.srsRuntime.resources.requests.cpu": "1", + "srs.srsRuntime.resources.requests.memory": "2Gi", + "srs.srsStorage.provisionInternalESCluster": "false", + "srs.srsStorage.domain": "es-id.managed.cloudiest.io", + "srs.srsStorage.port": "443", + "srs.srsStorage.protocol": "https", + "srs.srsStorage.tls.enabled": "false", "srs.srsStorage.basicAuthentication.enabled": "false", }, - []string{"charts/srs/templates/srsservice_deployment.yaml"}), + []string{"charts/srs/templates/srsservice_deployment.yaml"}), ) var srsDeploymentObj appsv1.Deployment @@ -153,8 +154,46 @@ func TestSRSDeploymentVariablesDefaultInternetEgress(t *testing.T){ }) } +func TestSRSDeploymentWithAffinity(t *testing.T) { + + var affintiyBasePath = "srs.srsRuntime.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]." + + helmChartParser := NewHelmConfigParser( + NewHelmTestFromTemplate(t, helmChartRelativePath, map[string]string{ + "srs.enabled": "true", + "srs.deploymentName": "test-srs", + "global.imageCredentials.registry": "docker-registry.io", + "srs.srsRuntime.replicaCount": "1", + "srs.srsRuntime.srsImage": "platform-services/search-n-reporting-service:latest", + "srs.srsRuntime.env.AuthEnabled": "false", + "srs.srsRuntime.env.OAuthPublicKeyURL": "", + "srs.srsStorage.tls.enabled": "true", + "srs.srsStorage.basicAuthentication.enabled": "false", + affintiyBasePath + "key": "kubernetes.io/os", + affintiyBasePath + "operator": "In", + affintiyBasePath + "values[0]": "linux", + }, + []string{"charts/srs/templates/srsservice_deployment.yaml"}), + ) + + var srsDeploymentObj appsv1.Deployment + helmChartParser.getResourceYAML(SearchResourceOption{ + Name: "test-srs", + Kind: "Deployment", + }, &srsDeploymentObj) + + deploymentSpec := srsDeploymentObj.Spec.Template.Spec + require.Equal(t, deploymentSpec.Containers[0].Name, "srs-service") + require.Equal(t, deploymentSpec.Containers[0].Image, "platform-services/search-n-reporting-service:latest") + require.Equal(t, deploymentSpec.Containers[0].Ports[0].Name, "srs-port") + deploymentAffinity := deploymentSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + require.Equal(t, "kubernetes.io/os", deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key) + require.Equal(t, "In", string(deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator)) + require.Equal(t, "linux", deploymentAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0]) +} + func VerifySRSDeployment(t *testing.T, deploymentObj appsv1.Deployment, expectedDeployment srsDeployment) { - require.Equal(t, expectedDeployment.replicaCount, *deploymentObj.Spec.Replicas ) + require.Equal(t, expectedDeployment.replicaCount, *deploymentObj.Spec.Replicas) require.Equal(t, expectedDeployment.appName, deploymentObj.Spec.Selector.MatchLabels["app.kubernetes.io/name"]) if expectedDeployment.internetEgress { require.Equal(t, "true", deploymentObj.Spec.Selector.MatchLabels["networking/allow-internet-egress"]) @@ -171,7 +210,7 @@ func VerifyDeployment(t *testing.T, pod *k8score.PodSpec, expectedSpec srsDeploy require.Equal(t, pod.Containers[0].Ports[0].Name, "srs-port") require.Equal(t, pod.Containers[0].Ports[0].ContainerPort, int32(8080)) var envIndex int32 = 0 - require.Equal(t, "ELASTICSEARCH_HOST", pod.Containers[0].Env[envIndex].Name ) + require.Equal(t, "ELASTICSEARCH_HOST", pod.Containers[0].Env[envIndex].Name) require.Equal(t, expectedSpec.elasticsearchEndPoint.domain, pod.Containers[0].Env[envIndex].Value) envIndex++ require.Equal(t, "ELASTICSEARCH_PORT", pod.Containers[0].Env[envIndex].Name) @@ -185,39 +224,39 @@ func VerifyDeployment(t *testing.T, pod *k8score.PodSpec, expectedSpec srsDeploy var authProvider string = pod.Containers[0].Env[envIndex].Value envIndex++ if strings.EqualFold("aws-iam", authProvider) { - require.Equal(t, "ELASTICSEARCH_REGION", pod.Containers[0].Env[envIndex].Name) - require.Equal(t, expectedSpec.elasticsearchEndPoint.region, pod.Containers[0].Env[envIndex].Value) - envIndex++ + require.Equal(t, "ELASTICSEARCH_REGION", pod.Containers[0].Env[envIndex].Name) + require.Equal(t, expectedSpec.elasticsearchEndPoint.region, pod.Containers[0].Env[envIndex].Value) + envIndex++ } if strings.EqualFold("basic-authentication", authProvider) { - require.Equal(t, "ELASTICSEARCH_USERNAME", pod.Containers[0].Env[envIndex].Name) - require.Equal(t, "srs-elastic-credentials", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Name) - require.Equal(t, "username", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Key) - envIndex++ - require.Equal(t, "ELASTICSEARCH_PASSWORD", pod.Containers[0].Env[envIndex].Name) - require.Equal(t, "srs-elastic-credentials", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Name) - require.Equal(t, "password", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Key) - envIndex++ + require.Equal(t, "ELASTICSEARCH_USERNAME", pod.Containers[0].Env[envIndex].Name) + require.Equal(t, "srs-elastic-credentials", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Name) + require.Equal(t, "username", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Key) + envIndex++ + require.Equal(t, "ELASTICSEARCH_PASSWORD", pod.Containers[0].Env[envIndex].Name) + require.Equal(t, "srs-elastic-credentials", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Name) + require.Equal(t, "password", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Key) + envIndex++ } if strings.EqualFold("tls", authProvider) { - require.Equal(t, "ELASTICSEARCH_USERNAME", pod.Containers[0].Env[envIndex].Name) - require.Equal(t, "srs-elastic-credentials", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Name) - require.Equal(t, "username", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Key) - envIndex++ - require.Equal(t, "ELASTICSEARCH_PASSWORD", pod.Containers[0].Env[envIndex].Name) - require.Equal(t, "srs-elastic-credentials", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Name) - require.Equal(t, "password", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Key) - envIndex++ - require.Equal(t, "PATH_TO_TRUSTSTORE", pod.Containers[0].Env[envIndex].Name) - require.Equal(t, "/usr/share/elastic-certificates.p12", pod.Containers[0].Env[envIndex].Value) - envIndex++ - require.Equal(t, "PATH_TO_KEYSTORE", pod.Containers[0].Env[envIndex].Name) - require.Equal(t, "", pod.Containers[0].Env[envIndex].Value) - envIndex++ + require.Equal(t, "ELASTICSEARCH_USERNAME", pod.Containers[0].Env[envIndex].Name) + require.Equal(t, "srs-elastic-credentials", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Name) + require.Equal(t, "username", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Key) + envIndex++ + require.Equal(t, "ELASTICSEARCH_PASSWORD", pod.Containers[0].Env[envIndex].Name) + require.Equal(t, "srs-elastic-credentials", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Name) + require.Equal(t, "password", pod.Containers[0].Env[envIndex].ValueFrom.SecretKeyRef.Key) + envIndex++ + require.Equal(t, "PATH_TO_TRUSTSTORE", pod.Containers[0].Env[envIndex].Name) + require.Equal(t, "/usr/share/elastic-certificates.p12", pod.Containers[0].Env[envIndex].Value) + envIndex++ + require.Equal(t, "PATH_TO_KEYSTORE", pod.Containers[0].Env[envIndex].Name) + require.Equal(t, "", pod.Containers[0].Env[envIndex].Value) + envIndex++ } require.Equal(t, "APPLICATION_HOST", pod.Containers[0].Env[envIndex].Name) - require.Equal(t, "0.0.0.0", pod.Containers[0].Env[envIndex].Value) - envIndex++ + require.Equal(t, "0.0.0.0", pod.Containers[0].Env[envIndex].Value) + envIndex++ require.Equal(t, "APPLICATION_PORT", pod.Containers[0].Env[envIndex].Name) require.Equal(t, "8080", pod.Containers[0].Env[envIndex].Value) envIndex++ @@ -240,36 +279,39 @@ func VerifyDeployment(t *testing.T, pod *k8score.PodSpec, expectedSpec srsDeploy require.Equal(t, pod.Containers[0].ReadinessProbe.HTTPGet.Port, intstr.FromString("srs-port")) require.Equal(t, pod.Containers[0].ReadinessProbe.HTTPGet.Scheme, k8score.URIScheme("HTTP")) - require.Equal(t, pod.ImagePullSecrets[0].Name, expectedSpec.name + "-reg-secret") + require.Equal(t, pod.ImagePullSecrets[0].Name, expectedSpec.name+"-reg-secret") if expectedSpec.imagePullSecretNames { require.Equal(t, pod.ImagePullSecrets[1].Name, "secret1") require.Equal(t, pod.ImagePullSecrets[2].Name, "secret2") } + + podAffinity := pod.Affinity + require.Empty(t, podAffinity) } type srsDeployment struct { - name string - appName string - replicaCount int32 - imageURI string - authEnabled string - oauthPublicKeyURL string - internetEgress bool - podLimits podResources - elasticsearchEndPoint esDomain - imagePullSecretNames bool + name string + appName string + replicaCount int32 + imageURI string + authEnabled string + oauthPublicKeyURL string + internetEgress bool + podLimits podResources + elasticsearchEndPoint esDomain + imagePullSecretNames bool } type podResources struct { - cpuLimit string - memoryLimit string - cpuRequest string - memoryRequest string + cpuLimit string + memoryLimit string + cpuRequest string + memoryRequest string } type esDomain struct { - domain string - port string - protocol string - region string + domain string + port string + protocol string + region string } diff --git a/terratest/src/test/pega/clustering-service-deployment_test.go b/terratest/src/test/pega/clustering-service-deployment_test.go index 5d772a92a..271f92ecd 100644 --- a/terratest/src/test/pega/clustering-service-deployment_test.go +++ b/terratest/src/test/pega/clustering-service-deployment_test.go @@ -62,6 +62,8 @@ func VerifyClusteringServiceDeployment(t *testing.T, yamlContent string) { require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[0].MountPath, "/opt/hazelcast/logs") require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[1].Name, "hazelcast-volume-credentials") require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[1].MountPath, "/opt/hazelcast/secrets") + statefulsetAffinity := statefulsetObj.Spec.Template.Spec.Affinity + require.Empty(t, statefulsetAffinity) } } } @@ -98,3 +100,43 @@ func TestClusteringServiceDeploymentSecurityContext(t *testing.T) { } } } + +func TestClusteringServiceDeploymentWithAffinity(t *testing.T) { + var supportedVendors = []string{"k8s", "openshift", "eks", "gke", "aks", "pks"} + var supportedOperations = []string{"deploy", "install-deploy"} + + helmChartPath, err := filepath.Abs(PegaHelmChartPath) + require.NoError(t, err) + + var affintiyBasePath = "hazelcast.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]." + + for _, vendor := range supportedVendors { + + for _, operation := range supportedOperations { + + fmt.Println(vendor + "-" + operation) + + var options = &helm.Options{ + SetValues: map[string]string{ + "global.provider": vendor, + "global.actions.execute": operation, + "hazelcast.clusteringServiceEnabled": "true", + affintiyBasePath + "key": "kubernetes.io/os", + affintiyBasePath + "operator": "In", + affintiyBasePath + "values[0]": "linux", + }, + } + + yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/hazelcast/templates/clustering-service-deployment.yaml"}) + yamlSplit := strings.Split(yamlContent, "---") + var statefulsetObj appsv1beta2.StatefulSet + UnmarshalK8SYaml(t, yamlSplit[1], &statefulsetObj) + + statefulsetAffinity := statefulsetObj.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + require.Equal(t, "kubernetes.io/os", statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key) + require.Equal(t, "In", string(statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator)) + require.Equal(t, "linux", statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0]) + + } + } +} diff --git a/terratest/src/test/pega/pega-hz-deployment_test.go b/terratest/src/test/pega/pega-hz-deployment_test.go index 43501dc6b..172b5dd4a 100644 --- a/terratest/src/test/pega/pega-hz-deployment_test.go +++ b/terratest/src/test/pega/pega-hz-deployment_test.go @@ -1,13 +1,14 @@ package pega import ( + "path/filepath" + "strings" + "testing" + "github.com/gruntwork-io/terratest/modules/helm" "github.com/stretchr/testify/require" appsv1beta2 "k8s.io/api/apps/v1beta2" "k8s.io/apimachinery/pkg/util/intstr" - "path/filepath" - "strings" - "testing" ) func TestHazelcastDeployment(t *testing.T) { @@ -58,6 +59,55 @@ func VerifyHazelcastDeployment(t *testing.T, yamlContent string) { require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[0].MountPath, "/opt/hazelcast/logs") require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[1].Name, "hazelcast-volume-credentials") require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[1].MountPath, "/opt/hazelcast/secrets") + statefulsetAffinity := statefulsetObj.Spec.Template.Spec.Affinity + require.Empty(t, statefulsetAffinity) + } + } +} + +func TestHazelcastDeploymentWithPodAffinity(t *testing.T) { + var supportedVendors = []string{"k8s", "openshift", "eks", "gke", "aks", "pks"} + var supportedOperations = []string{"deploy", "install-deploy"} + + helmChartPath, err := filepath.Abs(PegaHelmChartPath) + require.NoError(t, err) + + var affintiyBasePath = "hazelcast.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]." + + for _, vendor := range supportedVendors { + + for _, operation := range supportedOperations { + + var options = &helm.Options{ + SetValues: map[string]string{ + "global.provider": vendor, + "global.actions.execute": operation, + "hazelcast.enabled": "true", + affintiyBasePath + "key": "kubernetes.io/os", + affintiyBasePath + "operator": "In", + affintiyBasePath + "values[0]": "linux", + }, + } + + yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/hazelcast/templates/pega-hz-deployment.yaml"}) + VerifyHazelcastDeploymentWithAffinity(t, yamlContent, options) + + } + } +} + +func VerifyHazelcastDeploymentWithAffinity(t *testing.T, yamlContent string, options *helm.Options) { + var statefulsetObj appsv1beta2.StatefulSet + statefulSlice := strings.Split(yamlContent, "---") + for index, statefulInfo := range statefulSlice { + if index >= 1 { + UnmarshalK8SYaml(t, statefulInfo, &statefulsetObj) + require.Equal(t, *statefulsetObj.Spec.Replicas, int32(3)) + require.Equal(t, statefulsetObj.Spec.ServiceName, "pega-hazelcast-service") + statefulsetAffinity := statefulsetObj.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + require.Equal(t, "kubernetes.io/os", statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key) + require.Equal(t, "In", string(statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator)) + require.Equal(t, "linux", statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0]) } } } diff --git a/terratest/src/test/pega/pega-installer-job_test.go b/terratest/src/test/pega/pega-installer-job_test.go index 0a892f9a9..5a43e97f4 100644 --- a/terratest/src/test/pega/pega-installer-job_test.go +++ b/terratest/src/test/pega/pega-installer-job_test.go @@ -106,6 +106,39 @@ func TestPegaInstallerJobWithNodeSelector(t *testing.T) { } +func TestPegaInstallerJobWithAffinity(t *testing.T) { + + var affintiyBasePath = "installer.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]." + + var options = &helm.Options{ + SetValues: map[string]string{ + "global.deployment.name": "install-ns", + "global.provider": "k8s", + "global.actions.execute": "install", + "installer.imagePullPolicy": "Always", + "installer.upgrade.upgradeType": "zero-downtime", + affintiyBasePath + "key": "kubernetes.io/os", + affintiyBasePath + "operator": "In", + affintiyBasePath + "values[0]": "linux", + }, + } + + helmChartPath, err := filepath.Abs(PegaHelmChartPath) + require.NoError(t, err) + + yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-installer-job.yaml"}) + yamlSplit := strings.Split(yamlContent, "---") + + var jobObj k8sbatch.Job + UnmarshalK8SYaml(t, yamlSplit[1], &jobObj) + + jobAffinity := jobObj.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + + require.Equal(t, "kubernetes.io/os", jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key) + require.Equal(t, "In", string(jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator)) + require.Equal(t, "linux", jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0]) +} + func assertJob(t *testing.T, jobYaml string, expectedJob pegaDbJob, options *helm.Options, pullPolicy string) { var jobObj k8sbatch.Job UnmarshalK8SYaml(t, jobYaml, &jobObj) diff --git a/terratest/src/test/pega/pega-search-deployment_test.go b/terratest/src/test/pega/pega-search-deployment_test.go index 1096585f1..dfc87a802 100644 --- a/terratest/src/test/pega/pega-search-deployment_test.go +++ b/terratest/src/test/pega/pega-search-deployment_test.go @@ -1,48 +1,45 @@ package pega import ( + "path/filepath" + "testing" + "github.com/gruntwork-io/terratest/modules/helm" "github.com/stretchr/testify/require" appsv1beta2 "k8s.io/api/apps/v1beta2" k8score "k8s.io/api/core/v1" - "path/filepath" - "testing" ) - - -func TestPegaSearchDeployment(t *testing.T){ - var supportedVendors = []string{"k8s","openshift","eks","gke","aks","pks"} - var supportedOperations = []string{"deploy","install-deploy","upgrade-deploy"} - var deploymentNames = []string{"pega","myapp-dev"} +func TestPegaSearchDeployment(t *testing.T) { + var supportedVendors = []string{"k8s", "openshift", "eks", "gke", "aks", "pks"} + var supportedOperations = []string{"deploy", "install-deploy", "upgrade-deploy"} + var deploymentNames = []string{"pega", "myapp-dev"} helmChartPath, err := filepath.Abs(PegaHelmChartPath) require.NoError(t, err) + for _, vendor := range supportedVendors { - for _,vendor := range supportedVendors{ + for _, operation := range supportedOperations { - for _,operation := range supportedOperations{ + for _, depName := range deploymentNames { - for _, depName := range deploymentNames { - - var options = &helm.Options{ - SetValues: map[string]string{ - "global.deployment.name": depName, - "global.provider": vendor, - "global.actions.execute": operation, + var options = &helm.Options{ + SetValues: map[string]string{ + "global.deployment.name": depName, + "global.provider": vendor, + "global.actions.execute": operation, "installer.upgrade.upgradeType": "zero-downtime", - "global.storageClassName": "storage-class", - }, - } + "global.storageClassName": "storage-class", + }, + } - yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/pegasearch/templates/pega-search-deployment.yaml"}) - VerifySearchDeployment(t, yamlContent, options) - } + yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/pegasearch/templates/pega-search-deployment.yaml"}) + VerifySearchDeployment(t, yamlContent, options) + } } } - } func VerifySearchDeployment(t *testing.T, yamlContent string, options *helm.Options) { @@ -58,4 +55,58 @@ func VerifySearchDeployment(t *testing.T, yamlContent string, options *helm.Opti statefulsetSpec := statefulsetObj.Spec.Template.Spec require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[0].Name, "esstorage") require.Equal(t, statefulsetSpec.Containers[0].VolumeMounts[0].MountPath, "/usr/share/elasticsearch/data") -} \ No newline at end of file + statefulsetAffinity := statefulsetObj.Spec.Template.Spec.Affinity + require.Empty(t, statefulsetAffinity) +} + +func TestPegaSearchDeploymentWithPodAffinity(t *testing.T) { + var supportedVendors = []string{"k8s", "eks", "gke", "aks", "pks"} + var supportedOperations = []string{"deploy", "install-deploy", "upgrade-deploy"} + var deploymentNames = []string{"pega", "myapp-dev"} + + helmChartPath, err := filepath.Abs(PegaHelmChartPath) + require.NoError(t, err) + + var affintiyBasePath = "pegasearch.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]." + + for _, vendor := range supportedVendors { + + for _, operation := range supportedOperations { + + for _, depName := range deploymentNames { + + var options = &helm.Options{ + SetValues: map[string]string{ + "global.deployment.name": depName, + "global.provider": vendor, + "global.actions.execute": operation, + "installer.upgrade.upgradeType": "zero-downtime", + "global.storageClassName": "storage-class", + affintiyBasePath + "key": "kubernetes.io/os", + affintiyBasePath + "operator": "In", + affintiyBasePath + "values[0]": "linux", + }, + } + + yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/pegasearch/templates/pega-search-deployment.yaml"}) + VerifySearchDeploymentWithAffinity(t, yamlContent, options) + } + } + } +} + +func VerifySearchDeploymentWithAffinity(t *testing.T, yamlContent string, options *helm.Options) { + var statefulsetObj appsv1beta2.StatefulSet + storageClassName := "storage-class" + UnmarshalK8SYaml(t, yamlContent, &statefulsetObj) + require.Equal(t, statefulsetObj.ObjectMeta.Name, getObjName(options, "-search")) + require.Equal(t, *statefulsetObj.Spec.Replicas, int32(1)) + require.Equal(t, statefulsetObj.Spec.VolumeClaimTemplates[0].Name, "esstorage") + require.Equal(t, statefulsetObj.Spec.VolumeClaimTemplates[0].Spec.AccessModes[0], k8score.PersistentVolumeAccessMode("ReadWriteOnce")) + require.Equal(t, statefulsetObj.Spec.VolumeClaimTemplates[0].Spec.StorageClassName, &storageClassName) + require.Equal(t, statefulsetObj.Spec.ServiceName, getObjName(options, "-search")) + statefulsetAffinity := statefulsetObj.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + require.Equal(t, "kubernetes.io/os", statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key) + require.Equal(t, "In", string(statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator)) + require.Equal(t, "linux", statefulsetAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0]) +} diff --git a/terratest/src/test/pega/pega-tier-deployment_test.go b/terratest/src/test/pega/pega-tier-deployment_test.go index de280bade..b45eeefbc 100644 --- a/terratest/src/test/pega/pega-tier-deployment_test.go +++ b/terratest/src/test/pega/pega-tier-deployment_test.go @@ -62,6 +62,49 @@ func assertStreamWithSorageClass(t *testing.T, streamYaml string, options *helm. require.Equal(t, &storageClassName, statefulsetObj.Spec.VolumeClaimTemplates[0].Spec.StorageClassName) } +func TestPegaTierDeploymentWithPodAffinity(t *testing.T) { + var supportedVendors = []string{"k8s", "eks", "gke", "aks", "pks"} + helmChartPath, err := filepath.Abs(PegaHelmChartPath) + var affintiyBasePath = "global.tier[0].affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution." + require.NoError(t, err) + + var depObjWeb appsv1.Deployment + var depObjBatch appsv1.Deployment + var depObjStream appsv1.Deployment + + for _, vendor := range supportedVendors { + var options = &helm.Options{ + SetValues: map[string]string{ + "global.provider": vendor, + "global.actions.execute": "deploy", + "global.deployment.name": "pega", + "installer.upgrade.upgradeType": "zero-downtime", + "global.tier[0].name": "web", + "global.tier[1].name": "batch", + "global.tier[2].name": "stream", + affintiyBasePath + "nodeSelectorTerms[0].matchExpressions[0].key": "kubernetes.io/os", + affintiyBasePath + "nodeSelectorTerms[0].matchExpressions[0].operator": "In", + affintiyBasePath + "nodeSelectorTerms[0].matchExpressions[0].values[0]": "linux", + }, + } + + yamlContent := RenderTemplate(t, options, helmChartPath, []string{"templates/pega-tier-deployment.yaml"}) + yamlSplit := strings.Split(yamlContent, "---") + + UnmarshalK8SYaml(t, yamlSplit[1], &depObjWeb) + deploymentNodeAffinityWeb := depObjWeb.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + require.Equal(t, "kubernetes.io/os", deploymentNodeAffinityWeb.NodeSelectorTerms[0].MatchExpressions[0].Key) + require.Equal(t, "In", string(deploymentNodeAffinityWeb.NodeSelectorTerms[0].MatchExpressions[0].Operator)) + require.Equal(t, "linux", deploymentNodeAffinityWeb.NodeSelectorTerms[0].MatchExpressions[0].Values[0]) + UnmarshalK8SYaml(t, yamlSplit[2], &depObjBatch) + deploymentAffinityBatch := depObjBatch.Spec.Template.Spec.Affinity + require.Empty(t, deploymentAffinityBatch) + UnmarshalK8SYaml(t, yamlSplit[3], &depObjStream) + deploymentAffinityStream := depObjStream.Spec.Template.Spec.Affinity + require.Empty(t, deploymentAffinityStream) + } +} + func TestPegaTierDeploymentWithFSGroup(t *testing.T) { var supportedVendors = []string{"k8s", "eks", "gke", "aks", "pks"} customFsGroups := map[string]int64{