Introduce Elasticsearch config and additional secret mounts to stack …

…config policy (#7233)

Introduce two new fields in StackConfigPolicy, ElasticsearchConfig and SecretMounts. The ElasticsearchConfig field can be used to add configurations for the Elasticsearch clusters managed by the StackConfigurationPolicy, these configurations typically go into the elasticsearch.yml file.SecretMounts are used add additional secrets that need to be mounted into the Elasticsearch pods.

---------

Co-authored-by: Michael Morello <michael.morello@gmail.com>
Co-authored-by: Thibault Richard <thbkrkr@users.noreply.github.com>
Co-authored-by: Peter Brachwitz <peter.brachwitz@gmail.com>

config/crds/v1/all-crds.yaml

@@ -10186,6 +10186,10 @@ spec:
                       (/_cluster/settings)
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
+                  config:
+                    description: Config holds the settings that go into elasticsearch.yml.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
                   indexLifecyclePolicies:
                     description: IndexLifecyclePolicies holds the Index Lifecycle
                       policies settings (/_ilm/policy)
@@ -10212,6 +10216,24 @@ spec:
                       (/_ingest/pipeline)
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
+                  secretMounts:
+                    description: SecretMounts are additional Secrets that need to
+                      be mounted into the Elasticsearch pods.
+                    items:
+                      description: SecretMount contains information about additional
+                        secrets to be mounted to the elasticsearch pods
+                      properties:
+                        mountPath:
+                          description: MountPath denotes the path to which the secret
+                            should be mounted to inside the elasticsearch pod
+                          type: string
+                        secretName:
+                          description: SecretName denotes the name of the secret that
+                            needs to be mounted to the elasticsearch pod
+                          type: string
+                      type: object
+                    type: array
+                    x-kubernetes-preserve-unknown-fields: true
                   securityRoleMappings:
                     description: SecurityRoleMappings holds the Role Mappings settings
                       (/_security/role_mapping)
@@ -10228,6 +10250,31 @@ spec:
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
                 type: object
+              kibana:
+                properties:
+                  config:
+                    description: Config holds the settings that go into kibana.yml.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  secretMounts:
+                    description: SecretMounts are additional secrets that need to
+                      be mounted into the Kibana pods.
+                    items:
+                      description: SecretMount contains information about additional
+                        secrets to be mounted to the elasticsearch pods
+                      properties:
+                        mountPath:
+                          description: MountPath denotes the path to which the secret
+                            should be mounted to inside the elasticsearch pod
+                          type: string
+                        secretName:
+                          description: SecretName denotes the name of the secret that
+                            needs to be mounted to the elasticsearch pod
+                          type: string
+                      type: object
+                    type: array
+                    x-kubernetes-preserve-unknown-fields: true
+                type: object
               resourceSelector:
                 description: A label selector is a label query over a set of resources.
                   The result of matchLabels and matchExpressions are ANDed. An empty

config/crds/v1/bases/stackconfigpolicy.k8s.elastic.co_stackconfigpolicies.yaml

@@ -56,6 +56,10 @@ spec:
                       (/_cluster/settings)
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
+                  config:
+                    description: Config holds the settings that go into elasticsearch.yml.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
                   indexLifecyclePolicies:
                     description: IndexLifecyclePolicies holds the Index Lifecycle
                       policies settings (/_ilm/policy)
@@ -82,6 +86,24 @@ spec:
                       (/_ingest/pipeline)
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
+                  secretMounts:
+                    description: SecretMounts are additional Secrets that need to
+                      be mounted into the Elasticsearch pods.
+                    items:
+                      description: SecretMount contains information about additional
+                        secrets to be mounted to the elasticsearch pods
+                      properties:
+                        mountPath:
+                          description: MountPath denotes the path to which the secret
+                            should be mounted to inside the elasticsearch pod
+                          type: string
+                        secretName:
+                          description: SecretName denotes the name of the secret that
+                            needs to be mounted to the elasticsearch pod
+                          type: string
+                      type: object
+                    type: array
+                    x-kubernetes-preserve-unknown-fields: true
                   securityRoleMappings:
                     description: SecurityRoleMappings holds the Role Mappings settings
                       (/_security/role_mapping)
@@ -98,6 +120,31 @@ spec:
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
                 type: object
+              kibana:
+                properties:
+                  config:
+                    description: Config holds the settings that go into kibana.yml.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  secretMounts:
+                    description: SecretMounts are additional secrets that need to
+                      be mounted into the Kibana pods.
+                    items:
+                      description: SecretMount contains information about additional
+                        secrets to be mounted to the elasticsearch pods
+                      properties:
+                        mountPath:
+                          description: MountPath denotes the path to which the secret
+                            should be mounted to inside the elasticsearch pod
+                          type: string
+                        secretName:
+                          description: SecretName denotes the name of the secret that
+                            needs to be mounted to the elasticsearch pod
+                          type: string
+                      type: object
+                    type: array
+                    x-kubernetes-preserve-unknown-fields: true
+                type: object
               resourceSelector:
                 description: A label selector is a label query over a set of resources.
                   The result of matchLabels and matchExpressions are ANDed. An empty

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -10246,6 +10246,10 @@ spec:
                       (/_cluster/settings)
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
+                  config:
+                    description: Config holds the settings that go into elasticsearch.yml.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
                   indexLifecyclePolicies:
                     description: IndexLifecyclePolicies holds the Index Lifecycle
                       policies settings (/_ilm/policy)
@@ -10272,6 +10276,24 @@ spec:
                       (/_ingest/pipeline)
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
+                  secretMounts:
+                    description: SecretMounts are additional Secrets that need to
+                      be mounted into the Elasticsearch pods.
+                    items:
+                      description: SecretMount contains information about additional
+                        secrets to be mounted to the elasticsearch pods
+                      properties:
+                        mountPath:
+                          description: MountPath denotes the path to which the secret
+                            should be mounted to inside the elasticsearch pod
+                          type: string
+                        secretName:
+                          description: SecretName denotes the name of the secret that
+                            needs to be mounted to the elasticsearch pod
+                          type: string
+                      type: object
+                    type: array
+                    x-kubernetes-preserve-unknown-fields: true
                   securityRoleMappings:
                     description: SecurityRoleMappings holds the Role Mappings settings
                       (/_security/role_mapping)
@@ -10288,6 +10310,31 @@ spec:
                     type: object
                     x-kubernetes-preserve-unknown-fields: true
                 type: object
+              kibana:
+                properties:
+                  config:
+                    description: Config holds the settings that go into kibana.yml.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  secretMounts:
+                    description: SecretMounts are additional secrets that need to
+                      be mounted into the Kibana pods.
+                    items:
+                      description: SecretMount contains information about additional
+                        secrets to be mounted to the elasticsearch pods
+                      properties:
+                        mountPath:
+                          description: MountPath denotes the path to which the secret
+                            should be mounted to inside the elasticsearch pod
+                          type: string
+                        secretName:
+                          description: SecretName denotes the name of the secret that
+                            needs to be mounted to the elasticsearch pod
+                          type: string
+                      type: object
+                    type: array
+                    x-kubernetes-preserve-unknown-fields: true
+                type: object
               resourceSelector:
                 description: A label selector is a label query over a set of resources.
                   The result of matchLabels and matchExpressions are ANDed. An empty

docs/reference/api-docs.asciidoc

@@ -448,6 +448,7 @@ Config represents untyped YAML configuration.
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-enterprisesearch-v1-enterprisesearchspec[$$EnterpriseSearchSpec$$]
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-enterprisesearch-v1beta1-enterprisesearchspec[$$EnterpriseSearchSpec$$]
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-indextemplates[$$IndexTemplates$$]
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-kibanaconfigpolicyspec[$$KibanaConfigPolicySpec$$]
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-kibana-v1-kibanaspec[$$KibanaSpec$$]
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-logstash-v1alpha1-logstashspec[$$LogstashSpec$$]
 - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-maps-v1alpha1-mapsspec[$$MapsSpec$$]
@@ -2076,6 +2077,8 @@ Package v1alpha1 contains API schema definitions for managing StackConfigPolicy
 | *`indexLifecyclePolicies`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-config[$$Config$$]__ | IndexLifecyclePolicies holds the Index Lifecycle policies settings (/_ilm/policy)
 | *`ingestPipelines`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-config[$$Config$$]__ | IngestPipelines holds the Ingest Pipelines settings (/_ingest/pipeline)
 | *`indexTemplates`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-indextemplates[$$IndexTemplates$$]__ | IndexTemplates holds the Index and Component Templates settings
+| *`config`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-config[$$Config$$]__ | Config holds the settings that go into elasticsearch.yml.
+| *`secretMounts`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-secretmount[$$SecretMount$$] array__ | SecretMounts are additional Secrets that need to be mounted into the Elasticsearch pods.
 |===
 
 
@@ -2097,6 +2100,43 @@ Package v1alpha1 contains API schema definitions for managing StackConfigPolicy
 |===
 
 
+[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-kibanaconfigpolicyspec"]
+=== KibanaConfigPolicySpec 
+
+
+
+.Appears In:
+****
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-stackconfigpolicyspec[$$StackConfigPolicySpec$$]
+****
+
+[cols="25a,75a", options="header"]
+|===
+| Field | Description
+| *`config`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-config[$$Config$$]__ | Config holds the settings that go into kibana.yml.
+| *`secretMounts`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-secretmount[$$SecretMount$$] array__ | SecretMounts are additional secrets that need to be mounted into the Kibana pods.
+|===
+
+
+
+
+[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-secretmount"]
+=== SecretMount 
+
+SecretMount contains information about additional secrets to be mounted to the elasticsearch pods
+
+.Appears In:
+****
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-elasticsearchconfigpolicyspec[$$ElasticsearchConfigPolicySpec$$]
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-kibanaconfigpolicyspec[$$KibanaConfigPolicySpec$$]
+****
+
+[cols="25a,75a", options="header"]
+|===
+| Field | Description
+| *`secretName`* __string__ | SecretName denotes the name of the secret that needs to be mounted to the elasticsearch pod
+| *`mountPath`* __string__ | MountPath denotes the path to which the secret should be mounted to inside the elasticsearch pod
+|===
 
 
 [id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-stackconfigpolicy"]
@@ -2133,6 +2173,7 @@ StackConfigPolicy represents a StackConfigPolicy resource in a Kubernetes cluste
 | *`resourceSelector`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#labelselector-v1-meta[$$LabelSelector$$]__ | 
 | *`secureSettings`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-secretsource[$$SecretSource$$] array__ | 
 | *`elasticsearch`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-elasticsearchconfigpolicyspec[$$ElasticsearchConfigPolicySpec$$]__ | 
+| *`kibana`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-stackconfigpolicy-v1alpha1-kibanaconfigpolicyspec[$$KibanaConfigPolicySpec$$]__ | 
 |===
 
 

pkg/apis/elasticsearch/v1/name.go

@@ -12,13 +12,15 @@ import (
 	apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation"
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
 
+	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/hash"
 	common_name "github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/name"
 )
 
 const (
 	configSecretSuffix                           = "config"
 	secureSettingsSecretSuffix                   = "secure-settings"
 	fileSettingsSecretSuffix                     = "file-settings"
+	policyEsConfigSecretSuffix                   = "policy-config" //nolint:gosec
 	httpServiceSuffix                            = "http"
 	internalHTTPServiceSuffix                    = "internal-http"
 	transportServiceSuffix                       = "transport"
@@ -174,3 +176,14 @@ func RemoteCaSecretName(esName string) string {
 func FileSettingsSecretName(esName string) string {
 	return ESNamer.Suffix(esName, fileSettingsSecretSuffix)
 }
+
+func StackConfigElasticsearchConfigSecretName(esName string) string {
+	return ESNamer.Suffix(esName, policyEsConfigSecretSuffix)
+}
+
+// StackConfigAdditionalSecretName returns the name of the stack config policy Secret suffixed with a hash to prevent conflicts.
+// This also helps keep the secret name size to within kubernetes name limits even if the secret name created by the user is long.
+func StackConfigAdditionalSecretName(esName string, secretName string) string {
+	secretNameHash := hash.HashObject(secretName)
+	return ESNamer.Suffix(esName, "scp", secretNameHash)
+}