Allow Agent and Elastic stack in different namespaces. (elastic#7382)

* Allow Agent, Fleet and Elastic stack in different namespaces.
* Documentation updates for feature.
---------

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>

.buildkite/pipeline.yml

@@ -11,7 +11,7 @@ steps:
         agents:
           image: docker.elastic.co/ci-agent-images/cloud-k8s-operator/buildkite-agent:bfddf2b3
           cpu: "6"
-          memory: "6G"
+          memory: "7G"
 
       - label: ":go: generate"
         command: "make generate check-local-changes"

docs/orchestrating-elastic-stack-applications/agent-fleet.asciidoc

@@ -702,12 +702,15 @@ Deploys single instance Elastic Agent Deployment in Fleet mode with APM integrat
 [id="{p}-elastic-agent-fleet-known-limitations"]
 == Known limitations
 
-=== Running as root and within a single namespace (ECK < 2.10.0 and Agent < 7.14.0)
-Until version 7.14.0 and ECK version 2.10.0, Elastic Agent in Fleet mode has to run as root and in the same namespace as the Elasticsearch cluster it connects to. 
+=== Running as root (ECK < 2.10.0 and Agent < 7.14.0)
+Until version 7.14.0 and ECK version 2.10.0, Elastic Agent and Fleet Server were required to run as root.
 
-This was due to configuration limitations in Fleet/Elastic Agent. ECK needed to establish trust between Elastic Agents and Elasticsearch. ECK was only able to fetch the required Elasticsearch CA correctly if both resources are in the same namespace.
 As of Elastic Stack version 7.14.0 and ECK version 2.10.0 it is also possible to run Elastic Agent and Fleet as a non-root user. See <<{p}_storing_local_state_in_host_path_volume>> for instructions.
-To establish trust, the Pod needs to update the CA store through a call to `update-ca-trust` before Elastic Agent runs. To call it successfully, the Pod needs to run with elevated privileges.
+
+=== Elastic Agent running in the same namespace as the Elastic stack.
+Until ECK version 2.11.0, Elastic Agent and Fleet Server were required to run within the same Namespace as Elasticsearch.
+
+As of ECK version 2.11.0, Elastic Agent, Fleet Server and Elasticsearch can all be deployed in different Namespaces.
 
 === Running Endpoint Security integration
 Running Endpoint Security link:https://www.elastic.co/guide/en/security/current/install-endpoint.html[integration] is not yet supported in containerized environments, like Kubernetes. This is not an ECK limitation, but the limitation of the integration itself. Note that you can use ECK to deploy Elasticsearch, Kibana and Fleet Server, and add Endpoint Security integration to your policies if Elastic Agents running those policies are deployed in non-containerized environments.

pkg/apis/common/v1/association.go

@@ -204,7 +204,10 @@ type AssociationConf struct {
 	IsServiceAccount bool   `json:"isServiceAccount"`
 	CACertProvided   bool   `json:"caCertProvided"`
 	CASecretName     string `json:"caSecretName"`
-	URL              string `json:"url"`
+	// AdditionalSecretsHash is a hash of additional secrets such that when any of the underlying
+	// secrets change, the CRD annotation is updated and the pods are restarted.
+	AdditionalSecretsHash string `json:"additionalSecretsHash,omitempty"`
+	URL                   string `json:"url"`
 	// Version of the referenced resource. If a version upgrade is in progress,
 	// matches the lowest running version. May be empty if unknown.
 	Version string `json:"version"`

pkg/controller/agent/pod.go

@@ -333,16 +333,6 @@ func applyRelatedEsAssoc(agent agentv1alpha1.Agent, esAssociation commonv1.Assoc
 		return builder, nil
 	}
 
-	esRef := esAssociation.AssociationRef()
-	if !esRef.IsExternal() && !agent.Spec.FleetServerEnabled && agent.Namespace != esRef.Namespace {
-		// check agent and ES share the same namespace
-		return nil, fmt.Errorf(
-			"agent namespace %s is different than referenced Elasticsearch namespace %s, this is not supported yet",
-			agent.Namespace,
-			esAssociation.AssociationRef().Namespace,
-		)
-	}
-
 	// no ES CA to configure, skip
 	assocConf, err := esAssociation.AssociationConf()
 	if err != nil {

pkg/controller/agent/pod_test.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"fmt"
 	"path"
 	"testing"
 
@@ -845,6 +846,9 @@ func Test_applyRelatedEsAssoc(t *testing.T) {
 			},
 		},
 	}).GetAssociations()[0]
+	assocToOtherNs.SetAssociationConf(&commonv1.AssociationConf{
+		CASecretName: "elasticsearch-es-http-certs-public",
+	})
 
 	expectedCAVolume := []corev1.Volume{
 		{
@@ -857,26 +861,30 @@ func Test_applyRelatedEsAssoc(t *testing.T) {
 			},
 		},
 	}
-	expectedCAVolumeMount := []corev1.VolumeMount{
-		{
-			Name:      "elasticsearch-certs",
-			ReadOnly:  true,
-			MountPath: "/mnt/elastic-internal/elasticsearch-association/agent-ns/elasticsearch/certs",
-		},
+	expectedCAVolumeMountFunc := func(ns string) []corev1.VolumeMount {
+		return []corev1.VolumeMount{
+			{
+				Name:      "elasticsearch-certs",
+				ReadOnly:  true,
+				MountPath: fmt.Sprintf("/mnt/elastic-internal/elasticsearch-association/%s/elasticsearch/certs", ns),
+			},
+		}
 	}
-	expectedCmd := []string{"/usr/bin/env", "bash", "-c", `#!/usr/bin/env bash
+	expectedCmdFunc := func(ns string) []string {
+		return []string{"/usr/bin/env", "bash", "-c", fmt.Sprintf(`#!/usr/bin/env bash
 set -e
-if [[ -f /mnt/elastic-internal/elasticsearch-association/agent-ns/elasticsearch/certs/ca.crt ]]; then
+if [[ -f /mnt/elastic-internal/elasticsearch-association/%[1]s/elasticsearch/certs/ca.crt ]]; then
   if [[ -f /usr/bin/update-ca-trust ]]; then
-    cp /mnt/elastic-internal/elasticsearch-association/agent-ns/elasticsearch/certs/ca.crt /etc/pki/ca-trust/source/anchors/
+    cp /mnt/elastic-internal/elasticsearch-association/%[1]s/elasticsearch/certs/ca.crt /etc/pki/ca-trust/source/anchors/
     /usr/bin/update-ca-trust
   elif [[ -f /usr/sbin/update-ca-certificates ]]; then
-    cp /mnt/elastic-internal/elasticsearch-association/agent-ns/elasticsearch/certs/ca.crt /usr/local/share/ca-certificates/
+    cp /mnt/elastic-internal/elasticsearch-association/%[1]s/elasticsearch/certs/ca.crt /usr/local/share/ca-certificates/
     /usr/sbin/update-ca-certificates
   fi
 fi
 /usr/bin/tini -- /usr/local/bin/docker-entrypoint -e
-`}
+`, ns)}
+	}
 	for _, tt := range []struct {
 		name        string
 		agent       agentv1alpha1.Agent
@@ -919,8 +927,8 @@ fi
 			wantErr: false,
 			wantPodSpec: generatePodSpec(func(ps corev1.PodSpec) corev1.PodSpec {
 				ps.Volumes = expectedCAVolume
-				ps.Containers[0].VolumeMounts = expectedCAVolumeMount
-				ps.Containers[0].Command = expectedCmd
+				ps.Containers[0].VolumeMounts = expectedCAVolumeMountFunc(agentNs)
+				ps.Containers[0].Command = expectedCmdFunc(agentNs)
 				return ps
 			}),
 		},
@@ -940,25 +948,45 @@ fi
 			wantErr: false,
 			wantPodSpec: generatePodSpec(func(ps corev1.PodSpec) corev1.PodSpec {
 				ps.Volumes = expectedCAVolume
-				ps.Containers[0].VolumeMounts = expectedCAVolumeMount
+				ps.Containers[0].VolumeMounts = expectedCAVolumeMountFunc(agentNs)
 				ps.Containers[0].Command = nil
 				return ps
 			}),
 		},
 		{
-			name: "fleet server disabled, different namespace",
+			name: "fleet server disabled, different namespace still has volumes and volumeMount configured",
 			agent: agentv1alpha1.Agent{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "agent",
 					Namespace: agentNs,
 				},
 				Spec: agentv1alpha1.AgentSpec{
-					FleetServerEnabled: false,
 					Version:            "7.16.2",
+					FleetServerEnabled: false,
+					DaemonSet: &agentv1alpha1.DaemonSetSpec{
+						PodTemplate: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{
+									{
+										Name: "agent",
+										SecurityContext: &corev1.SecurityContext{
+											RunAsUser: pointer.Int64(0),
+										},
+									},
+								},
+							},
+						},
+					},
 				},
 			},
 			assoc:   assocToOtherNs,
-			wantErr: true,
+			wantErr: false,
+			wantPodSpec: generatePodSpec(func(ps corev1.PodSpec) corev1.PodSpec {
+				ps.Volumes = expectedCAVolume
+				ps.Containers[0].VolumeMounts = expectedCAVolumeMountFunc("elasticsearch-ns")
+				ps.Containers[0].Command = expectedCmdFunc("elasticsearch-ns")
+				return ps
+			}),
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {

pkg/controller/association/ca.go

@@ -47,7 +47,6 @@ func (r *Reconciler) ReconcileCASecret(ctx context.Context, association commonv1
 	}
 
 	labels := r.AssociationResourceLabels(k8s.ExtractNamespacedName(association), association.AssociationRef().NamespacedName())
-
 	// Certificate data should be copied over a secret in the association namespace
 	expectedSecret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{

pkg/controller/association/controller/agent_fleetserver.go

@@ -17,6 +17,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/association"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/operator"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/utils/k8s"
+	ulog "github.com/elastic/cloud-on-k8s/v2/pkg/utils/log"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/utils/rbac"
 )
 
@@ -30,6 +31,7 @@ func AddAgentFleetServer(mgr manager.Manager, accessReviewer rbac.AccessReviewer
 		AssociationName:           "agent-fleetserver",
 		AssociatedShortName:       "agent",
 		AssociationType:           commonv1.FleetServerAssociationType,
+		AdditionalSecrets:         additionalSecrets,
 		Labels: func(associated types.NamespacedName) map[string]string {
 			return map[string]string{
 				AgentAssociationLabelName:      associated.Name,
@@ -45,6 +47,48 @@ func AddAgentFleetServer(mgr manager.Manager, accessReviewer rbac.AccessReviewer
 	})
 }
 
+func additionalSecrets(ctx context.Context, c k8s.Client, assoc commonv1.Association) ([]types.NamespacedName, error) {
+	log := ulog.FromContext(ctx)
+	associated := assoc.Associated()
+	var agent agentv1alpha1.Agent
+	nsn := types.NamespacedName{Namespace: associated.GetNamespace(), Name: associated.GetName()}
+	if err := c.Get(ctx, nsn, &agent); err != nil {
+		return nil, err
+	}
+	fleetServerRef := assoc.AssociationRef()
+	if !fleetServerRef.IsDefined() {
+		return nil, nil
+	}
+	fleetServer := agentv1alpha1.Agent{}
+	if err := c.Get(ctx, fleetServerRef.NamespacedName(), &fleetServer); err != nil {
+		return nil, err
+	}
+
+	// If the Fleet Server Agent is not associated with an Elasticsearch cluster
+	// (potentially because of a manual setup) we should do nothing.
+	if len(fleetServer.Spec.ElasticsearchRefs) == 0 {
+		return nil, nil
+	}
+	esAssociation, err := association.SingleAssociationOfType(fleetServer.GetAssociations(), commonv1.ElasticsearchAssociationType)
+	if err != nil {
+		return nil, err
+	}
+
+	conf, err := esAssociation.AssociationConf()
+	if err != nil {
+		log.V(1).Info("no additional secrets because no assoc conf")
+		return nil, err
+	}
+	if conf == nil || !conf.CACertProvided {
+		log.V(1).Info("no additional secrets because conf nil or no CA provided")
+		return nil, nil
+	}
+	return []types.NamespacedName{{
+		Namespace: fleetServer.Namespace,
+		Name:      conf.CASecretName,
+	}}, nil
+}
+
 func getFleetServerExternalURL(c k8s.Client, assoc commonv1.Association) (string, error) {
 	fleetServerRef := assoc.AssociationRef()
 	if !fleetServerRef.IsDefined() {

pkg/controller/association/dynamic_watches.go

@@ -5,6 +5,7 @@
 package association
 
 import (
+	"context"
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/types"
@@ -35,6 +36,12 @@ func serviceWatchName(associated types.NamespacedName) string {
 	return fmt.Sprintf("%s-%s-svc-watch", associated.Namespace, associated.Name)
 }
 
+// additionalSecretWatchName returns the name of the watch setup on any additional secrets that
+// are copied during the association reconciliation.
+func additionalSecretWatchName(associated types.NamespacedName) string {
+	return fmt.Sprintf("%s-%s-secrets-watch", associated.Namespace, associated.Name)
+}
+
 // reconcileWatches sets up dynamic watches for:
 // * the referenced resource(s) managed or not by ECK (e.g. Elasticsearch for Kibana -> Elasticsearch associations)
 // * the CA secret of the referenced resource in the referenced resource namespace
@@ -43,7 +50,7 @@ func serviceWatchName(associated types.NamespacedName) string {
 // * if there's an ES user to create, watch the user Secret in ES namespace
 // All watches for all given associations are set under the same watch name and replaced with each reconciliation.
 // The given associations are expected to be of the same type (e.g. Kibana -> Elasticsearch, not Kibana -> Enterprise Search).
-func (r *Reconciler) reconcileWatches(associated types.NamespacedName, associations []commonv1.Association) error {
+func (r *Reconciler) reconcileWatches(ctx context.Context, associated types.NamespacedName, associations []commonv1.Association) error {
 	managedElasticRef := filterManagedElasticRef(associations)
 	unmanagedElasticRef := filterUnmanagedElasticRef(associations)
 
@@ -93,41 +100,80 @@ func (r *Reconciler) reconcileWatches(associated types.NamespacedName, associati
 		}
 	}
 
+	if r.AdditionalSecrets != nil {
+		if err := reconcileGenericWatch(associated, managedElasticRef, r.watches.Secrets, additionalSecretWatchName(associated), func() ([]types.NamespacedName, error) {
+			var toWatch []types.NamespacedName
+			for _, association := range associations {
+				secs, err := r.AdditionalSecrets(ctx, r.Client, association)
+				if err != nil {
+					return nil, err
+				}
+				// Watch the source secrets
+				toWatch = append(toWatch, secs...)
+				// Also watch the target secrets
+				for _, sec := range secs {
+					toWatch = append(toWatch, types.NamespacedName{
+						Name:      sec.Name,
+						Namespace: association.GetNamespace(),
+					})
+				}
+			}
+			return toWatch, nil
+		}); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
-// ReconcileWatch sets or removes `watchName` watch in `dynamicRequest` based on `associated` and `associations` and
-// `watchedFunc`. No watch is added if watchedFunc(association) refers to an empty namespaced name.
-func ReconcileWatch(
+func reconcileGenericWatch(
 	associated types.NamespacedName,
 	associations []commonv1.Association,
 	dynamicRequest *watches.DynamicEnqueueRequest,
 	watchName string,
-	watchedFunc func(association commonv1.Association) types.NamespacedName,
+	watchedFunc func() ([]types.NamespacedName, error),
 ) error {
 	if len(associations) == 0 {
 		// clean up if there are none
 		RemoveWatch(dynamicRequest, watchName)
 		return nil
 	}
 
-	emptyNamespacedName := types.NamespacedName{}
-
-	toWatch := make([]types.NamespacedName, 0, len(associations))
-	for _, association := range associations {
-		watchedNamespacedName := watchedFunc(association)
-		if watchedNamespacedName != emptyNamespacedName {
-			toWatch = append(toWatch, watchedFunc(association))
-		}
+	watched, err := watchedFunc()
+	if err != nil {
+		return err
 	}
-
 	return dynamicRequest.AddHandler(watches.NamedWatch{
 		Name:    watchName,
-		Watched: toWatch,
+		Watched: watched,
 		Watcher: associated,
 	})
 }
 
+// ReconcileWatch sets or removes `watchName` watch in `dynamicRequest` based on `associated` and `associations` and
+// `watchedFunc`. No watch is added if watchedFunc(association) refers to an empty namespaced name.
+func ReconcileWatch(
+	associated types.NamespacedName,
+	associations []commonv1.Association,
+	dynamicRequest *watches.DynamicEnqueueRequest,
+	watchName string,
+	watchedFunc func(association commonv1.Association) types.NamespacedName,
+) error {
+	return reconcileGenericWatch(associated, associations, dynamicRequest, watchName, func() ([]types.NamespacedName, error) {
+		emptyNamespacedName := types.NamespacedName{}
+
+		toWatch := make([]types.NamespacedName, 0, len(associations))
+		for _, association := range associations {
+			watchedNamespacedName := watchedFunc(association)
+			if watchedNamespacedName != emptyNamespacedName {
+				toWatch = append(toWatch, watchedFunc(association))
+			}
+		}
+		return toWatch, nil
+	})
+}
+
 // RemoveWatch removes `watchName` watch from `dynamicRequest`.
 func RemoveWatch(dynamicRequest *watches.DynamicEnqueueRequest, watchName string) {
 	dynamicRequest.RemoveHandlerForKey(watchName)
@@ -142,4 +188,6 @@ func (r *Reconciler) removeWatches(associated types.NamespacedName) {
 	RemoveWatch(r.watches.Services, serviceWatchName(associated))
 	// - ES user secret
 	RemoveWatch(r.watches.Secrets, esUserWatchName(associated))
+	// - Additional secrets (typically in the case of Agent -> Fleet Server -> Elasticsearch)
+	RemoveWatch(r.watches.Secrets, additionalSecretWatchName(associated))
 }