Monolithic: use internal Tempo server (#847)

If the gateway is enabled, we want to prevent direct connections to the Tempo API by making it listen to localhost. All connections must go via gateway for authentication and authorization. The internal Tempo server listens on all interfaces and responds to health checks. Signed-off-by: Andreas Gerstmayr <agerstmayr@redhat.com>
grafana · Mar 13, 2024 · 03d8e5f · 03d8e5f
1 parent 3cdf1ad
commit 03d8e5f
Show file tree

Hide file tree

Showing 6 changed files with 98 additions and 7 deletions.
diff --git a/.chloggen/monolithic_internal_server.yaml b/.chloggen/monolithic_internal_server.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. operator, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Enable internal server for health checks in TempoMonolithic CR
+
+# One or more tracking issues related to the change
+issues: [847]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/internal/manifests/manifestutils/constants.go b/internal/manifests/manifestutils/constants.go
@@ -25,7 +25,9 @@ const (
 	HttpPortName = "http"
 	// PortHTTPServer declares the port number of the tempo http port.
 	PortHTTPServer = 3200
-	// PortInternalHTTPServer declares the port number of the tempo http port.
+	// TempoInternalServerPortName declares the name of the internal Tempo HTTP Server (for healthchecks).
+	TempoInternalServerPortName = "tempo-internal"
+	// PortInternalHTTPServer declares the port number of the internal tempo http port.
 	PortInternalHTTPServer = 3101
 	// PortJaegerQuery declares the port number of the jaeger query UI port.
 	PortJaegerQuery = 16686

diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go
@@ -24,7 +24,8 @@ type tempoReceiverTLSConfig struct {
 }
 
 type tempoReceiverConfig struct {
-	TLS tempoReceiverTLSConfig `yaml:"tls,omitempty"`
+	TLS      tempoReceiverTLSConfig `yaml:"tls,omitempty"`
+	Endpoint string                 `yaml:"endpoint,omitempty"`
 }
 
 type tempoLocalConfig struct {
@@ -48,9 +49,16 @@ type tempoGCSConfig struct {
 
 type tempoConfig struct {
 	Server struct {
-		HttpListenPort int `yaml:"http_listen_port"`
+		HTTPListenAddress string `yaml:"http_listen_address,omitempty"`
+		HttpListenPort    int    `yaml:"http_listen_port,omitempty"`
+		GRPCListenAddress string `yaml:"grpc_listen_address,omitempty"`
 	} `yaml:"server"`
 
+	InternalServer struct {
+		Enable            bool   `yaml:"enable,omitempty"`
+		HTTPListenAddress string `yaml:"http_listen_address,omitempty"`
+	} `yaml:"internal_server"`
+
 	Storage struct {
 		Trace struct {
 			Backend string `yaml:"backend"`
@@ -146,6 +154,12 @@ func buildTempoConfig(opts Options) ([]byte, error) {
 	config := tempoConfig{}
 	config.Server.HttpListenPort = manifestutils.PortHTTPServer
 
+	// The internal server is required because if the gateway is enabled,
+	// the Tempo API will listen on localhost only,
+	// and then Kubernetes cannot reach the health check endpoint.
+	config.InternalServer.Enable = true
+	config.InternalServer.HTTPListenAddress = "0.0.0.0"
+
 	if tempo.Spec.Storage != nil {
 		config.Storage.Trace.WAL.Path = "/var/tempo/wal"
 		switch tempo.Spec.Storage.Traces.Backend {

diff --git a/internal/manifests/monolithic/configmap_test.go b/internal/manifests/monolithic/configmap_test.go
@@ -64,6 +64,9 @@ func TestBuildConfig(t *testing.T) {
 			expected: `
 server:
   http_listen_port: 3200
+internal_server:
+  enable: true
+  http_listen_address: 0.0.0.0
 storage:
   trace:
     backend: local
@@ -93,6 +96,9 @@ usage_report:
 			expected: `
 server:
   http_listen_port: 3200
+internal_server:
+  enable: true
+  http_listen_address: 0.0.0.0
 storage:
   trace:
     backend: local
@@ -133,6 +139,9 @@ usage_report:
 			expected: `
 server:
   http_listen_port: 3200
+internal_server:
+  enable: true
+  http_listen_address: 0.0.0.0
 storage:
   trace:
     backend: local
@@ -164,6 +173,9 @@ usage_report:
 			expected: `
 server:
   http_listen_port: 3200
+internal_server:
+  enable: true
+  http_listen_address: 0.0.0.0
 storage:
   trace:
     backend: local

diff --git a/internal/manifests/monolithic/statefulset.go b/internal/manifests/monolithic/statefulset.go
@@ -10,6 +10,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 
 	"github.com/grafana/tempo-operator/apis/tempo/v1alpha1"
@@ -75,8 +76,18 @@ func BuildTempoStatefulset(opts Options, extraAnnotations map[string]string) (*a
 									ReadOnly:  true,
 								},
 							},
-							Ports:           buildTempoPorts(opts),
-							ReadinessProbe:  manifestutils.TempoReadinessProbe(false),
+							Ports: buildTempoContainerPorts(opts),
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Scheme: corev1.URISchemeHTTP,
+										Path:   manifestutils.TempoReadinessPath,
+										Port:   intstr.FromString(manifestutils.TempoInternalServerPortName),
+									},
+								},
+								InitialDelaySeconds: 15,
+								TimeoutSeconds:      1,
+							},
 							SecurityContext: manifestutils.TempoContainerSecurityContext(),
 							Resources:       ptr.Deref(tempo.Spec.Resources, corev1.ResourceRequirements{}),
 						},
@@ -162,14 +173,19 @@ func buildAffinity(scheduler *v1alpha1.MonolithicSchedulerSpec, labels labels.Se
 	return manifestutils.DefaultAffinity(labels)
 }
 
-func buildTempoPorts(opts Options) []corev1.ContainerPort {
+func buildTempoContainerPorts(opts Options) []corev1.ContainerPort {
 	tempo := opts.Tempo
 	ports := []corev1.ContainerPort{
 		{
 			Name:          manifestutils.HttpPortName,
 			ContainerPort: manifestutils.PortHTTPServer,
 			Protocol:      corev1.ProtocolTCP,
 		},
+		{
+			Name:          manifestutils.TempoInternalServerPortName,
+			ContainerPort: manifestutils.PortInternalHTTPServer,
+			Protocol:      corev1.ProtocolTCP,
+		},
 	}
 
 	if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil {

diff --git a/internal/manifests/monolithic/statefulset_test.go b/internal/manifests/monolithic/statefulset_test.go
@@ -9,6 +9,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 
 	configv1alpha1 "github.com/grafana/tempo-operator/apis/config/v1alpha1"
@@ -110,13 +111,28 @@ func TestStatefulsetMemoryStorage(t *testing.T) {
 									ContainerPort: 3200,
 									Protocol:      corev1.ProtocolTCP,
 								},
+								{
+									Name:          "tempo-internal",
+									ContainerPort: 3101,
+									Protocol:      corev1.ProtocolTCP,
+								},
 								{
 									Name:          "otlp-grpc",
 									ContainerPort: 4317,
 									Protocol:      corev1.ProtocolTCP,
 								},
 							},
-							ReadinessProbe:  manifestutils.TempoReadinessProbe(false),
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Scheme: corev1.URISchemeHTTP,
+										Path:   "/ready",
+										Port:   intstr.FromString("tempo-internal"),
+									},
+								},
+								InitialDelaySeconds: 15,
+								TimeoutSeconds:      1,
+							},
 							SecurityContext: manifestutils.TempoContainerSecurityContext(),
 							Resources: corev1.ResourceRequirements{
 								Requests: corev1.ResourceList{
@@ -497,6 +513,11 @@ func TestStatefulsetPorts(t *testing.T) {
 					ContainerPort: 3200,
 					Protocol:      corev1.ProtocolTCP,
 				},
+				{
+					Name:          "tempo-internal",
+					ContainerPort: 3101,
+					Protocol:      corev1.ProtocolTCP,
+				},
 			},
 		},
 		{
@@ -514,6 +535,11 @@ func TestStatefulsetPorts(t *testing.T) {
 					ContainerPort: 3200,
 					Protocol:      corev1.ProtocolTCP,
 				},
+				{
+					Name:          "tempo-internal",
+					ContainerPort: 3101,
+					Protocol:      corev1.ProtocolTCP,
+				},
 				{
 					Name:          "otlp-grpc",
 					ContainerPort: 4317,
@@ -536,6 +562,11 @@ func TestStatefulsetPorts(t *testing.T) {
 					ContainerPort: 3200,
 					Protocol:      corev1.ProtocolTCP,
 				},
+				{
+					Name:          "tempo-internal",
+					ContainerPort: 3101,
+					Protocol:      corev1.ProtocolTCP,
+				},
 				{
 					Name:          "otlp-http",
 					ContainerPort: 4318,