From 03d8e5f116e76c27ecf04c365113555ff589e4cb Mon Sep 17 00:00:00 2001 From: Andreas Gerstmayr Date: Wed, 13 Mar 2024 13:16:34 +0100 Subject: [PATCH] Monolithic: use internal Tempo server (#847) If the gateway is enabled, we want to prevent direct connections to the Tempo API by making it listen to localhost. All connections must go via gateway for authentication and authorization. The internal Tempo server listens on all interfaces and responds to health checks. Signed-off-by: Andreas Gerstmayr --- .chloggen/monolithic_internal_server.yaml | 16 +++++++++ internal/manifests/manifestutils/constants.go | 4 ++- internal/manifests/monolithic/configmap.go | 18 ++++++++-- .../manifests/monolithic/configmap_test.go | 12 +++++++ internal/manifests/monolithic/statefulset.go | 22 +++++++++++-- .../manifests/monolithic/statefulset_test.go | 33 ++++++++++++++++++- 6 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 .chloggen/monolithic_internal_server.yaml diff --git a/.chloggen/monolithic_internal_server.yaml b/.chloggen/monolithic_internal_server.yaml new file mode 100644 index 000000000..137d7b13a --- /dev/null +++ b/.chloggen/monolithic_internal_server.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. operator, github action) +component: operator + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Enable internal server for health checks in TempoMonolithic CR + +# One or more tracking issues related to the change +issues: [847] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/internal/manifests/manifestutils/constants.go b/internal/manifests/manifestutils/constants.go index 60e2744b3..7959ea0fc 100644 --- a/internal/manifests/manifestutils/constants.go +++ b/internal/manifests/manifestutils/constants.go @@ -25,7 +25,9 @@ const ( HttpPortName = "http" // PortHTTPServer declares the port number of the tempo http port. PortHTTPServer = 3200 - // PortInternalHTTPServer declares the port number of the tempo http port. + // TempoInternalServerPortName declares the name of the internal Tempo HTTP Server (for healthchecks). + TempoInternalServerPortName = "tempo-internal" + // PortInternalHTTPServer declares the port number of the internal tempo http port. PortInternalHTTPServer = 3101 // PortJaegerQuery declares the port number of the jaeger query UI port. PortJaegerQuery = 16686 diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go index 6fe8e295c..546d0e4e5 100644 --- a/internal/manifests/monolithic/configmap.go +++ b/internal/manifests/monolithic/configmap.go @@ -24,7 +24,8 @@ type tempoReceiverTLSConfig struct { } type tempoReceiverConfig struct { - TLS tempoReceiverTLSConfig `yaml:"tls,omitempty"` + TLS tempoReceiverTLSConfig `yaml:"tls,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` } type tempoLocalConfig struct { @@ -48,9 +49,16 @@ type tempoGCSConfig struct { type tempoConfig struct { Server struct { - HttpListenPort int `yaml:"http_listen_port"` + HTTPListenAddress string `yaml:"http_listen_address,omitempty"` + HttpListenPort int `yaml:"http_listen_port,omitempty"` + GRPCListenAddress string `yaml:"grpc_listen_address,omitempty"` } `yaml:"server"` + InternalServer struct { + Enable bool `yaml:"enable,omitempty"` + HTTPListenAddress string `yaml:"http_listen_address,omitempty"` + } `yaml:"internal_server"` + Storage struct { Trace struct { Backend string `yaml:"backend"` @@ -146,6 +154,12 @@ func buildTempoConfig(opts Options) ([]byte, error) { config := tempoConfig{} config.Server.HttpListenPort = manifestutils.PortHTTPServer + // The internal server is required because if the gateway is enabled, + // the Tempo API will listen on localhost only, + // and then Kubernetes cannot reach the health check endpoint. + config.InternalServer.Enable = true + config.InternalServer.HTTPListenAddress = "0.0.0.0" + if tempo.Spec.Storage != nil { config.Storage.Trace.WAL.Path = "/var/tempo/wal" switch tempo.Spec.Storage.Traces.Backend { diff --git a/internal/manifests/monolithic/configmap_test.go b/internal/manifests/monolithic/configmap_test.go index 1a4272f26..53ee57a00 100644 --- a/internal/manifests/monolithic/configmap_test.go +++ b/internal/manifests/monolithic/configmap_test.go @@ -64,6 +64,9 @@ func TestBuildConfig(t *testing.T) { expected: ` server: http_listen_port: 3200 +internal_server: + enable: true + http_listen_address: 0.0.0.0 storage: trace: backend: local @@ -93,6 +96,9 @@ usage_report: expected: ` server: http_listen_port: 3200 +internal_server: + enable: true + http_listen_address: 0.0.0.0 storage: trace: backend: local @@ -133,6 +139,9 @@ usage_report: expected: ` server: http_listen_port: 3200 +internal_server: + enable: true + http_listen_address: 0.0.0.0 storage: trace: backend: local @@ -164,6 +173,9 @@ usage_report: expected: ` server: http_listen_port: 3200 +internal_server: + enable: true + http_listen_address: 0.0.0.0 storage: trace: backend: local diff --git a/internal/manifests/monolithic/statefulset.go b/internal/manifests/monolithic/statefulset.go index 0ff8a803d..01b774831 100644 --- a/internal/manifests/monolithic/statefulset.go +++ b/internal/manifests/monolithic/statefulset.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "github.com/grafana/tempo-operator/apis/tempo/v1alpha1" @@ -75,8 +76,18 @@ func BuildTempoStatefulset(opts Options, extraAnnotations map[string]string) (*a ReadOnly: true, }, }, - Ports: buildTempoPorts(opts), - ReadinessProbe: manifestutils.TempoReadinessProbe(false), + Ports: buildTempoContainerPorts(opts), + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + Path: manifestutils.TempoReadinessPath, + Port: intstr.FromString(manifestutils.TempoInternalServerPortName), + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + }, SecurityContext: manifestutils.TempoContainerSecurityContext(), Resources: ptr.Deref(tempo.Spec.Resources, corev1.ResourceRequirements{}), }, @@ -162,7 +173,7 @@ func buildAffinity(scheduler *v1alpha1.MonolithicSchedulerSpec, labels labels.Se return manifestutils.DefaultAffinity(labels) } -func buildTempoPorts(opts Options) []corev1.ContainerPort { +func buildTempoContainerPorts(opts Options) []corev1.ContainerPort { tempo := opts.Tempo ports := []corev1.ContainerPort{ { @@ -170,6 +181,11 @@ func buildTempoPorts(opts Options) []corev1.ContainerPort { ContainerPort: manifestutils.PortHTTPServer, Protocol: corev1.ProtocolTCP, }, + { + Name: manifestutils.TempoInternalServerPortName, + ContainerPort: manifestutils.PortInternalHTTPServer, + Protocol: corev1.ProtocolTCP, + }, } if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil { diff --git a/internal/manifests/monolithic/statefulset_test.go b/internal/manifests/monolithic/statefulset_test.go index a63f68451..4c9fa5769 100644 --- a/internal/manifests/monolithic/statefulset_test.go +++ b/internal/manifests/monolithic/statefulset_test.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" configv1alpha1 "github.com/grafana/tempo-operator/apis/config/v1alpha1" @@ -110,13 +111,28 @@ func TestStatefulsetMemoryStorage(t *testing.T) { ContainerPort: 3200, Protocol: corev1.ProtocolTCP, }, + { + Name: "tempo-internal", + ContainerPort: 3101, + Protocol: corev1.ProtocolTCP, + }, { Name: "otlp-grpc", ContainerPort: 4317, Protocol: corev1.ProtocolTCP, }, }, - ReadinessProbe: manifestutils.TempoReadinessProbe(false), + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTP, + Path: "/ready", + Port: intstr.FromString("tempo-internal"), + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 1, + }, SecurityContext: manifestutils.TempoContainerSecurityContext(), Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -497,6 +513,11 @@ func TestStatefulsetPorts(t *testing.T) { ContainerPort: 3200, Protocol: corev1.ProtocolTCP, }, + { + Name: "tempo-internal", + ContainerPort: 3101, + Protocol: corev1.ProtocolTCP, + }, }, }, { @@ -514,6 +535,11 @@ func TestStatefulsetPorts(t *testing.T) { ContainerPort: 3200, Protocol: corev1.ProtocolTCP, }, + { + Name: "tempo-internal", + ContainerPort: 3101, + Protocol: corev1.ProtocolTCP, + }, { Name: "otlp-grpc", ContainerPort: 4317, @@ -536,6 +562,11 @@ func TestStatefulsetPorts(t *testing.T) { ContainerPort: 3200, Protocol: corev1.ProtocolTCP, }, + { + Name: "tempo-internal", + ContainerPort: 3101, + Protocol: corev1.ProtocolTCP, + }, { Name: "otlp-http", ContainerPort: 4318,