diff --git a/.chloggen/monolithic_multitenancy.yaml b/.chloggen/monolithic_multitenancy.yaml new file mode 100644 index 000000000..4b96b8703 --- /dev/null +++ b/.chloggen/monolithic_multitenancy.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. operator, github action) +component: operator + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Support multi-tenancy in TempoMonolithic CR + +# One or more tracking issues related to the change +issues: [816] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/controllers/tempo/tempomonolithic_controller.go b/controllers/tempo/tempomonolithic_controller.go index 8e7cda22d..b55ad1a0f 100644 --- a/controllers/tempo/tempomonolithic_controller.go +++ b/controllers/tempo/tempomonolithic_controller.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -77,7 +78,13 @@ func (r *TempoMonolithicReconciler) Reconcile(ctx context.Context, req ctrl.Requ } func (r *TempoMonolithicReconciler) createOrUpdate(ctx context.Context, tempo v1alpha1.TempoMonolithic) error { - storageParams, errs := storage.GetStorageParamsForTempoMonolithic(ctx, r.Client, tempo) + opts := monolithic.Options{ + CtrlConfig: r.CtrlConfig, + Tempo: tempo, + } + + var errs field.ErrorList + opts.StorageParams, errs = storage.GetStorageParamsForTempoMonolithic(ctx, r.Client, tempo) if len(errs) > 0 { return &status.ConfigurationError{ Reason: v1alpha1.ReasonInvalidStorageConfig, @@ -85,11 +92,15 @@ func (r *TempoMonolithicReconciler) createOrUpdate(ctx context.Context, tempo v1 } } - managedObjects, err := monolithic.BuildAll(monolithic.Options{ - CtrlConfig: r.CtrlConfig, - Tempo: tempo, - StorageParams: storageParams, - }) + if tempo.Spec.Multitenancy.IsGatewayEnabled() { + var err error + opts.GatewayTenantSecret, opts.GatewayTenantsData, err = getTenantParams(ctx, r.Client, &r.CtrlConfig, tempo.Namespace, tempo.Name, tempo.Spec.Multitenancy.TenantsSpec, true) + if err != nil { + return err + } + } + + managedObjects, err := monolithic.BuildAll(opts) if err != nil { return fmt.Errorf("error building manifests: %w", err) } @@ -180,6 +191,7 @@ func (r *TempoMonolithicReconciler) SetupWithManager(mgr ctrl.Manager) error { builder := ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.TempoMonolithic{}). Owns(&corev1.ConfigMap{}). + Owns(&corev1.Secret{}). Owns(&corev1.Service{}). Owns(&corev1.ServiceAccount{}). Owns(&appsv1.StatefulSet{}). diff --git a/internal/manifests/gateway/gateway.go b/internal/manifests/gateway/gateway.go index f2d9deb67..8f6ca67d7 100644 --- a/internal/manifests/gateway/gateway.go +++ b/internal/manifests/gateway/gateway.go @@ -91,10 +91,15 @@ func BuildGateway(params manifestutils.Params) ([]client.Object, error) { gatewayObjectName, ), serviceAccount(params.Tempo), - configMapCABundle(params.Tempo), }...) if params.CtrlConfig.Gates.OpenShift.ServingCertsService { + objs = append(objs, manifestutils.NewConfigMapCABundle( + tempo.Namespace, + naming.Name("gateway-cabundle", tempo.Name), + manifestutils.ComponentLabels(manifestutils.GatewayComponentName, tempo.Name), + )) + dep, err = patchOCPServingCerts(params.Tempo, dep) if err != nil { return nil, err diff --git a/internal/manifests/gateway/openshift.go b/internal/manifests/gateway/openshift.go index 20d616ce9..318b4c1a2 100644 --- a/internal/manifests/gateway/openshift.go +++ b/internal/manifests/gateway/openshift.go @@ -132,17 +132,6 @@ func route(tempo v1alpha1.TempoStack) (*routev1.Route, error) { }, nil } -func configMapCABundle(tempo v1alpha1.TempoStack) *corev1.ConfigMap { - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: naming.Name("gateway-cabundle", tempo.Name), - Namespace: tempo.Namespace, - Labels: manifestutils.ComponentLabels(manifestutils.GatewayComponentName, tempo.Name), - Annotations: map[string]string{"service.beta.openshift.io/inject-cabundle": "true"}, - }, - } -} - func patchOCPServingCerts(tempo v1alpha1.TempoStack, dep *v1.Deployment) (*v1.Deployment, error) { container := corev1.Container{ VolumeMounts: []corev1.VolumeMount{ @@ -207,7 +196,7 @@ func patchOCPServiceAccount(tempo v1alpha1.TempoStack, dep *v1.Deployment) *v1.D func patchOCPOPAContainer(params manifestutils.Params, dep *v1.Deployment) (*v1.Deployment, error) { pod := corev1.PodSpec{ - Containers: []corev1.Container{NewOpaContainer(params.CtrlConfig, *params.Tempo.Spec.Tenants, "tempostack")}, + Containers: []corev1.Container{NewOpaContainer(params.CtrlConfig, *params.Tempo.Spec.Tenants, "tempostack", corev1.ResourceRequirements{})}, } err := mergo.Merge(&dep.Spec.Template.Spec, pod, mergo.WithAppendSlice) if err != nil { @@ -217,7 +206,7 @@ func patchOCPOPAContainer(params manifestutils.Params, dep *v1.Deployment) (*v1. } // NewOpaContainer creates an OPA (https://github.com/observatorium/opa-openshift) container. -func NewOpaContainer(ctrlConfig configv1alpha1.ProjectConfig, tenants v1alpha1.TenantsSpec, opaPackage string) corev1.Container { +func NewOpaContainer(ctrlConfig configv1alpha1.ProjectConfig, tenants v1alpha1.TenantsSpec, opaPackage string, resources corev1.ResourceRequirements) corev1.Container { var args = []string{ "--log.level=warn", "--opa.admin-groups=system:cluster-admins,cluster-admin,dedicated-admin", @@ -227,7 +216,7 @@ func NewOpaContainer(ctrlConfig configv1alpha1.ProjectConfig, tenants v1alpha1.T fmt.Sprintf("--opa.package=%s", opaPackage), } for _, t := range tenants.Authentication { - args = append(args, fmt.Sprintf(`--openshift.mappings=%s=%s`, t.TenantName, "tempo.grafana.com")) + args = append(args, fmt.Sprintf("--openshift.mappings=%s=%s", t.TenantName, "tempo.grafana.com")) } return corev1.Container{ @@ -270,5 +259,6 @@ func NewOpaContainer(ctrlConfig configv1alpha1.ProjectConfig, tenants v1alpha1.T PeriodSeconds: 5, FailureThreshold: 12, }, + Resources: resources, } } diff --git a/internal/manifests/manifestutils/tls.go b/internal/manifests/manifestutils/tls.go index 6c8a17014..a2d657ae5 100644 --- a/internal/manifests/manifestutils/tls.go +++ b/internal/manifests/manifestutils/tls.go @@ -4,6 +4,8 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "github.com/grafana/tempo-operator/apis/tempo/v1alpha1" ) @@ -93,6 +95,19 @@ func MountTLSSpecVolumes( return nil } +// NewConfigMapCABundle creates a new ConfigMap with an annotation that triggers the +// service-ca-operator to inject the cluster CA bundle in this ConfigMap (service-ca.crt key). +func NewConfigMapCABundle(namespace string, name string, labels labels.Set) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + Annotations: map[string]string{"service.beta.openshift.io/inject-cabundle": "true"}, + }, + } +} + func findContainerIndex(pod *corev1.PodSpec, containerName string) (int, error) { for i, container := range pod.Containers { if container.Name == containerName { diff --git a/internal/manifests/monolithic/build.go b/internal/manifests/monolithic/build.go index f92ad3863..6c5b75700 100644 --- a/internal/manifests/monolithic/build.go +++ b/internal/manifests/monolithic/build.go @@ -4,6 +4,9 @@ import ( "maps" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/grafana/tempo-operator/internal/manifests/manifestutils" + "github.com/grafana/tempo-operator/internal/manifests/naming" ) // BuildAll generates all manifests. @@ -16,6 +19,7 @@ func BuildAll(opts Options) ([]client.Object, error) { if err != nil { return nil, err } + manifests = append(manifests, configMap) maps.Copy(extraStsAnnotations, annotations) @@ -23,15 +27,33 @@ func BuildAll(opts Options) ([]client.Object, error) { manifests = append(manifests, BuildServiceAccount(opts)) } + if tempo.Spec.Multitenancy.IsGatewayEnabled() { + objs, annotations, err := BuildGatewayObjects(opts) + if err != nil { + return nil, err + } + + manifests = append(manifests, objs...) + maps.Copy(extraStsAnnotations, annotations) + } + statefulSet, err := BuildTempoStatefulset(opts, extraStsAnnotations) if err != nil { return nil, err } manifests = append(manifests, statefulSet) - + manifests = append(manifests, BuildServiceAccount(opts)) manifests = append(manifests, BuildServices(opts)...) + if opts.CtrlConfig.Gates.OpenShift.ServingCertsService { + manifests = append(manifests, manifestutils.NewConfigMapCABundle( + tempo.Namespace, + naming.ServingCABundleName(tempo.Name), + CommonLabels(tempo.Name), + )) + } + if tempo.Spec.JaegerUI != nil && tempo.Spec.JaegerUI.Enabled { if tempo.Spec.JaegerUI.Ingress != nil && tempo.Spec.JaegerUI.Ingress.Enabled { manifests = append(manifests, BuildJaegerUIIngress(opts)) diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go index 546d0e4e5..413fc13aa 100644 --- a/internal/manifests/monolithic/configmap.go +++ b/internal/manifests/monolithic/configmap.go @@ -48,6 +48,8 @@ type tempoGCSConfig struct { } type tempoConfig struct { + MultitenancyEnabled bool `yaml:"multitenancy_enabled,omitempty"` + Server struct { HTTPListenAddress string `yaml:"http_listen_address,omitempty"` HttpListenPort int `yaml:"http_listen_port,omitempty"` @@ -152,7 +154,19 @@ func buildTempoConfig(opts Options) ([]byte, error) { tempo := opts.Tempo config := tempoConfig{} + config.MultitenancyEnabled = tempo.Spec.Multitenancy != nil && tempo.Spec.Multitenancy.Enabled config.Server.HttpListenPort = manifestutils.PortHTTPServer + if tempo.Spec.Multitenancy.IsGatewayEnabled() { + // all connections to tempo must go via gateway + config.Server.HTTPListenAddress = "localhost" + config.Server.GRPCListenAddress = "localhost" + } + + // The internal server is required because if the gateway is enabled, + // the Tempo API will listen on localhost only, + // and then Kubernetes cannot reach the health check endpoint. + config.InternalServer.Enable = true + config.InternalServer.HTTPListenAddress = "0.0.0.0" // The internal server is required because if the gateway is enabled, // the Tempo API will listen on localhost only, @@ -207,7 +221,12 @@ func buildTempoConfig(opts Options) ([]byte, error) { config.Distributor.Receivers.OTLP.Protocols.GRPC = &tempoReceiverConfig{ TLS: configureReceiverTLS(tempo.Spec.Ingestion.OTLP.GRPC.TLS), } + if tempo.Spec.Multitenancy.IsGatewayEnabled() { + // all connections to tempo must go via gateway + config.Distributor.Receivers.OTLP.Protocols.GRPC.Endpoint = fmt.Sprintf("localhost:%d", manifestutils.PortOtlpGrpcServer) + } } + if tempo.Spec.Ingestion.OTLP.HTTP != nil && tempo.Spec.Ingestion.OTLP.HTTP.Enabled { config.Distributor.Receivers.OTLP.Protocols.HTTP = &tempoReceiverConfig{ TLS: configureReceiverTLS(tempo.Spec.Ingestion.OTLP.HTTP.TLS), diff --git a/internal/manifests/monolithic/gateway.go b/internal/manifests/monolithic/gateway.go new file mode 100644 index 000000000..35264fa23 --- /dev/null +++ b/internal/manifests/monolithic/gateway.go @@ -0,0 +1,74 @@ +package monolithic + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/grafana/tempo-operator/apis/tempo/v1alpha1" + "github.com/grafana/tempo-operator/internal/manifests/gateway" + "github.com/grafana/tempo-operator/internal/manifests/manifestutils" + "github.com/grafana/tempo-operator/internal/manifests/naming" +) + +const ( + opaPackage = "tempomonolithic" +) + +// BuildGatewayObjects builds auxiliary objects required for multitenancy. +// +// Enabling multi-tenancy (multitenancy.enabled=true) and configuring at least one tenant has the following effect on the deployment: +// * a tempo-$name-gateway ConfigMap (rbac.yaml) and Secret (tenants.yaml) will be created +// * a tempo-gateway container is added to the StatefulSet +// * the tempo-$name service will point to the gateway instead of the StatefulSet (http, otlp-grpc and jaeger-ui ports) +// +// additionally, if mode=openshift +// * a tempo-gateway-opa container is added to the StatefulSet +// * the ServiceAccount will get additional annotations (serviceaccounts.openshift.io/oauth-redirectreference.$tenantName) +// * a ClusterRole (tokenreviews/create and subjectaccessreviews/create) and ClusterRoleBinding will be created for the ServiceAccount. +func BuildGatewayObjects(opts Options) ([]client.Object, map[string]string, error) { + tempo := opts.Tempo + manifests := []client.Object{} + extraAnnotations := map[string]string{} + labels := ComponentLabels(manifestutils.GatewayComponentName, tempo.Name) + gatewayObjectName := naming.Name(manifestutils.GatewayComponentName, tempo.Name) + + cfgOpts := gateway.NewConfigOptions( + tempo.Namespace, + tempo.Name, + naming.DefaultServiceAccountName(tempo.Name), + naming.RouteFqdn(tempo.Namespace, tempo.Name, "jaegerui", opts.CtrlConfig.Gates.OpenShift.BaseDomain), + opaPackage, + tempo.Spec.Multitenancy.TenantsSpec, + opts.GatewayTenantSecret, + opts.GatewayTenantsData, + ) + + rbacConfigMap, rbacHash, err := gateway.NewRBACConfigMap(cfgOpts, tempo.Namespace, gatewayObjectName, labels) + if err != nil { + return nil, nil, err + } + extraAnnotations["tempo.grafana.com/rbacConfig.hash"] = rbacHash + manifests = append(manifests, rbacConfigMap) + + tenantsSecret, tenantsHash, err := gateway.NewTenantsSecret(cfgOpts, tempo.Namespace, gatewayObjectName, labels) + if err != nil { + return nil, nil, err + } + extraAnnotations["tempo.grafana.com/tenantsConfig.hash"] = tenantsHash + manifests = append(manifests, tenantsSecret) + + if tempo.Spec.Multitenancy.TenantsSpec.Mode == v1alpha1.ModeOpenShift { + manifests = append(manifests, gateway.NewAccessReviewClusterRole( + gatewayObjectName, + ComponentLabels(manifestutils.GatewayComponentName, tempo.Name), + )) + + manifests = append(manifests, gateway.NewAccessReviewClusterRoleBinding( + gatewayObjectName, + ComponentLabels(manifestutils.GatewayComponentName, tempo.Name), + tempo.Namespace, + naming.DefaultServiceAccountName(tempo.Name), + )) + } + + return manifests, extraAnnotations, nil +} diff --git a/internal/manifests/monolithic/gateway_test.go b/internal/manifests/monolithic/gateway_test.go new file mode 100644 index 000000000..106deae9a --- /dev/null +++ b/internal/manifests/monolithic/gateway_test.go @@ -0,0 +1,53 @@ +package monolithic + +import ( + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1alpha1 "github.com/grafana/tempo-operator/apis/config/v1alpha1" + "github.com/grafana/tempo-operator/apis/tempo/v1alpha1" +) + +func TestGateway(t *testing.T) { + opts := Options{ + CtrlConfig: configv1alpha1.ProjectConfig{ + Gates: configv1alpha1.FeatureGates{ + OpenShift: configv1alpha1.OpenShiftFeatureGates{ + ServingCertsService: true, + }, + }, + }, + Tempo: v1alpha1.TempoMonolithic{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample", + Namespace: "default", + }, + Spec: v1alpha1.TempoMonolithicSpec{ + Multitenancy: &v1alpha1.MonolithicMultitenancySpec{ + Enabled: true, + TenantsSpec: v1alpha1.TenantsSpec{ + Mode: v1alpha1.ModeOpenShift, + Authentication: []v1alpha1.AuthenticationSpec{ + { + TenantName: "dev", + TenantID: "1610b0c3-c509-4592-a256-a1871353dbfa", + }, + { + TenantName: "prod", + TenantID: "1610b0c3-c509-4592-a256-a1871353dbfb", + }, + }, + }, + }, + }, + }, + } + objs, annotations, err := BuildGatewayObjects(opts) + require.NoError(t, err) + require.Len(t, objs, 4) + + require.Contains(t, annotations, "tempo.grafana.com/rbacConfig.hash") + require.Contains(t, annotations, "tempo.grafana.com/tenantsConfig.hash") +} diff --git a/internal/manifests/monolithic/options.go b/internal/manifests/monolithic/options.go index f28df6098..a3950c5f3 100644 --- a/internal/manifests/monolithic/options.go +++ b/internal/manifests/monolithic/options.go @@ -8,8 +8,9 @@ import ( // Options defines calculated options required to generate all manifests. type Options struct { - CtrlConfig configv1alpha1.ProjectConfig - Tempo v1alpha1.TempoMonolithic - ConfigChecksum string - StorageParams manifestutils.StorageParams + CtrlConfig configv1alpha1.ProjectConfig + Tempo v1alpha1.TempoMonolithic + StorageParams manifestutils.StorageParams + GatewayTenantSecret []*manifestutils.GatewayTenantOIDCSecret + GatewayTenantsData []*manifestutils.GatewayTenantsData } diff --git a/internal/manifests/monolithic/serviceaccount.go b/internal/manifests/monolithic/serviceaccount.go index 15c9d868d..c3b36e962 100644 --- a/internal/manifests/monolithic/serviceaccount.go +++ b/internal/manifests/monolithic/serviceaccount.go @@ -4,6 +4,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/grafana/tempo-operator/apis/tempo/v1alpha1" + "github.com/grafana/tempo-operator/internal/manifests/gateway" "github.com/grafana/tempo-operator/internal/manifests/naming" ) @@ -15,12 +17,17 @@ const ( func BuildServiceAccount(opts Options) *corev1.ServiceAccount { tempo := opts.Tempo labels := ComponentLabels(componentName, tempo.Name) + var annotations map[string]string + if tempo.Spec.Multitenancy.IsGatewayEnabled() && tempo.Spec.Multitenancy.Mode == v1alpha1.ModeOpenShift { + annotations = gateway.BuildServiceAccountAnnotations(tempo.Spec.Multitenancy.TenantsSpec, naming.Name("jaegerui", tempo.Name)) + } return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: naming.DefaultServiceAccountName(tempo.Name), - Namespace: tempo.Namespace, - Labels: labels, + Name: naming.DefaultServiceAccountName(tempo.Name), + Namespace: tempo.Namespace, + Labels: labels, + Annotations: annotations, }, } } diff --git a/internal/manifests/monolithic/servicemonitor.go b/internal/manifests/monolithic/servicemonitor.go index e4f70ac3b..57932dbe0 100644 --- a/internal/manifests/monolithic/servicemonitor.go +++ b/internal/manifests/monolithic/servicemonitor.go @@ -7,9 +7,14 @@ import ( "github.com/grafana/tempo-operator/internal/manifests/servicemonitor" ) -// BuildServiceMonitor create a ServiceMonitor. +// BuildServiceMonitor creates a ServiceMonitor. func BuildServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { tempo := opts.Tempo - labels := ComponentLabels(manifestutils.TempoMonolithComponentName, tempo.Name) - return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.TempoMonolithComponentName, manifestutils.HttpPortName) + if tempo.Spec.Multitenancy.IsGatewayEnabled() { + labels := ComponentLabels(manifestutils.GatewayComponentName, tempo.Name) + return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.GatewayComponentName, manifestutils.GatewayInternalHttpPortName) + } else { + labels := ComponentLabels(manifestutils.TempoMonolithComponentName, tempo.Name) + return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.TempoMonolithComponentName, manifestutils.HttpPortName) + } } diff --git a/internal/manifests/monolithic/servicemonitor_test.go b/internal/manifests/monolithic/servicemonitor_test.go index e5399b1ca..3423db654 100644 --- a/internal/manifests/monolithic/servicemonitor_test.go +++ b/internal/manifests/monolithic/servicemonitor_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + configv1alpha1 "github.com/grafana/tempo-operator/apis/config/v1alpha1" "github.com/grafana/tempo-operator/apis/tempo/v1alpha1" ) @@ -59,3 +60,72 @@ func TestBuildServiceMonitor(t *testing.T) { }, }, sm) } + +func TestBuildServiceMonitorGateway(t *testing.T) { + opts := Options{ + CtrlConfig: configv1alpha1.ProjectConfig{ + Gates: configv1alpha1.FeatureGates{ + OpenShift: configv1alpha1.OpenShiftFeatureGates{ + ServingCertsService: true, + }, + }, + }, + Tempo: v1alpha1.TempoMonolithic{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample", + Namespace: "default", + }, + Spec: v1alpha1.TempoMonolithicSpec{ + Multitenancy: &v1alpha1.MonolithicMultitenancySpec{ + Enabled: true, + TenantsSpec: v1alpha1.TenantsSpec{ + Authentication: []v1alpha1.AuthenticationSpec{ + { + TenantName: "dev", + TenantID: "dev", + }, + }, + }, + }, + }, + }, + } + sm := BuildServiceMonitor(opts) + + labels := ComponentLabels("gateway", "sample") + require.Equal(t, &monitoringv1.ServiceMonitor{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "monitoring.coreos.com/v1", + Kind: "ServiceMonitor", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tempo-sample-gateway", + Namespace: "default", + Labels: labels, + }, + Spec: monitoringv1.ServiceMonitorSpec{ + Endpoints: []monitoringv1.Endpoint{{ + Scheme: "http", + Port: "internal", + Path: "/metrics", + RelabelConfigs: []*monitoringv1.RelabelConfig{ + { + SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_service_label_app_kubernetes_io_instance"}, + TargetLabel: "cluster", + }, + { + SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_namespace", "__meta_kubernetes_service_label_app_kubernetes_io_component"}, + Separator: "/", + TargetLabel: "job", + }, + }, + }}, + NamespaceSelector: monitoringv1.NamespaceSelector{ + MatchNames: []string{"default"}, + }, + Selector: metav1.LabelSelector{ + MatchLabels: labels, + }, + }, + }, sm) +} diff --git a/internal/manifests/monolithic/services.go b/internal/manifests/monolithic/services.go index 127176c5f..075a07364 100644 --- a/internal/manifests/monolithic/services.go +++ b/internal/manifests/monolithic/services.go @@ -14,11 +14,16 @@ import ( // BuildServices creates all services for a monolithic deployment. func BuildServices(opts Options) []client.Object { tempo := opts.Tempo - objs := []client.Object{buildTempoService(opts)} - if tempo.Spec.JaegerUI != nil && tempo.Spec.JaegerUI.Enabled { - objs = append(objs, buildJaegerUIService(opts)) + + if tempo.Spec.Multitenancy.IsGatewayEnabled() { + return []client.Object{buildGatewayService(opts)} + } else { + objs := []client.Object{buildTempoService(opts)} + if tempo.Spec.JaegerUI != nil && tempo.Spec.JaegerUI.Enabled { + objs = append(objs, buildJaegerUIService(opts)) + } + return objs } - return objs } // buildTempoService creates the service for a monolithic deployment. @@ -108,3 +113,58 @@ func buildJaegerUIService(opts Options) *corev1.Service { }, } } + +// buildGatewayService creates the service for a monolithic deployment if the gateway is enabled. +// Unfortunately the gateway is not a "100% drop-in service", because the paths for Jaeger UI and Tempo API are different +// when accessed via gateway, and the gateway exposes a different set of metrics. +func buildGatewayService(opts Options) *corev1.Service { + tempo := opts.Tempo + annotations := map[string]string{} + + if opts.CtrlConfig.Gates.OpenShift.ServingCertsService { + annotations["service.beta.openshift.io/serving-cert-secret-name"] = naming.ServingCertName(manifestutils.GatewayComponentName, tempo.Name) + } + + ports := []corev1.ServicePort{ + { + Name: manifestutils.GatewayHttpPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.GatewayPortHTTPServer, + // proxies Tempo API and optionally Jaeger UI + TargetPort: intstr.FromString(manifestutils.GatewayHttpPortName), + }, + { + Name: manifestutils.GatewayInternalHttpPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.GatewayPortInternalHTTPServer, + TargetPort: intstr.FromString(manifestutils.GatewayInternalHttpPortName), + }, + } + + if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil && + tempo.Spec.Ingestion.OTLP.GRPC != nil && tempo.Spec.Ingestion.OTLP.GRPC.Enabled { + ports = append(ports, corev1.ServicePort{ + Name: manifestutils.OtlpGrpcPortName, + Protocol: corev1.ProtocolTCP, + Port: manifestutils.PortOtlpGrpcServer, + TargetPort: intstr.FromString(manifestutils.GatewayGrpcPortName), + }) + } + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: naming.Name(manifestutils.GatewayComponentName, tempo.Name), + Namespace: tempo.Namespace, + Labels: ComponentLabels(manifestutils.GatewayComponentName, tempo.Name), + Annotations: annotations, + }, + Spec: corev1.ServiceSpec{ + Ports: ports, + Selector: ComponentLabels(manifestutils.TempoMonolithComponentName, tempo.Name), + }, + } +} diff --git a/internal/manifests/monolithic/services_test.go b/internal/manifests/monolithic/services_test.go index 6e1b87146..30f7f28ac 100644 --- a/internal/manifests/monolithic/services_test.go +++ b/internal/manifests/monolithic/services_test.go @@ -211,6 +211,71 @@ func TestBuildServices(t *testing.T) { }, }, }, + { + name: "enable gateway, OTLP/gRPC, and JaegerUI", + input: v1alpha1.TempoMonolithicSpec{ + Ingestion: &v1alpha1.MonolithicIngestionSpec{ + OTLP: &v1alpha1.MonolithicIngestionOTLPSpec{ + GRPC: &v1alpha1.MonolithicIngestionOTLPProtocolsGRPCSpec{ + Enabled: true, + }, + }, + }, + JaegerUI: &v1alpha1.MonolithicJaegerUISpec{ + Enabled: true, + }, + Multitenancy: &v1alpha1.MonolithicMultitenancySpec{ + Enabled: true, + TenantsSpec: v1alpha1.TenantsSpec{ + Authentication: []v1alpha1.AuthenticationSpec{ + { + TenantName: "dev", + TenantID: "dev", + }, + }, + }, + }, + }, + expected: []client.Object{ + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tempo-sample-gateway", + Namespace: "default", + Labels: ComponentLabels("gateway", "sample"), + Annotations: map[string]string{ + "service.beta.openshift.io/serving-cert-secret-name": "tempo-sample-gateway-serving-cert", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "public", + Protocol: corev1.ProtocolTCP, + Port: 8080, + TargetPort: intstr.FromString("public"), + }, + { + Name: "internal", + Protocol: corev1.ProtocolTCP, + Port: 8081, + TargetPort: intstr.FromString("internal"), + }, + { + Name: "otlp-grpc", + Protocol: corev1.ProtocolTCP, + Port: 4317, + TargetPort: intstr.FromString("grpc-public"), + }, + }, + Selector: ComponentLabels("tempo", "sample"), + }, + }, + }, + }, } for _, test := range tests { diff --git a/internal/manifests/monolithic/statefulset.go b/internal/manifests/monolithic/statefulset.go index 01b774831..0803d93b9 100644 --- a/internal/manifests/monolithic/statefulset.go +++ b/internal/manifests/monolithic/statefulset.go @@ -3,6 +3,7 @@ package monolithic import ( "errors" "fmt" + "path" "github.com/operator-framework/operator-lib/proxy" appsv1 "k8s.io/api/apps/v1" @@ -14,6 +15,7 @@ import ( "k8s.io/utils/ptr" "github.com/grafana/tempo-operator/apis/tempo/v1alpha1" + "github.com/grafana/tempo-operator/internal/manifests/gateway" "github.com/grafana/tempo-operator/internal/manifests/manifestutils" "github.com/grafana/tempo-operator/internal/manifests/naming" ) @@ -114,10 +116,6 @@ func BuildTempoStatefulset(opts Options, extraAnnotations map[string]string) (*a return nil, err } - if tempo.Spec.JaegerUI != nil && tempo.Spec.JaegerUI.Enabled { - configureJaegerUI(opts, sts) - } - if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil { if tempo.Spec.Ingestion.OTLP.GRPC != nil && tempo.Spec.Ingestion.OTLP.GRPC.Enabled && tempo.Spec.Ingestion.OTLP.GRPC.TLS != nil && tempo.Spec.Ingestion.OTLP.GRPC.TLS.Enabled { @@ -142,6 +140,17 @@ func BuildTempoStatefulset(opts Options, extraAnnotations map[string]string) (*a } } + if tempo.Spec.JaegerUI != nil && tempo.Spec.JaegerUI.Enabled { + configureJaegerUI(opts, sts) + } + + if tempo.Spec.Multitenancy.IsGatewayEnabled() { + err = configureGateway(opts, sts) + if err != nil { + return nil, err + } + } + return sts, nil } @@ -294,15 +303,37 @@ func configureStorage(opts Options, sts *appsv1.StatefulSet) error { } func configureJaegerUI(opts Options, sts *appsv1.StatefulSet) { + const tmpVolumeName = "tempo-query-tmp" + tempo := opts.Tempo + + args := []string{ + "--query.base-path=/", + "--grpc-storage-plugin.configuration-file=/conf/tempo-query.yaml", + "--query.bearer-token-propagation=true", + } + + // multi-tenancy enabled, possibly without gateway. forward X-Scope-OrgID header. + if tempo.Spec.Multitenancy != nil && tempo.Spec.Multitenancy.Enabled { + args = append(args, []string{ + "--multi-tenancy.enabled=true", + fmt.Sprintf("--multi-tenancy.header=%s", manifestutils.TenantHeader), + }...) + } + + // all connections to Jaeger UI must go via gateway + if tempo.Spec.Multitenancy.IsGatewayEnabled() { + args = append(args, []string{ + fmt.Sprintf("--query.grpc-server.host-port=localhost:%d", manifestutils.PortJaegerGRPCQuery), + fmt.Sprintf("--query.http-server.host-port=localhost:%d", manifestutils.PortJaegerUI), + fmt.Sprintf("--admin.http.host-port=localhost:%d", manifestutils.PortJaegerMetrics), + }...) + } + tempoQuery := corev1.Container{ Name: "tempo-query", Image: opts.CtrlConfig.DefaultImages.TempoQuery, Env: proxy.ReadProxyVarsFromEnv(), - Args: []string{ - "--query.base-path=/", - "--grpc-storage-plugin.configuration-file=/conf/tempo-query.yaml", - "--query.bearer-token-propagation=true", - }, + Args: args, Ports: []corev1.ContainerPort{ { Name: manifestutils.JaegerGRPCQuery, @@ -326,9 +357,162 @@ func configureJaegerUI(opts Options, sts *appsv1.StatefulSet) { MountPath: "/conf", ReadOnly: true, }, + { + Name: tmpVolumeName, + MountPath: "/tmp", + }, + }, + Resources: ptr.Deref(opts.Tempo.Spec.JaegerUI.Resources, corev1.ResourceRequirements{}), + SecurityContext: manifestutils.TempoContainerSecurityContext(), + } + + tempoQueryVolume := corev1.Volume{ + Name: tmpVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, }, - Resources: ptr.Deref(opts.Tempo.Spec.JaegerUI.Resources, corev1.ResourceRequirements{}), } sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, tempoQuery) + sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, tempoQueryVolume) +} + +func configureGateway(opts Options, sts *appsv1.StatefulSet) error { + var ( + containerName = "tempo-gateway" + gatewayMountDir = "/etc/tempo-gateway" + servingCADir = path.Join(gatewayMountDir, "serving-ca") + servingCertDir = path.Join(gatewayMountDir, "serving-cert") + ) + + tempo := opts.Tempo + args := []string{ + fmt.Sprintf("--web.listen=0.0.0.0:%d", manifestutils.GatewayPortHTTPServer), // proxies Tempo API and optionally Jaeger UI + fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer), // serves health checks + fmt.Sprintf("--traces.tenant-header=%s", manifestutils.TenantHeader), + fmt.Sprintf("--traces.tempo.endpoint=http://localhost:%d", manifestutils.PortHTTPServer), // Tempo API upstream + fmt.Sprintf("--rbac.config=%s", path.Join(gatewayMountDir, "rbac", manifestutils.GatewayRBACFileName)), + fmt.Sprintf("--tenants.config=%s", path.Join(gatewayMountDir, "tenants", manifestutils.GatewayTenantFileName)), + "--log.level=info", + } + ports := []corev1.ContainerPort{ + { + Name: manifestutils.GatewayHttpPortName, + ContainerPort: manifestutils.GatewayPortHTTPServer, + Protocol: corev1.ProtocolTCP, + }, + { + Name: manifestutils.GatewayInternalHttpPortName, + ContainerPort: manifestutils.GatewayPortInternalHTTPServer, + Protocol: corev1.ProtocolTCP, + }, + } + + if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil && tempo.Spec.Ingestion.OTLP.GRPC != nil && tempo.Spec.Ingestion.OTLP.GRPC.Enabled { + args = append(args, fmt.Sprintf("--grpc.listen=0.0.0.0:%d", manifestutils.GatewayPortGRPCServer)) // proxies Tempo Distributor gRPC + args = append(args, fmt.Sprintf("--traces.write.endpoint=localhost:%d", manifestutils.PortOtlpGrpcServer)) // Tempo Distributor gRPC upstream + ports = append(ports, corev1.ContainerPort{ + Name: manifestutils.GatewayGrpcPortName, + ContainerPort: manifestutils.GatewayPortGRPCServer, + Protocol: corev1.ProtocolTCP, + }) + } + + if tempo.Spec.JaegerUI != nil && tempo.Spec.JaegerUI.Enabled { + args = append(args, fmt.Sprintf("--traces.read.endpoint=http://localhost:%d", manifestutils.PortJaegerQuery)) // Jaeger UI upstream + } + + if opts.CtrlConfig.Gates.OpenShift.ServingCertsService { + args = append(args, []string{ + fmt.Sprintf("--tls.server.cert-file=%s", path.Join(servingCertDir, "tls.crt")), // TLS of public HTTP (8080) and gRPC (8090) server + fmt.Sprintf("--tls.server.key-file=%s", path.Join(servingCertDir, "tls.key")), + fmt.Sprintf("--tls.healthchecks.server-ca-file=%s", path.Join(servingCADir, "service-ca.crt")), + fmt.Sprintf("--tls.healthchecks.server-name=%s", naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.GatewayComponentName)), + "--web.healthchecks.url=https://localhost:8080", + }...) + } + + gatewayContainer := corev1.Container{ + Name: containerName, + Image: opts.CtrlConfig.DefaultImages.TempoGateway, + Env: proxy.ReadProxyVarsFromEnv(), + Args: args, + Ports: ports, + LivenessProbe: gateway.LivenessProbe(false), + ReadinessProbe: gateway.ReadinessProbe(false), + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gateway-rbac", + ReadOnly: true, + MountPath: path.Join(gatewayMountDir, "rbac"), + }, + { + Name: "gateway-tenants", + ReadOnly: true, + MountPath: path.Join(gatewayMountDir, "tenants"), + }, + }, + Resources: ptr.Deref(opts.Tempo.Spec.Multitenancy.Resources, corev1.ResourceRequirements{}), + SecurityContext: manifestutils.TempoContainerSecurityContext(), + } + + gatewayVolumes := []corev1.Volume{ + { + Name: "gateway-rbac", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: naming.Name(manifestutils.GatewayComponentName, tempo.Name), + }, + Items: []corev1.KeyToPath{ + { + Key: manifestutils.GatewayRBACFileName, + Path: manifestutils.GatewayRBACFileName, + }, + }, + }, + }, + }, + { + Name: "gateway-tenants", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: naming.Name(manifestutils.GatewayComponentName, tempo.Name), + Items: []corev1.KeyToPath{ + { + Key: manifestutils.GatewayTenantFileName, + Path: manifestutils.GatewayTenantFileName, + }, + }, + }, + }, + }, + } + + sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, gatewayContainer) + sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, gatewayVolumes...) + + if tempo.Spec.Multitenancy.TenantsSpec.Mode == v1alpha1.ModeOpenShift { + opaContainer := gateway.NewOpaContainer( + opts.CtrlConfig, + tempo.Spec.Multitenancy.TenantsSpec, + opaPackage, + ptr.Deref(opts.Tempo.Spec.Multitenancy.Resources, corev1.ResourceRequirements{}), + ) + sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, opaContainer) + } + + if opts.CtrlConfig.Gates.OpenShift.ServingCertsService { + err := manifestutils.MountCAConfigMap(&sts.Spec.Template.Spec, containerName, naming.ServingCABundleName(tempo.Name), servingCADir) + if err != nil { + return err + } + + err = manifestutils.MountCertSecret(&sts.Spec.Template.Spec, containerName, naming.ServingCertName(manifestutils.GatewayComponentName, tempo.Name), servingCertDir) + if err != nil { + return err + } + } + + return nil } diff --git a/internal/manifests/monolithic/statefulset_test.go b/internal/manifests/monolithic/statefulset_test.go index 4c9fa5769..08b102ee3 100644 --- a/internal/manifests/monolithic/statefulset_test.go +++ b/internal/manifests/monolithic/statefulset_test.go @@ -712,3 +712,279 @@ func TestStatefulsetCustomServiceAccount(t *testing.T) { }) } } + +func TestStatefulsetGateway(t *testing.T) { + opts := Options{ + CtrlConfig: configv1alpha1.ProjectConfig{ + DefaultImages: configv1alpha1.ImagesSpec{ + TempoGateway: "quay.io/observatorium/api:x.y.z", + TempoGatewayOpa: "quay.io/observatorium/opa-openshift:x.y.z", + }, + Gates: configv1alpha1.FeatureGates{ + OpenShift: configv1alpha1.OpenShiftFeatureGates{ + ServingCertsService: true, + }, + }, + }, + Tempo: v1alpha1.TempoMonolithic{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample", + Namespace: "default", + }, + Spec: v1alpha1.TempoMonolithicSpec{ + Storage: &v1alpha1.MonolithicStorageSpec{ + Traces: v1alpha1.MonolithicTracesStorageSpec{ + Backend: "memory", + }, + }, + Ingestion: &v1alpha1.MonolithicIngestionSpec{ + OTLP: &v1alpha1.MonolithicIngestionOTLPSpec{ + GRPC: &v1alpha1.MonolithicIngestionOTLPProtocolsGRPCSpec{ + Enabled: true, + }, + }, + }, + JaegerUI: &v1alpha1.MonolithicJaegerUISpec{ + Enabled: true, + }, + Multitenancy: &v1alpha1.MonolithicMultitenancySpec{ + Enabled: true, + TenantsSpec: v1alpha1.TenantsSpec{ + Mode: v1alpha1.ModeOpenShift, + Authentication: []v1alpha1.AuthenticationSpec{ + { + TenantName: "dev", + TenantID: "1610b0c3-c509-4592-a256-a1871353dbfa", + }, + { + TenantName: "prod", + TenantID: "1610b0c3-c509-4592-a256-a1871353dbfb", + }, + }, + }, + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1Gi"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3Gi"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }, + }, + }, + }, + }, + } + sts, err := BuildTempoStatefulset(opts, map[string]string{}) + require.NoError(t, err) + + require.Equal(t, corev1.Container{ + Name: "tempo-gateway", + Image: "quay.io/observatorium/api:x.y.z", + Env: proxy.ReadProxyVarsFromEnv(), + Args: []string{ + "--web.listen=0.0.0.0:8080", + "--web.internal.listen=0.0.0.0:8081", + "--traces.tenant-header=x-scope-orgid", + "--traces.tempo.endpoint=http://localhost:3200", + "--rbac.config=/etc/tempo-gateway/rbac/rbac.yaml", + "--tenants.config=/etc/tempo-gateway/tenants/tenants.yaml", + "--log.level=info", + "--grpc.listen=0.0.0.0:8090", + "--traces.write.endpoint=localhost:4317", + "--traces.read.endpoint=http://localhost:16686", + "--tls.server.cert-file=/etc/tempo-gateway/serving-cert/tls.crt", + "--tls.server.key-file=/etc/tempo-gateway/serving-cert/tls.key", + "--tls.healthchecks.server-ca-file=/etc/tempo-gateway/serving-ca/service-ca.crt", + "--tls.healthchecks.server-name=tempo-sample-gateway.default.svc.cluster.local", + "--web.healthchecks.url=https://localhost:8080", + }, + Ports: []corev1.ContainerPort{ + { + Name: "public", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "internal", + ContainerPort: 8081, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "grpc-public", + ContainerPort: 8090, + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/live", + Port: intstr.FromString("internal"), + Scheme: corev1.URISchemeHTTP, + }, + }, + TimeoutSeconds: 2, + PeriodSeconds: 30, + FailureThreshold: 10, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/ready", + Port: intstr.FromString("internal"), + Scheme: corev1.URISchemeHTTP, + }, + }, + TimeoutSeconds: 1, + PeriodSeconds: 5, + FailureThreshold: 12, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gateway-rbac", + ReadOnly: true, + MountPath: "/etc/tempo-gateway/rbac", + }, + { + Name: "gateway-tenants", + ReadOnly: true, + MountPath: "/etc/tempo-gateway/tenants", + }, + { + Name: "tempo-sample-serving-cabundle", + ReadOnly: true, + MountPath: "/etc/tempo-gateway/serving-ca", + }, + { + Name: "tempo-sample-gateway-serving-cert", + ReadOnly: true, + MountPath: "/etc/tempo-gateway/serving-cert", + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1Gi"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3Gi"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }, + }, + SecurityContext: manifestutils.TempoContainerSecurityContext(), + }, sts.Spec.Template.Spec.Containers[2]) + + require.Equal(t, []corev1.Volume{ + { + Name: "gateway-rbac", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tempo-sample-gateway", + }, + Items: []corev1.KeyToPath{ + { + Key: "rbac.yaml", + Path: "rbac.yaml", + }, + }, + }, + }, + }, + { + Name: "gateway-tenants", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "tempo-sample-gateway", + Items: []corev1.KeyToPath{ + { + Key: "tenants.yaml", + Path: "tenants.yaml", + }, + }, + }, + }, + }, + { + Name: "tempo-sample-serving-cabundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tempo-sample-serving-cabundle", + }, + }, + }, + }, + { + Name: "tempo-sample-gateway-serving-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "tempo-sample-gateway-serving-cert", + }, + }, + }, + }, sts.Spec.Template.Spec.Volumes[3:]) + + require.Equal(t, corev1.Container{ + Name: "tempo-gateway-opa", + Image: "quay.io/observatorium/opa-openshift:x.y.z", + Args: []string{ + "--log.level=warn", + "--opa.admin-groups=system:cluster-admins,cluster-admin,dedicated-admin", + "--web.listen=:8082", + "--web.internal.listen=:8083", + "--web.healthchecks.url=http://localhost:8082", + "--opa.package=tempomonolithic", + "--openshift.mappings=dev=tempo.grafana.com", + "--openshift.mappings=prod=tempo.grafana.com", + }, + Ports: []corev1.ContainerPort{ + { + Name: "public", + ContainerPort: 8082, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "opa-metrics", + ContainerPort: 8083, + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/live", + Port: intstr.FromInt(8083), + Scheme: corev1.URISchemeHTTP, + }, + }, + TimeoutSeconds: 2, + PeriodSeconds: 30, + FailureThreshold: 10, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/ready", + Port: intstr.FromInt(8083), + Scheme: corev1.URISchemeHTTP, + }, + }, + TimeoutSeconds: 1, + PeriodSeconds: 5, + FailureThreshold: 12, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1Gi"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3Gi"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }, + }, + }, sts.Spec.Template.Spec.Containers[3]) +} diff --git a/internal/manifests/naming/naming.go b/internal/manifests/naming/naming.go index b216adf98..ca690e380 100644 --- a/internal/manifests/naming/naming.go +++ b/internal/manifests/naming/naming.go @@ -42,3 +42,13 @@ func SigningCABundleName(name string) string { func PrometheusRuleName(stackName string) string { return fmt.Sprintf("%s-prometheus-rule", stackName) } + +// ServingCertName returns the Secret name of the serving certs generated by service-ca-operator. +func ServingCertName(component string, tempoStackName string) string { + return Name(component, tempoStackName) + "-serving-cert" +} + +// ServingCABundleName returns the ConfigMap name the CA bundle generated by service-ca-operator. +func ServingCABundleName(tempoStackName string) string { + return Name("", tempoStackName) + "-serving-cabundle" +} diff --git a/internal/webhooks/tempomonolithic_webhook.go b/internal/webhooks/tempomonolithic_webhook.go index 90966d37b..d2c769d29 100644 --- a/internal/webhooks/tempomonolithic_webhook.go +++ b/internal/webhooks/tempomonolithic_webhook.go @@ -87,6 +87,7 @@ func (v *monolithicValidator) validateTempoMonolithic(ctx context.Context, tempo errors = append(errors, validateName(tempo.Name)...) addValidationResults(v.validateStorage(ctx, tempo)) errors = append(errors, v.validateJaegerUI(tempo)...) + errors = append(errors, v.validateMultitenancy(tempo)...) errors = append(errors, v.validateObservability(tempo)...) errors = append(errors, v.validateServiceAccount(ctx, tempo)...) warnings = append(warnings, v.validateExtraConfig(tempo)...) @@ -137,6 +138,29 @@ func (v *monolithicValidator) validateJaegerUI(tempo tempov1alpha1.TempoMonolith return nil } +func (v *monolithicValidator) validateMultitenancy(tempo tempov1alpha1.TempoMonolithic) field.ErrorList { + if !tempo.Spec.Multitenancy.IsGatewayEnabled() { + return nil + } + + multitenancyBase := field.NewPath("spec", "multitenancy") + if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil && + tempo.Spec.Ingestion.OTLP.HTTP != nil && tempo.Spec.Ingestion.OTLP.HTTP.Enabled { + return field.ErrorList{field.Invalid( + multitenancyBase.Child("enabled"), + tempo.Spec.Multitenancy.Enabled, + "OTLP/HTTP ingestion must be disabled to enable multi-tenancy", + )} + } + + err := ValidateTenantConfigs(&tempo.Spec.Multitenancy.TenantsSpec, tempo.Spec.Multitenancy.IsGatewayEnabled()) + if err != nil { + return field.ErrorList{field.Invalid(multitenancyBase.Child("enabled"), tempo.Spec.Multitenancy.Enabled, err.Error())} + } + + return nil +} + func (v *monolithicValidator) validateObservability(tempo tempov1alpha1.TempoMonolithic) field.ErrorList { if tempo.Spec.Observability == nil { return nil @@ -185,6 +209,15 @@ func (v *monolithicValidator) validateObservability(tempo tempov1alpha1.TempoMon "the grafanaOperator feature gate must be enabled to create a data source for Tempo", )} } + + if tempo.Spec.Observability.Grafana.DataSource != nil && tempo.Spec.Observability.Grafana.DataSource.Enabled && + tempo.Spec.Multitenancy.IsGatewayEnabled() { + return field.ErrorList{field.Invalid( + grafanaBase.Child("dataSource", "enabled"), + tempo.Spec.Observability.Grafana.DataSource.Enabled, + "creating a data source for Tempo is not support if the gateway is enabled", + )} + } } return nil diff --git a/internal/webhooks/tempomonolithic_webhook_test.go b/internal/webhooks/tempomonolithic_webhook_test.go index 4b5e1691e..0a8a88f1d 100644 --- a/internal/webhooks/tempomonolithic_webhook_test.go +++ b/internal/webhooks/tempomonolithic_webhook_test.go @@ -121,6 +121,59 @@ func TestMonolithicValidate(t *testing.T) { errors: field.ErrorList{}, }, + // multitenancy + { + name: "OTLP/HTTP enabled and multi-tenancy enabled", + tempo: v1alpha1.TempoMonolithic{ + Spec: v1alpha1.TempoMonolithicSpec{ + Ingestion: &v1alpha1.MonolithicIngestionSpec{ + OTLP: &v1alpha1.MonolithicIngestionOTLPSpec{ + HTTP: &v1alpha1.MonolithicIngestionOTLPProtocolsHTTPSpec{ + Enabled: true, + }, + }, + }, + Multitenancy: &v1alpha1.MonolithicMultitenancySpec{ + Enabled: true, + TenantsSpec: v1alpha1.TenantsSpec{ + Authentication: []v1alpha1.AuthenticationSpec{{ + TenantName: "abc", + }}, + }, + }, + }, + }, + warnings: admission.Warnings{}, + errors: field.ErrorList{field.Invalid( + field.NewPath("spec", "multitenancy", "enabled"), + true, + "OTLP/HTTP ingestion must be disabled to enable multi-tenancy", + )}, + }, + { + name: "multi-tenancy enabled, OpenShift mode, authorization set", + tempo: v1alpha1.TempoMonolithic{ + Spec: v1alpha1.TempoMonolithicSpec{ + Multitenancy: &v1alpha1.MonolithicMultitenancySpec{ + Enabled: true, + TenantsSpec: v1alpha1.TenantsSpec{ + Mode: v1alpha1.ModeOpenShift, + Authentication: []v1alpha1.AuthenticationSpec{{ + TenantName: "abc", + }}, + Authorization: &v1alpha1.AuthorizationSpec{}, + }, + }, + }, + }, + warnings: admission.Warnings{}, + errors: field.ErrorList{field.Invalid( + field.NewPath("spec", "multitenancy", "enabled"), + true, + "spec.tenants.authorization should not be defined in openshift mode", + )}, + }, + // observability { name: "serviceMonitors enabled but prometheusOperator feature gate not set", @@ -207,6 +260,39 @@ func TestMonolithicValidate(t *testing.T) { "the grafanaOperator feature gate must be enabled to create a data source for Tempo", )}, }, + { + name: "dataSource enabled, grafanaOperator feature gate set, and gateway enabled", + ctrlConfig: configv1alpha1.ProjectConfig{ + Gates: configv1alpha1.FeatureGates{ + GrafanaOperator: true, + }, + }, + tempo: v1alpha1.TempoMonolithic{ + Spec: v1alpha1.TempoMonolithicSpec{ + Observability: &v1alpha1.MonolithicObservabilitySpec{ + Grafana: &v1alpha1.MonolithicObservabilityGrafanaSpec{ + DataSource: &v1alpha1.MonolithicObservabilityGrafanaDataSourceSpec{ + Enabled: true, + }, + }, + }, + Multitenancy: &v1alpha1.MonolithicMultitenancySpec{ + Enabled: true, + TenantsSpec: v1alpha1.TenantsSpec{ + Authentication: []v1alpha1.AuthenticationSpec{{ + TenantName: "", + }}, + }, + }, + }, + }, + warnings: admission.Warnings{}, + errors: field.ErrorList{field.Invalid( + field.NewPath("spec", "observability", "grafana", "dataSource", "enabled"), + true, + "creating a data source for Tempo is not support if the gateway is enabled", + )}, + }, { name: "valid observability config", ctrlConfig: configv1alpha1.ProjectConfig{ diff --git a/internal/webhooks/tempostack_webhook_test.go b/internal/webhooks/tempostack_webhook_test.go index 3bbe28df4..9903ae0bd 100644 --- a/internal/webhooks/tempostack_webhook_test.go +++ b/internal/webhooks/tempostack_webhook_test.go @@ -1381,6 +1381,35 @@ func TestValidatorObservabilityGrafana(t *testing.T) { ), }, }, + { + name: "datasource enabled, feature gate set, gateway enabled", + input: v1alpha1.TempoStack{ + Spec: v1alpha1.TempoStackSpec{ + Observability: v1alpha1.ObservabilitySpec{ + Grafana: v1alpha1.GrafanaConfigSpec{ + CreateDatasource: true, + }, + }, + Template: v1alpha1.TempoTemplateSpec{ + Gateway: v1alpha1.TempoGatewaySpec{ + Enabled: true, + }, + }, + }, + }, + ctrlConfig: configv1alpha1.ProjectConfig{ + Gates: configv1alpha1.FeatureGates{ + GrafanaOperator: true, + }, + }, + expected: field.ErrorList{ + field.Invalid( + field.NewPath("spec").Child("observability").Child("grafana").Child("createDatasource"), + true, + "creating a data source for Tempo is not support if the gateway is enabled", + ), + }, + }, { name: "datasource enabled, feature gate set", input: v1alpha1.TempoStack{ diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/01-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/01-assert.yaml new file mode 100644 index 000000000..f8693bcd8 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/01-assert.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tempo-simplest + namespace: chainsaw-monolithic-multitenancy +status: + readyReplicas: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: tempo-simplest-gateway + namespace: chainsaw-monolithic-multitenancy +spec: + ports: + - name: public + port: 8080 + protocol: TCP + targetPort: public + - name: internal + port: 8081 + protocol: TCP + targetPort: internal + - name: otlp-grpc + port: 4317 + protocol: TCP + targetPort: grpc-public diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/01-install-tempo.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/01-install-tempo.yaml new file mode 100644 index 000000000..5c6917f66 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/01-install-tempo.yaml @@ -0,0 +1,88 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: chainsaw-monolithic-multitenancy # this test must use a known namespace because of the CN field of the TLS certificate +--- +apiVersion: tempo.grafana.com/v1alpha1 +kind: TempoMonolithic +metadata: + name: simplest + namespace: chainsaw-monolithic-multitenancy +spec: + jaegerui: + enabled: true + multitenancy: + enabled: true + mode: openshift + authentication: + - tenantName: dev + tenantId: "1610b0c3-c509-4592-a256-a1871353dbfa" + - tenantName: prod + tenantId: "1610b0c3-c509-4592-a256-a1871353dbfb" +--- + +# Grant the dev-collector Service Account permission to write traces to the 'dev' tenant +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: allow-write-traces-dev-tenant +rules: +- apiGroups: [tempo.grafana.com] + resources: [dev] # tenantName + resourceNames: [traces] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: allow-write-traces-dev-tenant +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: allow-write-traces-dev-tenant +subjects: +- kind: ServiceAccount + name: dev-collector + namespace: chainsaw-monolithic-multitenancy +--- + +# Grant the default Service Account (used by the verify-traces pod) permission to read traces of the 'dev' tenant +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: allow-read-traces-dev-tenant +rules: +- apiGroups: [tempo.grafana.com] + resources: [dev] # tenantName + resourceNames: [traces] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: allow-read-traces-dev-tenant +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: allow-read-traces-dev-tenant +subjects: +- kind: ServiceAccount + name: default + namespace: chainsaw-monolithic-multitenancy +--- +# Grant the default ServiceAccount (used by the verify-traces pod) view permissions of the chainsaw-monolithic-multitenancy namespace. +# If the ServiceAccount cannot access any namespaces, every 'get' request will be denied: +# https://github.com/observatorium/opa-openshift/pull/18/files +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: view + namespace: chainsaw-monolithic-multitenancy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view +subjects: +- kind: ServiceAccount + name: default + namespace: chainsaw-monolithic-multitenancy diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/02-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/02-assert.yaml new file mode 100644 index 000000000..653654c5e --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/02-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dev-collector + namespace: chainsaw-monolithic-multitenancy +status: + readyReplicas: 1 diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/02-install-otelcol.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/02-install-otelcol.yaml new file mode 100644 index 000000000..7c7a231a3 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/02-install-otelcol.yaml @@ -0,0 +1,32 @@ +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: dev + namespace: chainsaw-monolithic-multitenancy +spec: + config: | + extensions: + bearertokenauth: + filename: /var/run/secrets/kubernetes.io/serviceaccount/token + + receivers: + otlp: + protocols: + grpc: + + exporters: + otlp: + endpoint: tempo-simplest-gateway.chainsaw-monolithic-multitenancy.svc.cluster.local:4317 + tls: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + auth: + authenticator: bearertokenauth + headers: + X-Scope-OrgID: dev # tenantName + + service: + extensions: [bearertokenauth] + pipelines: + traces: + receivers: [otlp] + exporters: [otlp] diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/03-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/03-assert.yaml new file mode 100644 index 000000000..41b390c22 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/03-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-traces + namespace: chainsaw-monolithic-multitenancy +status: + conditions: + - status: "True" + type: Complete diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/03-generate-traces.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/03-generate-traces.yaml new file mode 100644 index 000000000..db40100dc --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/03-generate-traces.yaml @@ -0,0 +1,17 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-traces + namespace: chainsaw-monolithic-multitenancy +spec: + template: + spec: + containers: + - name: telemetrygen + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + args: + - traces + - --otlp-endpoint=dev-collector:4317 + - --otlp-insecure + - --traces=10 + restartPolicy: Never diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/04-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/04-assert.yaml new file mode 100644 index 000000000..75296b9d8 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/04-assert.yaml @@ -0,0 +1,19 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: verify-traces-jaegerui + namespace: chainsaw-monolithic-multitenancy +status: + conditions: + - status: "True" + type: Complete +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: verify-traces-traceql + namespace: chainsaw-monolithic-multitenancy +status: + conditions: + - status: "True" + type: Complete diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/04-verify-traces.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/04-verify-traces.yaml new file mode 100644 index 000000000..58ed1048c --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/04-verify-traces.yaml @@ -0,0 +1,55 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: verify-traces-jaegerui + namespace: chainsaw-monolithic-multitenancy +spec: + template: + spec: + containers: + - name: verify-traces + image: ghcr.io/grafana/tempo-operator/test-utils:main + command: ["/bin/bash", "-eux", "-c"] + args: + - | + curl -vG \ + --header "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ + --cacert /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt \ + https://tempo-simplest-gateway.chainsaw-monolithic-multitenancy.svc:8080/api/traces/v1/dev/api/traces \ + --data-urlencode "service=telemetrygen" \ + | tee /tmp/jaeger.out + + num_traces=$(jq ".data | length" /tmp/jaeger.out) + if [[ "$num_traces" != "10" ]]; then + echo && echo "The Jaeger API returned $num_traces instead of 10 traces." + exit 1 + fi + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: verify-traces-traceql + namespace: chainsaw-monolithic-multitenancy +spec: + template: + spec: + containers: + - name: verify-traces + image: ghcr.io/grafana/tempo-operator/test-utils:main + command: ["/bin/bash", "-eux", "-c"] + args: + - | + curl -vG \ + --header "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ + --cacert /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt \ + https://tempo-simplest-gateway.chainsaw-monolithic-multitenancy.svc:8080/api/traces/v1/dev/tempo/api/search \ + --data-urlencode 'q={ resource.service.name="telemetrygen" }' \ + | tee /tmp/tempo.out + + num_traces=$(jq ".traces | length" /tmp/tempo.out) + if [[ "$num_traces" != "10" ]]; then + echo && echo "The Tempo API returned $num_traces instead of 10 traces." + exit 1 + fi + restartPolicy: Never diff --git a/tests/e2e-openshift/monolithic-multitenancy-openshift/chainsaw-test.yaml b/tests/e2e-openshift/monolithic-multitenancy-openshift/chainsaw-test.yaml new file mode 100644 index 000000000..295bc38e6 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-openshift/chainsaw-test.yaml @@ -0,0 +1,31 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: monolithic-multitenancy-openshift +spec: + steps: + - name: step-01 + try: + - apply: + file: 01-install-tempo.yaml + - assert: + file: 01-assert.yaml + - name: step-02 + try: + - apply: + file: 02-install-otelcol.yaml + - assert: + file: 02-assert.yaml + - name: step-03 + try: + - apply: + file: 03-generate-traces.yaml + - assert: + file: 03-assert.yaml + - name: step-04 + try: + - apply: + file: 04-verify-traces.yaml + - assert: + file: 04-assert.yaml diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/00-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/00-assert.yaml new file mode 100644 index 000000000..52d78eec2 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/00-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dex +status: + readyReplicas: 1 diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/00-install-dex.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/00-install-dex.yaml new file mode 100644 index 000000000..ddf1bcef0 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/00-install-dex.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: dex-config +data: + dex.yaml: | + storage: + type: memory + + logger: + level: debug + + web: + http: 0.0.0.0:5555 + + issuer: http://dex:5555/dex + + oauth2: + passwordConnector: local + + staticClients: + - id: tenant1-oidc-client + name: tenant1-oidc-client + secret: ZXhhbXBsZS1hcHAtc2VjcmV0 + redirectURIs: + - http://tempo-sample-gateway:8080/oidc/tenant1/callback + + enablePasswordDB: true + staticPasswords: + - userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" + username: admin + email: admin@example.com + # bcrypt hash of the string "password" + hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dex +spec: + selector: + matchLabels: + app: dex + template: + metadata: + labels: + app: dex + spec: + containers: + - name: dex + # How to build this image: https://gist.github.com/iblancasa/bc5bae33fa14736b367716205229defb + # TODO: https://github.com/grafana/tempo-operator/issues/643 + image: ghcr.io/iblancasa/dex:v2.37.0 + args: ["dex", "serve", "/config/dex.yaml"] + ports: + - containerPort: 5555 + name: public + volumeMounts: + - mountPath: /config + name: dex-config + readOnly: true + volumes: + - name: dex-config + configMap: + name: dex-config +--- +apiVersion: v1 +kind: Service +metadata: + name: dex +spec: + selector: + app: dex + ports: + - name: public + port: 5555 + targetPort: public +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: dex +spec: + to: + kind: Service + name: dex + port: + targetPort: public + tls: + termination: edge diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/01-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/01-assert.yaml new file mode 100644 index 000000000..8884c6022 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/01-assert.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tempo-sample +status: + readyReplicas: 1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo-sample-gateway +data: + rbac.yaml: |- + roleBindings: + - name: assign-allow-rw-tenant1 + roles: + - allow-rw-tenant1 + + subjects: + - kind: user + name: admin@example.com + + roles: + - name: allow-rw-tenant1 + permissions: + - read + - write + + resources: + - traces + + tenants: + - tenant1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: tempo-sample-gateway +data: + tenants.yaml: dGVuYW50czoKLSBuYW1lOiB0ZW5hbnQxCiAgaWQ6IHRlbmFudDEKICBvaWRjOgogICAgY2xpZW50SUQ6IHRlbmFudDEtb2lkYy1jbGllbnQKICAgIGNsaWVudFNlY3JldDogWlhoaGJYQnNaUzFoY0hBdGMyVmpjbVYwCiAgICBpc3N1ZXJVUkw6IGh0dHA6Ly9kZXg6NTU1NS9kZXgKICAgIHJlZGlyZWN0VVJMOiBodHRwOi8vdGVtcG8tc2FtcGxlLWdhdGV3YXk6ODA4MC9vaWRjL3RlbmFudDEvY2FsbGJhY2sKICAgIHVzZXJuYW1lQ2xhaW06IGVtYWls diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/01-install-tempo.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/01-install-tempo.yaml new file mode 100644 index 000000000..704bedb61 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/01-install-tempo.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Secret +metadata: + name: tenant1-oidc-secret +stringData: + clientID: tenant1-oidc-client + clientSecret: ZXhhbXBsZS1hcHAtc2VjcmV0 +type: Opaque +--- +apiVersion: tempo.grafana.com/v1alpha1 +kind: TempoMonolithic +metadata: + name: sample +spec: + jaegerui: + enabled: true + route: + enabled: true + multitenancy: + enabled: true + mode: static + authentication: + - tenantName: tenant1 + tenantId: tenant1 + oidc: + issuerURL: http://dex:5555/dex + redirectURL: http://tempo-sample-gateway:8080/oidc/tenant1/callback + usernameClaim: email + secret: + name: tenant1-oidc-secret + authorization: + roles: + - name: allow-rw-tenant1 + permissions: + - read + - write + resources: + - traces + tenants: + - tenant1 + roleBindings: + - name: assign-allow-rw-tenant1 + roles: + - allow-rw-tenant1 + subjects: + - kind: user + name: admin@example.com diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/02-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/02-assert.yaml new file mode 100644 index 000000000..0d3eaf2bc --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/02-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: opentelemetry-collector +status: + readyReplicas: 1 diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/02-install-otel.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/02-install-otel.yaml new file mode 100644 index 000000000..a7e2e215d --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/02-install-otel.yaml @@ -0,0 +1,71 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: opentelemetry-collector-configmap +data: + config.yaml: | + extensions: + oauth2client: + client_id: tenant1-oidc-client + client_secret: ZXhhbXBsZS1hcHAtc2VjcmV0 + token_url: http://dex:5555/dex/token + + receivers: + otlp: + protocols: + grpc: + + exporters: + otlp: + endpoint: tempo-sample-gateway.chainsaw-monolithic-multitenancy-static.svc.cluster.local:4317 + tls: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + auth: + authenticator: oauth2client + headers: + X-Scope-OrgID: tenant1 + + service: + extensions: [oauth2client] + pipelines: + traces: + exporters: [otlp] + receivers: [otlp] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: opentelemetry-collector +spec: + selector: + matchLabels: + app: opentelemetry-collector + template: + metadata: + labels: + app: opentelemetry-collector + spec: + containers: + - name: opentelemetry-collector + image: otel/opentelemetry-collector-contrib:0.82.0 + command: ["/otelcol-contrib", "--config=/conf/config.yaml"] + volumeMounts: + - mountPath: /conf + name: opentelemetry-collector-configmap + volumes: + - name: opentelemetry-collector-configmap + configMap: + name: opentelemetry-collector-configmap +--- +apiVersion: v1 +kind: Service +metadata: + name: opentelemetry-collector +spec: + type: ClusterIP + ports: + - name: otlp-grpc + port: 4317 + targetPort: 4317 + selector: + app: opentelemetry-collector diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml new file mode 100644 index 000000000..d63e4dc65 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/03-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-traces +status: + conditions: + - status: "True" + type: Complete diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/03-generate-traces.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/03-generate-traces.yaml new file mode 100644 index 000000000..05328e3ab --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/03-generate-traces.yaml @@ -0,0 +1,17 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-traces +spec: + template: + spec: + containers: + - name: telemetrygen + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + args: + - traces + - --otlp-endpoint=opentelemetry-collector:4317 + - --otlp-insecure + - --traces=10 + restartPolicy: Never + backoffLimit: 4 diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/README.md b/tests/e2e-openshift/monolithic-multitenancy-static/README.md new file mode 100644 index 000000000..0a25c13a7 --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/README.md @@ -0,0 +1,8 @@ +# Manual verification in browser, with CRC +The gateway doesn't accept the OpenShift CA which signed the certificate of the OpenShift Route to dex, therefore: + +1. Start test: `chainsaw test --test-dir tests/e2e-openshift/monolithic-multitenancy-static --skip-delete` +2. Open: https://tempo-sample-jaegerui-chainsaw-monolithic-multitenancy-static.apps-crc.testing/tenant1 +3. Replace `http://dex:5555` with `https://dex-chainsaw-monolithic-multitenancy-static.apps-crc.testing` on redirect +4. Login with user: `admin@example.com` and password: `password` +5. Replace `http://tempo-sample-gateway:8080` with `https://tempo-sample-jaegerui-chainsaw-monolithic-multitenancy-static.apps-crc.testing` on redirect diff --git a/tests/e2e-openshift/monolithic-multitenancy-static/chainsaw-test.yaml b/tests/e2e-openshift/monolithic-multitenancy-static/chainsaw-test.yaml new file mode 100755 index 000000000..cc66032ed --- /dev/null +++ b/tests/e2e-openshift/monolithic-multitenancy-static/chainsaw-test.yaml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: monolithic-multitenancy-static +spec: + namespace: chainsaw-monolithic-multitenancy-static + steps: + - name: install-dex + try: + - apply: + file: 00-install-dex.yaml + - assert: + file: 00-assert.yaml + + - name: install-tempo + try: + - apply: + file: 01-install-tempo.yaml + #- assert: + # file: 01-assert.yaml + + - name: install-otel + try: + - apply: + file: 02-install-otel.yaml + - assert: + file: 02-assert.yaml + + - name: generate-traces + try: + - apply: + file: 03-generate-traces.yaml + - assert: + file: 03-assert.yaml