Skip to content

Commit

Permalink
Support multi-tenancy in TempoMonolithic CR
Browse files Browse the repository at this point in the history
Signed-off-by: Andreas Gerstmayr <agerstmayr@redhat.com>
  • Loading branch information
andreasgerstmayr committed Mar 13, 2024
1 parent 03d8e5f commit 1fdddba
Show file tree
Hide file tree
Showing 40 changed files with 1,693 additions and 46 deletions.
16 changes: 16 additions & 0 deletions .chloggen/monolithic_multitenancy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. operator, github action)
component: operator

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Support multi-tenancy in TempoMonolithic CR

# One or more tracking issues related to the change
issues: [816]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:
24 changes: 18 additions & 6 deletions controllers/tempo/tempomonolithic_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/validation/field"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

Expand Down Expand Up @@ -77,19 +78,29 @@ func (r *TempoMonolithicReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}

func (r *TempoMonolithicReconciler) createOrUpdate(ctx context.Context, tempo v1alpha1.TempoMonolithic) error {
storageParams, errs := storage.GetStorageParamsForTempoMonolithic(ctx, r.Client, tempo)
opts := monolithic.Options{
CtrlConfig: r.CtrlConfig,
Tempo: tempo,
}

var errs field.ErrorList
opts.StorageParams, errs = storage.GetStorageParamsForTempoMonolithic(ctx, r.Client, tempo)
if len(errs) > 0 {
return &status.ConfigurationError{
Reason: v1alpha1.ReasonInvalidStorageConfig,
Message: listFieldErrors(errs),
}
}

managedObjects, err := monolithic.BuildAll(monolithic.Options{
CtrlConfig: r.CtrlConfig,
Tempo: tempo,
StorageParams: storageParams,
})
if tempo.Spec.Multitenancy.IsGatewayEnabled() {
var err error
opts.GatewayTenantSecret, opts.GatewayTenantsData, err = getTenantParams(ctx, r.Client, &r.CtrlConfig, tempo.Namespace, tempo.Name, tempo.Spec.Multitenancy.TenantsSpec, true)
if err != nil {
return err
}
}

managedObjects, err := monolithic.BuildAll(opts)
if err != nil {
return fmt.Errorf("error building manifests: %w", err)
}
Expand Down Expand Up @@ -180,6 +191,7 @@ func (r *TempoMonolithicReconciler) SetupWithManager(mgr ctrl.Manager) error {
builder := ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.TempoMonolithic{}).
Owns(&corev1.ConfigMap{}).
Owns(&corev1.Secret{}).
Owns(&corev1.Service{}).
Owns(&corev1.ServiceAccount{}).
Owns(&appsv1.StatefulSet{}).
Expand Down
7 changes: 6 additions & 1 deletion internal/manifests/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,15 @@ func BuildGateway(params manifestutils.Params) ([]client.Object, error) {
gatewayObjectName,
),
serviceAccount(params.Tempo),
configMapCABundle(params.Tempo),
}...)

if params.CtrlConfig.Gates.OpenShift.ServingCertsService {
objs = append(objs, manifestutils.NewConfigMapCABundle(
tempo.Namespace,
naming.Name("gateway-cabundle", tempo.Name),
manifestutils.ComponentLabels(manifestutils.GatewayComponentName, tempo.Name),
))

dep, err = patchOCPServingCerts(params.Tempo, dep)
if err != nil {
return nil, err
Expand Down
18 changes: 4 additions & 14 deletions internal/manifests/gateway/openshift.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,17 +132,6 @@ func route(tempo v1alpha1.TempoStack) (*routev1.Route, error) {
}, nil
}

func configMapCABundle(tempo v1alpha1.TempoStack) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: naming.Name("gateway-cabundle", tempo.Name),
Namespace: tempo.Namespace,
Labels: manifestutils.ComponentLabels(manifestutils.GatewayComponentName, tempo.Name),
Annotations: map[string]string{"service.beta.openshift.io/inject-cabundle": "true"},
},
}
}

func patchOCPServingCerts(tempo v1alpha1.TempoStack, dep *v1.Deployment) (*v1.Deployment, error) {
container := corev1.Container{
VolumeMounts: []corev1.VolumeMount{
Expand Down Expand Up @@ -207,7 +196,7 @@ func patchOCPServiceAccount(tempo v1alpha1.TempoStack, dep *v1.Deployment) *v1.D

func patchOCPOPAContainer(params manifestutils.Params, dep *v1.Deployment) (*v1.Deployment, error) {
pod := corev1.PodSpec{
Containers: []corev1.Container{NewOpaContainer(params.CtrlConfig, *params.Tempo.Spec.Tenants, "tempostack")},
Containers: []corev1.Container{NewOpaContainer(params.CtrlConfig, *params.Tempo.Spec.Tenants, "tempostack", corev1.ResourceRequirements{})},
}
err := mergo.Merge(&dep.Spec.Template.Spec, pod, mergo.WithAppendSlice)
if err != nil {
Expand All @@ -217,7 +206,7 @@ func patchOCPOPAContainer(params manifestutils.Params, dep *v1.Deployment) (*v1.
}

// NewOpaContainer creates an OPA (https://github.com/observatorium/opa-openshift) container.
func NewOpaContainer(ctrlConfig configv1alpha1.ProjectConfig, tenants v1alpha1.TenantsSpec, opaPackage string) corev1.Container {
func NewOpaContainer(ctrlConfig configv1alpha1.ProjectConfig, tenants v1alpha1.TenantsSpec, opaPackage string, resources corev1.ResourceRequirements) corev1.Container {
var args = []string{
"--log.level=warn",
"--opa.admin-groups=system:cluster-admins,cluster-admin,dedicated-admin",
Expand All @@ -227,7 +216,7 @@ func NewOpaContainer(ctrlConfig configv1alpha1.ProjectConfig, tenants v1alpha1.T
fmt.Sprintf("--opa.package=%s", opaPackage),
}
for _, t := range tenants.Authentication {
args = append(args, fmt.Sprintf(`--openshift.mappings=%s=%s`, t.TenantName, "tempo.grafana.com"))
args = append(args, fmt.Sprintf("--openshift.mappings=%s=%s", t.TenantName, "tempo.grafana.com"))
}

return corev1.Container{
Expand Down Expand Up @@ -270,5 +259,6 @@ func NewOpaContainer(ctrlConfig configv1alpha1.ProjectConfig, tenants v1alpha1.T
PeriodSeconds: 5,
FailureThreshold: 12,
},
Resources: resources,
}
}
15 changes: 15 additions & 0 deletions internal/manifests/manifestutils/tls.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"fmt"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"

"github.com/grafana/tempo-operator/apis/tempo/v1alpha1"
)
Expand Down Expand Up @@ -93,6 +95,19 @@ func MountTLSSpecVolumes(
return nil
}

// NewConfigMapCABundle creates a new ConfigMap with an annotation that triggers the
// service-ca-operator to inject the cluster CA bundle in this ConfigMap (service-ca.crt key).
func NewConfigMapCABundle(namespace string, name string, labels labels.Set) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: labels,
Annotations: map[string]string{"service.beta.openshift.io/inject-cabundle": "true"},
},
}
}

func findContainerIndex(pod *corev1.PodSpec, containerName string) (int, error) {
for i, container := range pod.Containers {
if container.Name == containerName {
Expand Down
24 changes: 23 additions & 1 deletion internal/manifests/monolithic/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ import (
"maps"

"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/grafana/tempo-operator/internal/manifests/manifestutils"
"github.com/grafana/tempo-operator/internal/manifests/naming"
)

// BuildAll generates all manifests.
Expand All @@ -16,22 +19,41 @@ func BuildAll(opts Options) ([]client.Object, error) {
if err != nil {
return nil, err
}

manifests = append(manifests, configMap)
maps.Copy(extraStsAnnotations, annotations)

if tempo.Spec.ServiceAccount == "" {
manifests = append(manifests, BuildServiceAccount(opts))
}

if tempo.Spec.Multitenancy.IsGatewayEnabled() {
objs, annotations, err := BuildGatewayObjects(opts)
if err != nil {
return nil, err
}

manifests = append(manifests, objs...)
maps.Copy(extraStsAnnotations, annotations)
}

statefulSet, err := BuildTempoStatefulset(opts, extraStsAnnotations)
if err != nil {
return nil, err
}

manifests = append(manifests, statefulSet)

manifests = append(manifests, BuildServiceAccount(opts))
manifests = append(manifests, BuildServices(opts)...)

if opts.CtrlConfig.Gates.OpenShift.ServingCertsService {
manifests = append(manifests, manifestutils.NewConfigMapCABundle(
tempo.Namespace,
naming.ServingCABundleName(tempo.Name),
CommonLabels(tempo.Name),
))
}

if tempo.Spec.JaegerUI != nil && tempo.Spec.JaegerUI.Enabled {
if tempo.Spec.JaegerUI.Ingress != nil && tempo.Spec.JaegerUI.Ingress.Enabled {
manifests = append(manifests, BuildJaegerUIIngress(opts))
Expand Down
19 changes: 19 additions & 0 deletions internal/manifests/monolithic/configmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ type tempoGCSConfig struct {
}

type tempoConfig struct {
MultitenancyEnabled bool `yaml:"multitenancy_enabled,omitempty"`

Server struct {
HTTPListenAddress string `yaml:"http_listen_address,omitempty"`
HttpListenPort int `yaml:"http_listen_port,omitempty"`
Expand Down Expand Up @@ -152,7 +154,19 @@ func buildTempoConfig(opts Options) ([]byte, error) {
tempo := opts.Tempo

config := tempoConfig{}
config.MultitenancyEnabled = tempo.Spec.Multitenancy != nil && tempo.Spec.Multitenancy.Enabled
config.Server.HttpListenPort = manifestutils.PortHTTPServer
if tempo.Spec.Multitenancy.IsGatewayEnabled() {
// all connections to tempo must go via gateway
config.Server.HTTPListenAddress = "localhost"
config.Server.GRPCListenAddress = "localhost"
}

// The internal server is required because if the gateway is enabled,
// the Tempo API will listen on localhost only,
// and then Kubernetes cannot reach the health check endpoint.
config.InternalServer.Enable = true
config.InternalServer.HTTPListenAddress = "0.0.0.0"

// The internal server is required because if the gateway is enabled,
// the Tempo API will listen on localhost only,
Expand Down Expand Up @@ -207,7 +221,12 @@ func buildTempoConfig(opts Options) ([]byte, error) {
config.Distributor.Receivers.OTLP.Protocols.GRPC = &tempoReceiverConfig{
TLS: configureReceiverTLS(tempo.Spec.Ingestion.OTLP.GRPC.TLS),
}
if tempo.Spec.Multitenancy.IsGatewayEnabled() {
// all connections to tempo must go via gateway
config.Distributor.Receivers.OTLP.Protocols.GRPC.Endpoint = fmt.Sprintf("localhost:%d", manifestutils.PortOtlpGrpcServer)
}
}

if tempo.Spec.Ingestion.OTLP.HTTP != nil && tempo.Spec.Ingestion.OTLP.HTTP.Enabled {
config.Distributor.Receivers.OTLP.Protocols.HTTP = &tempoReceiverConfig{
TLS: configureReceiverTLS(tempo.Spec.Ingestion.OTLP.HTTP.TLS),
Expand Down
74 changes: 74 additions & 0 deletions internal/manifests/monolithic/gateway.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package monolithic

import (
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/grafana/tempo-operator/apis/tempo/v1alpha1"
"github.com/grafana/tempo-operator/internal/manifests/gateway"
"github.com/grafana/tempo-operator/internal/manifests/manifestutils"
"github.com/grafana/tempo-operator/internal/manifests/naming"
)

const (
opaPackage = "tempomonolithic"
)

// BuildGatewayObjects builds auxiliary objects required for multitenancy.
//
// Enabling multi-tenancy (multitenancy.enabled=true) and configuring at least one tenant has the following effect on the deployment:
// * a tempo-$name-gateway ConfigMap (rbac.yaml) and Secret (tenants.yaml) will be created
// * a tempo-gateway container is added to the StatefulSet
// * the tempo-$name service will point to the gateway instead of the StatefulSet (http, otlp-grpc and jaeger-ui ports)
//
// additionally, if mode=openshift
// * a tempo-gateway-opa container is added to the StatefulSet
// * the ServiceAccount will get additional annotations (serviceaccounts.openshift.io/oauth-redirectreference.$tenantName)
// * a ClusterRole (tokenreviews/create and subjectaccessreviews/create) and ClusterRoleBinding will be created for the ServiceAccount.
func BuildGatewayObjects(opts Options) ([]client.Object, map[string]string, error) {
tempo := opts.Tempo
manifests := []client.Object{}
extraAnnotations := map[string]string{}
labels := ComponentLabels(manifestutils.GatewayComponentName, tempo.Name)
gatewayObjectName := naming.Name(manifestutils.GatewayComponentName, tempo.Name)

cfgOpts := gateway.NewConfigOptions(
tempo.Namespace,
tempo.Name,
naming.DefaultServiceAccountName(tempo.Name),
naming.RouteFqdn(tempo.Namespace, tempo.Name, "jaegerui", opts.CtrlConfig.Gates.OpenShift.BaseDomain),
opaPackage,
tempo.Spec.Multitenancy.TenantsSpec,
opts.GatewayTenantSecret,
opts.GatewayTenantsData,
)

rbacConfigMap, rbacHash, err := gateway.NewRBACConfigMap(cfgOpts, tempo.Namespace, gatewayObjectName, labels)
if err != nil {
return nil, nil, err
}
extraAnnotations["tempo.grafana.com/rbacConfig.hash"] = rbacHash
manifests = append(manifests, rbacConfigMap)

tenantsSecret, tenantsHash, err := gateway.NewTenantsSecret(cfgOpts, tempo.Namespace, gatewayObjectName, labels)
if err != nil {
return nil, nil, err
}
extraAnnotations["tempo.grafana.com/tenantsConfig.hash"] = tenantsHash
manifests = append(manifests, tenantsSecret)

if tempo.Spec.Multitenancy.TenantsSpec.Mode == v1alpha1.ModeOpenShift {
manifests = append(manifests, gateway.NewAccessReviewClusterRole(
gatewayObjectName,
ComponentLabels(manifestutils.GatewayComponentName, tempo.Name),
))

manifests = append(manifests, gateway.NewAccessReviewClusterRoleBinding(
gatewayObjectName,
ComponentLabels(manifestutils.GatewayComponentName, tempo.Name),
tempo.Namespace,
naming.DefaultServiceAccountName(tempo.Name),
))
}

return manifests, extraAnnotations, nil
}
53 changes: 53 additions & 0 deletions internal/manifests/monolithic/gateway_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package monolithic

import (
"testing"

"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

configv1alpha1 "github.com/grafana/tempo-operator/apis/config/v1alpha1"
"github.com/grafana/tempo-operator/apis/tempo/v1alpha1"
)

func TestGateway(t *testing.T) {
opts := Options{
CtrlConfig: configv1alpha1.ProjectConfig{
Gates: configv1alpha1.FeatureGates{
OpenShift: configv1alpha1.OpenShiftFeatureGates{
ServingCertsService: true,
},
},
},
Tempo: v1alpha1.TempoMonolithic{
ObjectMeta: metav1.ObjectMeta{
Name: "sample",
Namespace: "default",
},
Spec: v1alpha1.TempoMonolithicSpec{
Multitenancy: &v1alpha1.MonolithicMultitenancySpec{
Enabled: true,
TenantsSpec: v1alpha1.TenantsSpec{
Mode: v1alpha1.ModeOpenShift,
Authentication: []v1alpha1.AuthenticationSpec{
{
TenantName: "dev",
TenantID: "1610b0c3-c509-4592-a256-a1871353dbfa",
},
{
TenantName: "prod",
TenantID: "1610b0c3-c509-4592-a256-a1871353dbfb",
},
},
},
},
},
},
}
objs, annotations, err := BuildGatewayObjects(opts)
require.NoError(t, err)
require.Len(t, objs, 4)

require.Contains(t, annotations, "tempo.grafana.com/rbacConfig.hash")
require.Contains(t, annotations, "tempo.grafana.com/tenantsConfig.hash")
}
Loading

0 comments on commit 1fdddba

Please sign in to comment.