Add e2e tests for multitenancy on Kubernetes

[frzifus]

Should look like:

Install

---
apiVersion: v1
kind: Secret
metadata:
   name: minio-test
stringData:
  endpoint: http://minio.minio.svc:9000
  bucket: tempo
  access_key_id: tempo
  access_key_secret: supersecret
type: Opaque
---
apiVersion: tempo.grafana.com/v1alpha1
kind: Microservices
metadata:
  name: foo
spec:
  storage:
    secret: minio-test
  storageSize: 200M
  tenants:
    mode: static
    authentication:
      - tenantName: test-oidc
        tenantId: test-oidc
        oidc:
          secret:
            name: test-oidc
          issuerURL: https://dex.klimlive.de/dex
    authorization:
      roleBindings:
      - name: test-oidc
        roles:
        - read-write
        subjects:
        - kind: user
          name: user
      roles:
      - name: read-write
        permissions:
        - read
        - write
        resources:
        - logs
        - metrics
        - traces
        tenants:
        - test-oidc

Assert

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: tempo-foo-gateway
  labels:
    app.kubernetes.io/component: gateway
    app.kubernetes.io/created-by: tempo-controller
    app.kubernetes.io/instance: foo
    app.kubernetes.io/managed-by: tempo-controller
    app.kubernetes.io/name: tempo
  annotations: {}
data:
  rbac.yaml: "roleBindings:\n- name: test-oidc\n  roles:\n  - read-write\n\n  subjects:\n  - kind: user\n    name: user\n\nroles:\n- name: read-write\n  permissions:\n  - read\n  - write\n\n  resources:\n  - logs\n  - traces\n  - metrics\n\n  tenants:\n  - test-oidc"
  tempo-gateway.rego: "package tempo\n\nimport input\nimport data.roles\nimport data.roleBindings\n\ndefault allow = false\n\nallow {\n  some roleNames\n  roleNames = roleBindings[matched_role_binding[_]].roles\n  roles[i].name == roleNames[_]\n  roles[i].resources[_] = input.resource\n  roles[i].permissions[_] = input.permission\n  roles[i].tenants[_] = input.tenant\n}\n\nmatched_role_binding[i] {\n  roleBindings[i].subjects[_] == {\"name\": input.subject, \"kind\": \"user\"}\n}\n\nmatched_role_binding[i] {\n  roleBindings[i].subjects[_] == {\"name\": input.groups[_], \"kind\": \"group\"}\n}\n"
---
apiVersion: v1
kind: Secret
metadata:
  name: tempo-foo-gateway
  labels:
    app.kubernetes.io/component: gateway
    app.kubernetes.io/created-by: tempo-controller
    app.kubernetes.io/instance: foo
    app.kubernetes.io/managed-by: tempo-controller
    app.kubernetes.io/name: tempo
data:
  tenants.yaml: dGVuYW50czoKLSBuYW1lOiB0ZXN0LW9pZGMKICBpZDogdGVzdC1vaWRjCiAgb2lkYzoKICAgIGlzc3VlclVSTDogaHR0cHM6Ly9kZXgua2xpbWxpdmUuZGUvZGV4CiAgICAKICBvcGE6CiAgICBxdWVyeTogZGF0YS50ZW1wby5hbGxvdwogICAgcGF0aHM6CiAgICAtIC9ldGMvdGVtcG8tZ2F0ZXdheS9yYmFjLnlhbWwKICAgIC0gL2V0Yy90ZW1wby1nYXRld2F5L3RlbXBvLWdhdGV3YXkucmVnbw==
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tempo-foo-gateway
  labels:
    app.kubernetes.io/component: gateway
    app.kubernetes.io/created-by: tempo-controller
    app.kubernetes.io/instance: foo
    app.kubernetes.io/managed-by: tempo-controller
    app.kubernetes.io/name: tempo
spec:
  selector:
    matchlabels:
      app.kubernetes.io/component: gateway
      app.kubernetes.io/created-by: tempo-controller
      app.kubernetes.io/instance: foo
      app.kubernetes.io/managed-by: tempo-controller
      app.kubernetes.io/name: tempo
  template:
    metadata:
      labels:
        app.kubernetes.io/component: gateway
        app.kubernetes.io/created-by: tempo-controller
        app.kubernetes.io/instance: foo
        app.kubernetes.io/managed-by: tempo-controller
        app.kubernetes.io/name: tempo
    spec:
      annotations:
        tempo.grafana.com/config.hash: e39fb269cbe21cc0310da630a79fe34ffb35d34022d8b5c264a697133eb312b7
    spec:
      volumes:
      - name: rbac
        volumesource:
          configmap:
            localobjectreference:
              name: tempo-foo-gateway
      - name: tenants
        volumesource:
          configmap:
            localobjectreference:
              name: tempo-foo-gateway
      - name: tempo-gateway
        volumesource:
          secret:
            secretname: tempo-foo-gateway
      containers:
      - name: tempo
        image: ""
        command: []
        args:
        - --web.listen=0.0.0.0:8080
        - --web.internal.listen=0.0.0.0:8081
        - --traces.write.endpoint=tempo-foo-distributor:4317
        - --traces.read.endpoint=tempo-foo-query:16686
        - --grpc.listen=0.0.0.0:8090
        - --rbac.config=/etc/observatorium/rbac.yaml
        - --tenants.config=/etc/observatorium/tenants.yaml
        - --log.level=error
        ports:
        - name: grpc-public
          hostport: 0
          containerport: 8090
          protocol: TCP
          hostip: ""
        - name: internal
          hostport: 0
          containerport: 8081
          protocol: TCP
          hostip: ""
        - name: public
          hostport: 0
          containerport: 8080
          protocol: TCP
          hostip: ""
        resources:
          limits: {}
          requests: {}
        volumemounts:
        - name: rbac
          readonly: true
          mountpath: /etc/tempo-gateway/rbac.yaml
          subpath: rbac.yaml
        - name: tenants
          readonly: true
          mountpath: /etc/tempo-gateway/tenants.yaml
          subpath: tenants.yaml
        - name: tempo-gateway
          readonly: true
          mountpath: /etc/tempo-gateway/tempo-gateway.rego
          subpath: lokistack-gateway.rego
        livenessprobe:
          probehandler:
            exec: null
            httpget:
              path: /live
              port:
                type: 0
                intval: 8081
                strval: ""
              host: ""
              scheme: HTTP
          timeoutseconds: 2
          periodseconds: 30
          failurethreshold: 10
        readinessprobe:
          probehandler:
            httpget:
              path: /ready
              port:
                type: 0
                intval: 8081
                strval: ""
              host: ""
              scheme: HTTP
          initialdelayseconds: 0
          timeoutseconds: 1
          periodseconds: 5
          successthreshold: 0
          failurethreshold: 12
        securitycontext:
          capabilities:
            add: []
            drop:
            - ALL
          privileged: null
          selinuxoptions: null
          windowsoptions: null
          runasuser: null
          runasgroup: null
          runasnonroot: null
          readonlyrootfilesystem: true
          allowprivilegeescalation: false
          procmount: null
          seccompprofile: null

[pavolloffay]

Data reporting/querying should be added to the test https://github.com/grafana/tempo-operator/tree/main/tests/e2e/gateway

[andreasgerstmayr]

A smoketest for monolithic + gateway in static mode on OpenShift is in #816.
The OpenTelemetry collector refuses to do authentication over HTTP, therefore this test is currently in the OpenShift testsuite. Once the gateway supports TLS on non-OpenShift, we can move it.