From b43b528580e5c26b6f24908dfcaeef58871dfc3c Mon Sep 17 00:00:00 2001 From: Ben B Date: Mon, 12 Dec 2022 11:18:50 +0100 Subject: [PATCH] Support openshift routes (#1206) * split service port from config calculation into its own function Signed-off-by: Benedikt Bongartz * register openshift route v1 as valid ingress enum Signed-off-by: Benedikt Bongartz * add routes to reconcile loop - determine platform - grant otel controller permission to route api Signed-off-by: Benedikt Bongartz * add openshift api to go mod Signed-off-by: Benedikt Bongartz * add naming method for openshift routes Signed-off-by: Benedikt Bongartz * add route reconcile routine Signed-off-by: Benedikt Bongartz * add route reconciler if platform changes to openshift Signed-off-by: Benedikt Bongartz * move route cr definition into testdata package Signed-off-by: Benedikt Bongartz * controllers: verify that route is created Signed-off-by: Benedikt Bongartz * crd: move route tls termination settings into extra section Signed-off-by: Benedikt Bongartz * fix: share platform state across copied config objects Signed-off-by: Benedikt Bongartz * controller: split opentelemetry collector callback Signed-off-by: Benedikt Bongartz * tests: add route e2e tests Signed-off-by: Benedikt Bongartz * fix govet linting ``` main.go:238:16: shadow: declaration of "err" shadows declaration at line 230 (govet) configBytes, err := yaml.Marshal(configs) ^ ``` Signed-off-by: Benedikt Bongartz * add ingress workaround description Signed-off-by: Benedikt Bongartz * regenerate Signed-off-by: Benedikt Bongartz Signed-off-by: Benedikt Bongartz --- Makefile | 6 +- apis/v1alpha1/ingress_type.go | 24 +- apis/v1alpha1/opentelemetrycollector_types.go | 18 ++ .../opentelemetrycollector_webhook.go | 3 + .../opentelemetrycollector_webhook_test.go | 29 +++ apis/v1alpha1/zz_generated.deepcopy.go | 16 ++ ...emetry-operator.clusterserviceversion.yaml | 12 + ...ntelemetry.io_opentelemetrycollectors.yaml | 15 ++ cmd/otel-allocator/main.go | 6 +- ...ntelemetry.io_opentelemetrycollectors.yaml | 15 ++ config/rbac/role.yaml | 12 + .../opentelemetrycollector_controller.go | 79 +++++- .../opentelemetrycollector_controller_test.go | 29 ++- controllers/suite_test.go | 9 + docs/api.md | 38 ++- go.mod | 1 + go.sum | 2 + hack/install-openshift-routes.sh | 6 + internal/config/main.go | 56 +++-- internal/config/options.go | 4 +- main.go | 2 + pkg/collector/reconcile/ingress.go | 64 ++--- pkg/collector/reconcile/route.go | 209 ++++++++++++++++ pkg/collector/reconcile/route_test.go | 236 ++++++++++++++++++ pkg/collector/reconcile/suite_test.go | 11 + pkg/collector/testdata/route_crd.go | 74 ++++++ pkg/naming/main.go | 5 + tests/e2e/route/00-assert.yaml | 31 +++ tests/e2e/route/00-install.yaml | 30 +++ 29 files changed, 974 insertions(+), 68 deletions(-) create mode 100755 hack/install-openshift-routes.sh create mode 100644 pkg/collector/reconcile/route.go create mode 100644 pkg/collector/reconcile/route_test.go create mode 100644 pkg/collector/testdata/route_crd.go create mode 100644 tests/e2e/route/00-assert.yaml create mode 100644 tests/e2e/route/00-install.yaml diff --git a/Makefile b/Makefile index 51e233c787..3ca33d1de8 100644 --- a/Makefile +++ b/Makefile @@ -171,7 +171,7 @@ e2e-log-operator: kubectl get deploy -A .PHONY: prepare-e2e -prepare-e2e: kuttl set-test-image-vars set-image-controller container container-target-allocator start-kind install-metrics-server load-image-all +prepare-e2e: kuttl set-test-image-vars set-image-controller container container-target-allocator start-kind install-metrics-server install-openshift-routes load-image-all mkdir -p tests/_build/crds tests/_build/manifests $(KUSTOMIZE) build config/default -o tests/_build/manifests/01-opentelemetry-operator.yaml $(KUSTOMIZE) build config/crd -o tests/_build/crds/ @@ -208,6 +208,10 @@ start-kind: install-metrics-server: ./hack/install-metrics-server.sh +.PHONY: install-openshift-routes +install-openshift-routes: + ./hack/install-openshift-routes.sh + .PHONY: load-image-all load-image-all: load-image-operator load-image-target-allocator diff --git a/apis/v1alpha1/ingress_type.go b/apis/v1alpha1/ingress_type.go index 5ef8528b04..f7377617cd 100644 --- a/apis/v1alpha1/ingress_type.go +++ b/apis/v1alpha1/ingress_type.go @@ -16,11 +16,33 @@ package v1alpha1 type ( // IngressType represents how a collector should be exposed (ingress vs route). - // +kubebuilder:validation:Enum=ingress + // +kubebuilder:validation:Enum=ingress;route IngressType string ) const ( // IngressTypeNginx specifies that an ingress entry should be created. IngressTypeNginx IngressType = "ingress" + // IngressTypeOpenshiftRoute specifies that an route entry should be created. + IngressTypeRoute IngressType = "route" +) + +type ( + // TLSRouteTerminationType is used to indicate which tls settings should be used. + // +kubebuilder:validation:Enum=insecure;edge;passthrough;reencrypt + TLSRouteTerminationType string +) + +const ( + // TLSRouteTerminationTypeInsecure indicates that insecure connections are allowed. + TLSRouteTerminationTypeInsecure TLSRouteTerminationType = "insecure" + // TLSRouteTerminationTypeEdge indicates that encryption should be terminated + // at the edge router. + TLSRouteTerminationTypeEdge TLSRouteTerminationType = "edge" + // TLSTerminationPassthrough indicates that the destination service is + // responsible for decrypting traffic. + TLSRouteTerminationTypePassthrough TLSRouteTerminationType = "passthrough" + // TLSTerminationReencrypt indicates that traffic will be decrypted on the edge + // and re-encrypt using a new certificate. + TLSRouteTerminationTypeReencrypt TLSRouteTerminationType = "reencrypt" ) diff --git a/apis/v1alpha1/opentelemetrycollector_types.go b/apis/v1alpha1/opentelemetrycollector_types.go index a296381b23..3331a5adae 100644 --- a/apis/v1alpha1/opentelemetrycollector_types.go +++ b/apis/v1alpha1/opentelemetrycollector_types.go @@ -24,6 +24,13 @@ import ( // Ingress is used to specify how OpenTelemetry Collector is exposed. This // functionality is only available if one of the valid modes is set. // Valid modes are: deployment, daemonset and statefulset. +// NOTE: If this feature is activated, all specified receivers are exposed. +// Currently this has a few limitations. Depending on the ingress controller +// there are problems with TLS and gRPC. +// SEE: https://github.com/open-telemetry/opentelemetry-operator/issues/1306. +// NOTE: As a workaround, port name and appProtocol could be specified directly +// in the CR. +// SEE: OpenTelemetryCollector.spec.ports[index]. type Ingress struct { // Type default value is: "" // Supported types are: ingress @@ -47,6 +54,17 @@ type Ingress struct { // serving this Ingress resource. // +optional IngressClassName *string `json:"ingressClassName,omitempty"` + + // Route is an OpenShift specific section that is only considered when + // type "route" is used. + // +optional + Route OpenShiftRoute `json:"route,omitempty"` +} + +// OpenShiftRoute defines openshift route specific settings. +type OpenShiftRoute struct { + // Termination indicates termination type. By default "edge" is used. + Termination TLSRouteTerminationType `json:"termination,omitempty"` } // OpenTelemetryCollectorSpec defines the desired state of OpenTelemetryCollector. diff --git a/apis/v1alpha1/opentelemetrycollector_webhook.go b/apis/v1alpha1/opentelemetrycollector_webhook.go index a4a1ee222c..3899218678 100644 --- a/apis/v1alpha1/opentelemetrycollector_webhook.go +++ b/apis/v1alpha1/opentelemetrycollector_webhook.go @@ -77,6 +77,9 @@ func (r *OpenTelemetryCollector) Default() { r.Spec.Autoscaler.TargetCPUUtilization = &defaultCPUTarget } } + if r.Spec.Ingress.Type == IngressTypeRoute && r.Spec.Ingress.Route.Termination == "" { + r.Spec.Ingress.Route.Termination = TLSRouteTerminationTypeEdge + } } // +kubebuilder:webhook:verbs=create;update,path=/validate-opentelemetry-io-v1alpha1-opentelemetrycollector,mutating=false,failurePolicy=fail,groups=opentelemetry.io,resources=opentelemetrycollectors,versions=v1alpha1,name=vopentelemetrycollectorcreateupdate.kb.io,sideEffects=none,admissionReviewVersions=v1 diff --git a/apis/v1alpha1/opentelemetrycollector_webhook_test.go b/apis/v1alpha1/opentelemetrycollector_webhook_test.go index 0ebc656903..29e29dfe38 100644 --- a/apis/v1alpha1/opentelemetrycollector_webhook_test.go +++ b/apis/v1alpha1/opentelemetrycollector_webhook_test.go @@ -96,6 +96,35 @@ func TestOTELColDefaultingWebhook(t *testing.T) { }, }, }, + { + name: "Missing route termination", + otelcol: OpenTelemetryCollector{ + Spec: OpenTelemetryCollectorSpec{ + Mode: ModeDeployment, + Ingress: Ingress{ + Type: IngressTypeRoute, + }, + }, + }, + expected: OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "opentelemetry-operator", + }, + }, + Spec: OpenTelemetryCollectorSpec{ + Mode: ModeDeployment, + Ingress: Ingress{ + Type: IngressTypeRoute, + Route: OpenShiftRoute{ + Termination: TLSRouteTerminationTypeEdge, + }, + }, + Replicas: &one, + UpgradeStrategy: UpgradeStrategyAutomatic, + }, + }, + }, } for _, test := range tests { diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 01c5f1dbdc..6803230c8c 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -115,6 +115,7 @@ func (in *Ingress) DeepCopyInto(out *Ingress) { *out = new(string) **out = **in } + out.Route = in.Route } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ingress. @@ -279,6 +280,21 @@ func (in *NodeJS) DeepCopy() *NodeJS { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenShiftRoute) DeepCopyInto(out *OpenShiftRoute) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenShiftRoute. +func (in *OpenShiftRoute) DeepCopy() *OpenShiftRoute { + if in == nil { + return nil + } + out := new(OpenShiftRoute) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenTelemetryCollector) DeepCopyInto(out *OpenTelemetryCollector) { *out = *in diff --git a/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml b/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml index e0c48808f9..e7e975ae01 100644 --- a/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml +++ b/bundle/manifests/opentelemetry-operator.clusterserviceversion.yaml @@ -257,6 +257,18 @@ spec: - get - patch - update + - apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - authentication.k8s.io resources: diff --git a/bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml b/bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml index 38dbce2c79..471ff6b949 100644 --- a/bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml +++ b/bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml @@ -1206,6 +1206,20 @@ spec: resource. Ingress controller implementations use this field to know whether they should be serving this Ingress resource. type: string + route: + description: Route is an OpenShift specific section that is only + considered when type "route" is used. + properties: + termination: + description: Termination indicates termination type. By default + "edge" is used. + enum: + - insecure + - edge + - passthrough + - reencrypt + type: string + type: object tls: description: TLS configuration. items: @@ -1236,6 +1250,7 @@ spec: description: 'Type default value is: "" Supported types are: ingress' enum: - ingress + - route type: string type: object maxReplicas: diff --git a/cmd/otel-allocator/main.go b/cmd/otel-allocator/main.go index f15b4a47f9..041e2dd263 100644 --- a/cmd/otel-allocator/main.go +++ b/cmd/otel-allocator/main.go @@ -235,12 +235,14 @@ func (s *server) ScrapeConfigsHandler(w http.ResponseWriter, r *http.Request) { } // if the hashes are different, we need to recompute the scrape config if hash != s.compareHash { - configBytes, err := yaml.Marshal(configs) + var configBytes []byte + configBytes, err = yaml.Marshal(configs) if err != nil { s.errorHandler(w, err) return } - jsonConfig, err := yaml2.YAMLToJSON(configBytes) + var jsonConfig []byte + jsonConfig, err = yaml2.YAMLToJSON(configBytes) if err != nil { s.errorHandler(w, err) return diff --git a/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml b/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml index c6990780b0..3bd1e672e3 100644 --- a/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml +++ b/config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml @@ -1204,6 +1204,20 @@ spec: resource. Ingress controller implementations use this field to know whether they should be serving this Ingress resource. type: string + route: + description: Route is an OpenShift specific section that is only + considered when type "route" is used. + properties: + termination: + description: Termination indicates termination type. By default + "edge" is used. + enum: + - insecure + - edge + - passthrough + - reencrypt + type: string + type: object tls: description: TLS configuration. items: @@ -1234,6 +1248,7 @@ spec: description: 'Type default value is: "" Supported types are: ingress' enum: - ingress + - route type: string type: object maxReplicas: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7dc982d34e..37e368696b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -168,3 +168,15 @@ rules: - get - patch - update +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/controllers/opentelemetrycollector_controller.go b/controllers/opentelemetrycollector_controller.go index 6d3bb96a67..4f62c93905 100644 --- a/controllers/opentelemetrycollector_controller.go +++ b/controllers/opentelemetrycollector_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" "fmt" + "sync" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -34,6 +35,7 @@ import ( "github.com/open-telemetry/opentelemetry-operator/internal/config" "github.com/open-telemetry/opentelemetry-operator/pkg/autodetect" "github.com/open-telemetry/opentelemetry-operator/pkg/collector/reconcile" + "github.com/open-telemetry/opentelemetry-operator/pkg/platform" ) // OpenTelemetryCollectorReconciler reconciles a OpenTelemetryCollector object. @@ -42,8 +44,10 @@ type OpenTelemetryCollectorReconciler struct { recorder record.EventRecorder scheme *runtime.Scheme log logr.Logger - tasks []Task config config.Config + + tasks []Task + muTasks sync.RWMutex } // Task represents a reconciliation task to be executed by the reconciler. @@ -63,10 +67,65 @@ type Params struct { Config config.Config } +func (r *OpenTelemetryCollectorReconciler) onPlatformChange() error { + // NOTE: At the time the reconciler gets created, the platform type is still unknown. + plt := r.config.Platform() + var ( + routesIdx = -1 + ) + r.muTasks.Lock() + for i, t := range r.tasks { + // search for route reconciler + switch t.Name { + case "routes": + routesIdx = i + } + } + r.muTasks.Unlock() + + if err := r.addRouteTask(plt, routesIdx); err != nil { + return err + } + + return r.removeRouteTask(plt, routesIdx) +} + +func (r *OpenTelemetryCollectorReconciler) addRouteTask(plt platform.Platform, routesIdx int) error { + r.muTasks.Lock() + defer r.muTasks.Unlock() + // if exists and platform is openshift + if routesIdx == -1 && plt == platform.OpenShift { + r.tasks = append([]Task{{reconcile.Routes, "routes", true}}, r.tasks...) + } + return nil +} + +func (r *OpenTelemetryCollectorReconciler) removeRouteTask(plt platform.Platform, routesIdx int) error { + r.muTasks.Lock() + defer r.muTasks.Unlock() + if len(r.tasks) < routesIdx { + return fmt.Errorf("can not remove route task from reconciler") + } + // if exists and platform is not openshift + if routesIdx != -1 && plt != platform.OpenShift { + r.tasks = append(r.tasks[:routesIdx], r.tasks[routesIdx+1:]...) + } + return nil +} + // NewReconciler creates a new reconciler for OpenTelemetryCollector objects. func NewReconciler(p Params) *OpenTelemetryCollectorReconciler { - if len(p.Tasks) == 0 { - p.Tasks = []Task{ + r := &OpenTelemetryCollectorReconciler{ + Client: p.Client, + log: p.Log, + scheme: p.Scheme, + config: p.Config, + tasks: p.Tasks, + recorder: p.Recorder, + } + + if len(r.tasks) == 0 { + r.tasks = []Task{ { reconcile.ConfigMaps, "config maps", @@ -113,22 +172,16 @@ func NewReconciler(p Params) *OpenTelemetryCollectorReconciler { true, }, } + r.config.RegisterPlatformChangeCallback(r.onPlatformChange) } - - return &OpenTelemetryCollectorReconciler{ - Client: p.Client, - log: p.Log, - scheme: p.Scheme, - config: p.Config, - tasks: p.Tasks, - recorder: p.Recorder, - } + return r } // +kubebuilder:rbac:groups=opentelemetry.io,resources=opentelemetrycollectors,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=opentelemetry.io,resources=opentelemetrycollectors/status,verbs=get;update;patch // +kubebuilder:rbac:groups=opentelemetry.io,resources=opentelemetrycollectors/finalizers,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch @@ -166,6 +219,8 @@ func (r *OpenTelemetryCollectorReconciler) Reconcile(ctx context.Context, req ct // RunTasks runs all the tasks associated with this reconciler. func (r *OpenTelemetryCollectorReconciler) RunTasks(ctx context.Context, params reconcile.Params) error { + r.muTasks.RLock() + defer r.muTasks.RUnlock() for _, task := range r.tasks { if err := task.Do(ctx, params); err != nil { // If we get an error that occurs because a pod is being terminated, then exit this loop diff --git a/controllers/opentelemetrycollector_controller_test.go b/controllers/opentelemetrycollector_controller_test.go index 9b52cbb185..b9d83572dc 100644 --- a/controllers/opentelemetrycollector_controller_test.go +++ b/controllers/opentelemetrycollector_controller_test.go @@ -20,6 +20,7 @@ import ( "fmt" "testing" + routev1 "github.com/openshift/api/route/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" @@ -46,11 +47,18 @@ var mockAutoDetector = &mockAutoDetect{ HPAVersionFunc: func() (autodetect.AutoscalingVersion, error) { return autodetect.AutoscalingVersionV2Beta2, nil }, + PlatformFunc: func() (platform.Platform, error) { + return platform.OpenShift, nil + }, } func TestNewObjectsOnReconciliation(t *testing.T) { // prepare - cfg := config.New(config.WithCollectorImage("default-collector"), config.WithTargetAllocatorImage("default-ta-allocator"), config.WithAutoDetect(mockAutoDetector)) + cfg := config.New( + config.WithCollectorImage("default-collector"), + config.WithTargetAllocatorImage("default-ta-allocator"), + config.WithAutoDetect(mockAutoDetector), + ) nsn := types.NamespacedName{Name: "my-instance", Namespace: "default"} reconciler := controllers.NewReconciler(controllers.Params{ Client: k8sClient, @@ -58,6 +66,7 @@ func TestNewObjectsOnReconciliation(t *testing.T) { Scheme: testScheme, Config: cfg, }) + require.NoError(t, cfg.AutoDetect()) created := &v1alpha1.OpenTelemetryCollector{ ObjectMeta: metav1.ObjectMeta{ Name: nsn.Name, @@ -65,6 +74,18 @@ func TestNewObjectsOnReconciliation(t *testing.T) { }, Spec: v1alpha1.OpenTelemetryCollectorSpec{ Mode: v1alpha1.ModeDeployment, + Ports: []corev1.ServicePort{ + { + Name: "telnet", + Port: 49935, + }, + }, + Ingress: v1alpha1.Ingress{ + Type: v1alpha1.IngressTypeRoute, + Route: v1alpha1.OpenShiftRoute{ + Termination: v1alpha1.TLSRouteTerminationTypeInsecure, + }, + }, }, } err := k8sClient.Create(context.Background(), created) @@ -128,6 +149,12 @@ func TestNewObjectsOnReconciliation(t *testing.T) { // attention! we expect statefulsets to be empty in the default configuration assert.Empty(t, list.Items) } + { + list := &routev1.RouteList{} + err = k8sClient.List(context.Background(), list, opts...) + assert.NoError(t, err) + assert.NotEmpty(t, list.Items) + } // cleanup require.NoError(t, k8sClient.Delete(context.Background(), created)) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 9b79181ead..be597e73c4 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -25,6 +25,8 @@ import ( "testing" "time" + routev1 "github.com/openshift/api/route/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" @@ -35,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" + "github.com/open-telemetry/opentelemetry-operator/pkg/collector/testdata" // +kubebuilder:scaffold:imports ) @@ -54,6 +57,7 @@ func TestMain(m *testing.M) { testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + CRDs: []*apiextensionsv1.CustomResourceDefinition{testdata.OpenShiftRouteCRD}, WebhookInstallOptions: envtest.WebhookInstallOptions{ Paths: []string{filepath.Join("..", "config", "webhook")}, }, @@ -64,6 +68,11 @@ func TestMain(m *testing.M) { os.Exit(1) } + if err = routev1.AddToScheme(testScheme); err != nil { + fmt.Printf("failed to register scheme: %v", err) + os.Exit(1) + } + if err = v1alpha1.AddToScheme(testScheme); err != nil { fmt.Printf("failed to register scheme: %v", err) os.Exit(1) diff --git a/docs/api.md b/docs/api.md index 9359a696c4..73ee92b8de 100644 --- a/docs/api.md +++ b/docs/api.md @@ -3843,6 +3843,13 @@ Ingress is used to specify how OpenTelemetry Collector is exposed. This function IngressClassName is the name of an IngressClass cluster resource. Ingress controller implementations use this field to know whether they should be serving this Ingress resource.
false + + route + object + + Route is an OpenShift specific section that is only considered when type "route" is used.
+ + false tls []object @@ -3856,7 +3863,36 @@ Ingress is used to specify how OpenTelemetry Collector is exposed. This function Type default value is: "" Supported types are: ingress

- Enum: ingress
+ Enum: ingress, route
+ + false + + + + +### OpenTelemetryCollector.spec.ingress.route +[↩ Parent](#opentelemetrycollectorspecingress) + + + +Route is an OpenShift specific section that is only considered when type "route" is used. + + + + + + + + + + + + + + diff --git a/go.mod b/go.mod index e14c053a58..0860325c33 100644 --- a/go.mod +++ b/go.mod @@ -97,6 +97,7 @@ require ( github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/openshift/api v3.9.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.2 // indirect diff --git a/go.sum b/go.sum index 561d25c1fb..300155a10f 100644 --- a/go.sum +++ b/go.sum @@ -822,6 +822,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= +github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= diff --git a/hack/install-openshift-routes.sh b/hack/install-openshift-routes.sh new file mode 100755 index 0000000000..573dee06a8 --- /dev/null +++ b/hack/install-openshift-routes.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/router_rbac.yaml +kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/route_crd.yaml +kubectl apply -f https://raw.githubusercontent.com/openshift/router/release-4.12/deploy/router.yaml +kubectl wait --for=condition=available deployment/ingress-router -n openshift-ingress --timeout=5m diff --git a/internal/config/main.go b/internal/config/main.go index 69f5ea9526..1f4ef087d9 100644 --- a/internal/config/main.go +++ b/internal/config/main.go @@ -16,6 +16,7 @@ package config import ( + "sync" "time" "github.com/go-logr/logr" @@ -46,7 +47,7 @@ type Config struct { autoInstrumentationJavaImage string onPlatformChange changeHandler labelsFilter []string - platform platform.Platform + platform platformStore autoDetectFrequency time.Duration autoscalingVersion autodetect.AutoscalingVersion } @@ -59,7 +60,7 @@ func New(opts ...Option) Config { collectorConfigMapEntry: defaultCollectorConfigMapEntry, targetAllocatorConfigMapEntry: defaultTargetAllocatorConfigMapEntry, logger: logf.Log.WithName("config"), - platform: platform.Unknown, + platform: newPlatformWrapper(), version: version.Get(), autoscalingVersion: autodetect.DefaultAutoscalingVersion, onPlatformChange: newOnChange(), @@ -108,25 +109,17 @@ func (c *Config) periodicAutoDetect() { // AutoDetect attempts to automatically detect relevant information for this operator. func (c *Config) AutoDetect() error { - changed := false c.logger.V(2).Info("auto-detecting the configuration based on the environment") - // TODO: once new things need to be detected, extract this into individual detection routines - if c.platform == platform.Unknown { - plt, err := c.autoDetect.Platform() - if err != nil { - return err - } - - if c.platform != plt { - c.logger.V(1).Info("platform detected", "platform", plt) - c.platform = plt - changed = true - } + plt, err := c.autoDetect.Platform() + if err != nil { + return err } - if changed { - if err := c.onPlatformChange.Do(); err != nil { + if c.platform.Get() != plt { + c.logger.V(1).Info("platform detected", "platform", plt) + c.platform.Set(plt) + if err = c.onPlatformChange.Do(); err != nil { // Don't fail if the callback failed, as auto-detection itself worked. c.logger.Error(err, "configuration change notification failed for callback") } @@ -164,7 +157,7 @@ func (c *Config) TargetAllocatorConfigMapEntry() string { // Platform represents the type of the platform this operator is running. func (c *Config) Platform() platform.Platform { - return c.platform + return c.platform.Get() } // AutoscalingVersion represents the preferred version of autoscaling. @@ -202,3 +195,30 @@ func (c *Config) LabelsFilter() []string { func (c *Config) RegisterPlatformChangeCallback(f func() error) { c.onPlatformChange.Register(f) } + +type platformStore interface { + Set(plt platform.Platform) + Get() platform.Platform +} + +func newPlatformWrapper() platformStore { + return &platformWrapper{} +} + +type platformWrapper struct { + mu sync.Mutex + current platform.Platform +} + +func (p *platformWrapper) Set(plt platform.Platform) { + p.mu.Lock() + p.current = plt + p.mu.Unlock() +} + +func (p *platformWrapper) Get() platform.Platform { + p.mu.Lock() + plt := p.current + p.mu.Unlock() + return plt +} diff --git a/internal/config/options.go b/internal/config/options.go index 34b5b13dde..c1e5993da2 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -43,7 +43,7 @@ type options struct { targetAllocatorImage string onPlatformChange changeHandler labelsFilter []string - platform platform.Platform + platform platformStore autoDetectFrequency time.Duration autoscalingVersion autodetect.AutoscalingVersion } @@ -94,7 +94,7 @@ func WithOnPlatformChangeCallback(f func() error) Option { } func WithPlatform(plt platform.Platform) Option { return func(o *options) { - o.platform = plt + o.platform.Set(plt) } } func WithVersion(v version.Version) Option { diff --git a/main.go b/main.go index 404abf83bd..797b6bccae 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "strings" "time" + routev1 "github.com/openshift/api/route/v1" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" @@ -66,6 +67,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(otelv1alpha1.AddToScheme(scheme)) + utilruntime.Must(routev1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } diff --git a/pkg/collector/reconcile/ingress.go b/pkg/collector/reconcile/ingress.go index 9ed20fbd59..fee7e8cd7f 100644 --- a/pkg/collector/reconcile/ingress.go +++ b/pkg/collector/reconcile/ingress.go @@ -36,36 +36,7 @@ func desiredIngresses(_ context.Context, params Params) *networkingv1.Ingress { return nil } - config, err := adapters.ConfigFromString(params.Instance.Spec.Config) - if err != nil { - params.Log.Error(err, "couldn't extract the configuration from the context") - return nil - } - - ports, err := adapters.ConfigToReceiverPorts(params.Log, config) - if err != nil { - params.Log.Error(err, "couldn't build the ingress for this instance") - return nil - } - - if len(params.Instance.Spec.Ports) > 0 { - // we should add all the ports from the CR - // there are two cases where problems might occur: - // 1) when the port number is already being used by a receiver - // 2) same, but for the port name - // - // in the first case, we remove the port we inferred from the list - // in the second case, we rename our inferred port to something like "port-%d" - portNumbers, portNames := extractPortNumbersAndNames(params.Instance.Spec.Ports) - resultingInferredPorts := []corev1.ServicePort{} - for _, inferred := range ports { - if filtered := filterPort(params.Log, inferred, portNumbers, portNames); filtered != nil { - resultingInferredPorts = append(resultingInferredPorts, *filtered) - } - } - - ports = append(params.Instance.Spec.Ports, resultingInferredPorts...) - } + ports := servicePortsFromCfg(params) // if we have no ports, we don't need a ingress entry if len(ports) == 0 { @@ -241,3 +212,36 @@ func deleteIngresses(ctx context.Context, params Params, expected []networkingv1 return nil } + +func servicePortsFromCfg(params Params) []corev1.ServicePort { + config, err := adapters.ConfigFromString(params.Instance.Spec.Config) + if err != nil { + params.Log.Error(err, "couldn't extract the configuration from the context") + return nil + } + + ports, err := adapters.ConfigToReceiverPorts(params.Log, config) + if err != nil { + params.Log.Error(err, "couldn't build the ingress for this instance") + } + + if len(params.Instance.Spec.Ports) > 0 { + // we should add all the ports from the CR + // there are two cases where problems might occur: + // 1) when the port number is already being used by a receiver + // 2) same, but for the port name + // + // in the first case, we remove the port we inferred from the list + // in the second case, we rename our inferred port to something like "port-%d" + portNumbers, portNames := extractPortNumbersAndNames(params.Instance.Spec.Ports) + resultingInferredPorts := []corev1.ServicePort{} + for _, inferred := range ports { + if filtered := filterPort(params.Log, inferred, portNumbers, portNames); filtered != nil { + resultingInferredPorts = append(resultingInferredPorts, *filtered) + } + } + + ports = append(params.Instance.Spec.Ports, resultingInferredPorts...) + } + return ports +} diff --git a/pkg/collector/reconcile/route.go b/pkg/collector/reconcile/route.go new file mode 100644 index 0000000000..50783611d7 --- /dev/null +++ b/pkg/collector/reconcile/route.go @@ -0,0 +1,209 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reconcile + +import ( + "context" + "fmt" + + routev1 "github.com/openshift/api/route/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" + "github.com/open-telemetry/opentelemetry-operator/pkg/naming" +) + +func desiredRoutes(_ context.Context, params Params) []routev1.Route { + var tlsCfg *routev1.TLSConfig + switch params.Instance.Spec.Ingress.Route.Termination { + case v1alpha1.TLSRouteTerminationTypeInsecure: + // NOTE: insecure, no tls cfg. + case v1alpha1.TLSRouteTerminationTypeEdge: + tlsCfg = &routev1.TLSConfig{Termination: routev1.TLSTerminationEdge} + case v1alpha1.TLSRouteTerminationTypePassthrough: + tlsCfg = &routev1.TLSConfig{Termination: routev1.TLSTerminationPassthrough} + case v1alpha1.TLSRouteTerminationTypeReencrypt: + tlsCfg = &routev1.TLSConfig{Termination: routev1.TLSTerminationReencrypt} + default: // NOTE: if unsupported, end here. + return nil + } + + ports := servicePortsFromCfg(params) + + // if we have no ports, we don't need a route entry + if len(ports) == 0 { + params.Log.V(1).Info( + "the instance's configuration didn't yield any ports to open, skipping route", + "instance.name", params.Instance.Name, + "instance.namespace", params.Instance.Namespace, + ) + return nil + } + + routes := make([]routev1.Route, len(ports)) + for i, p := range ports { + routes[i] = routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: naming.Route(params.Instance, p.Name), + Namespace: params.Instance.Namespace, + Annotations: params.Instance.Spec.Ingress.Annotations, + Labels: map[string]string{ + "app.kubernetes.io/name": naming.Route(params.Instance, p.Name), + "app.kubernetes.io/instance": fmt.Sprintf("%s.%s", params.Instance.Namespace, params.Instance.Name), + "app.kubernetes.io/managed-by": "opentelemetry-operator", + }, + }, + Spec: routev1.RouteSpec{ + Host: p.Name + "." + params.Instance.Spec.Ingress.Hostname, + Path: "/" + p.Name, + To: routev1.RouteTargetReference{ + Kind: "Service", + Name: naming.Service(params.Instance), + }, + Port: &routev1.RoutePort{ + // Valid names must be non-empty and no more than 15 characters long. + TargetPort: intstr.FromString(naming.Truncate(p.Name, 15)), + }, + WildcardPolicy: routev1.WildcardPolicyNone, + TLS: tlsCfg, + }, + } + } + return routes +} + +// Routes reconciles the route(s) required for the instance in the current context. +func Routes(ctx context.Context, params Params) error { + if params.Instance.Spec.Ingress.Type != v1alpha1.IngressTypeRoute { + return nil + } + + isSupportedMode := true + if params.Instance.Spec.Mode == v1alpha1.ModeSidecar { + params.Log.V(3).Info("ingress settings are not supported in sidecar mode") + isSupportedMode = false + } + + var desired []routev1.Route + if isSupportedMode { + if r := desiredRoutes(ctx, params); r != nil { + desired = append(desired, r...) + } + } + + // first, handle the create/update parts + if err := expectedRoutes(ctx, params, desired); err != nil { + return fmt.Errorf("failed to reconcile the expected routes: %w", err) + } + + // then, delete the extra objects + if err := deleteRoutes(ctx, params, desired); err != nil { + return fmt.Errorf("failed to reconcile the routes to be deleted: %w", err) + } + + return nil +} + +func expectedRoutes(ctx context.Context, params Params, expected []routev1.Route) error { + for _, obj := range expected { + desired := obj + + if err := controllerutil.SetControllerReference(¶ms.Instance, &desired, params.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + existing := &routev1.Route{} + nns := types.NamespacedName{Namespace: desired.Namespace, Name: desired.Name} + err := params.Client.Get(ctx, nns, existing) + if err != nil && k8serrors.IsNotFound(err) { + if err = params.Client.Create(ctx, &desired); err != nil { + return fmt.Errorf("failed to create: %w", err) + } + params.Log.V(2).Info("created", "route.name", desired.Name, "route.namespace", desired.Namespace) + continue + } else if err != nil { + return fmt.Errorf("failed to get: %w", err) + } + + // it exists already, merge the two if the end result isn't identical to the existing one + updated := existing.DeepCopy() + if updated.Annotations == nil { + updated.Annotations = map[string]string{} + } + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + updated.ObjectMeta.OwnerReferences = desired.ObjectMeta.OwnerReferences + updated.Spec.To = desired.Spec.To + updated.Spec.TLS = desired.Spec.TLS + updated.Spec.Port = desired.Spec.Port + updated.Spec.WildcardPolicy = desired.Spec.WildcardPolicy + + for k, v := range desired.ObjectMeta.Annotations { + updated.ObjectMeta.Annotations[k] = v + } + for k, v := range desired.ObjectMeta.Labels { + updated.ObjectMeta.Labels[k] = v + } + + patch := client.MergeFrom(existing) + + if err := params.Client.Patch(ctx, updated, patch); err != nil { + return fmt.Errorf("failed to apply changes: %w", err) + } + + params.Log.V(2).Info("applied", "route.name", desired.Name, "route.namespace", desired.Namespace) + } + return nil +} + +func deleteRoutes(ctx context.Context, params Params, expected []routev1.Route) error { + opts := []client.ListOption{ + client.InNamespace(params.Instance.Namespace), + client.MatchingLabels(map[string]string{ + "app.kubernetes.io/instance": fmt.Sprintf("%s.%s", params.Instance.Namespace, params.Instance.Name), + "app.kubernetes.io/managed-by": "opentelemetry-operator", + }), + } + list := &routev1.RouteList{} + if err := params.Client.List(ctx, list, opts...); err != nil { + return fmt.Errorf("failed to list: %w", err) + } + + for i := range list.Items { + existing := list.Items[i] + del := true + for _, keep := range expected { + if keep.Name == existing.Name && keep.Namespace == existing.Namespace { + del = false + break + } + } + + if del { + if err := params.Client.Delete(ctx, &existing); err != nil { + return fmt.Errorf("failed to delete: %w", err) + } + params.Log.V(2).Info("deleted", "route.name", existing.Name, "route.namespace", existing.Namespace) + } + } + + return nil +} diff --git a/pkg/collector/reconcile/route_test.go b/pkg/collector/reconcile/route_test.go new file mode 100644 index 0000000000..4ea82bc913 --- /dev/null +++ b/pkg/collector/reconcile/route_test.go @@ -0,0 +1,236 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reconcile + +import ( + "context" + _ "embed" + "fmt" + "strings" + "testing" + + routev1 "github.com/openshift/api/route/v1" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" + "github.com/open-telemetry/opentelemetry-operator/internal/config" + "github.com/open-telemetry/opentelemetry-operator/pkg/naming" +) + +func TestDesiredRoutes(t *testing.T) { + t.Run("should return nil invalid ingress type", func(t *testing.T) { + params := Params{ + Config: config.Config{}, + Client: k8sClient, + Log: logger, + Instance: v1alpha1.OpenTelemetryCollector{ + Spec: v1alpha1.OpenTelemetryCollectorSpec{ + Ingress: v1alpha1.Ingress{ + Type: v1alpha1.IngressType("unknown"), + }, + }, + }, + } + + actual := desiredRoutes(context.Background(), params) + assert.Nil(t, actual) + }) + + t.Run("should return nil unable to parse config", func(t *testing.T) { + params := Params{ + Config: config.Config{}, + Client: k8sClient, + Log: logger, + Instance: v1alpha1.OpenTelemetryCollector{ + Spec: v1alpha1.OpenTelemetryCollectorSpec{ + Config: "!!!", + Ingress: v1alpha1.Ingress{ + Type: v1alpha1.IngressTypeRoute, + }, + }, + }, + } + + actual := desiredRoutes(context.Background(), params) + assert.Nil(t, actual) + }) + + t.Run("should return nil unable to parse receiver ports", func(t *testing.T) { + params := Params{ + Config: config.Config{}, + Client: k8sClient, + Log: logger, + Instance: v1alpha1.OpenTelemetryCollector{ + Spec: v1alpha1.OpenTelemetryCollectorSpec{ + Config: "---", + Ingress: v1alpha1.Ingress{ + Type: v1alpha1.IngressTypeRoute, + }, + }, + }, + } + + actual := desiredRoutes(context.Background(), params) + assert.Nil(t, actual) + }) + + t.Run("should return nil unable to do something else", func(t *testing.T) { + var ( + ns = "test" + hostname = "example.com" + ) + + params, err := newParams("something:tag", testFileIngress) + if err != nil { + t.Fatal(err) + } + + params.Instance.Namespace = ns + params.Instance.Spec.Ingress = v1alpha1.Ingress{ + Type: v1alpha1.IngressTypeRoute, + Hostname: hostname, + Annotations: map[string]string{"some.key": "some.value"}, + Route: v1alpha1.OpenShiftRoute{ + Termination: v1alpha1.TLSRouteTerminationTypeInsecure, + }, + } + + got := desiredRoutes(context.Background(), params)[0] + + assert.NotEqual(t, &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: naming.Route(params.Instance, ""), + Namespace: ns, + Annotations: params.Instance.Spec.Ingress.Annotations, + Labels: map[string]string{ + "app.kubernetes.io/name": naming.Route(params.Instance, ""), + "app.kubernetes.io/instance": fmt.Sprintf("%s.%s", params.Instance.Namespace, params.Instance.Name), + "app.kubernetes.io/managed-by": "opentelemetry-operator", + }, + }, + Spec: routev1.RouteSpec{ + Host: hostname, + Path: "/abc", + To: routev1.RouteTargetReference{ + Kind: "service", + Name: "test-collector", + }, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromString("another-port"), + }, + WildcardPolicy: routev1.WildcardPolicyNone, + TLS: &routev1.TLSConfig{ + Termination: routev1.TLSTerminationPassthrough, + InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyAllow, + }, + }, + }, got) + }) +} + +func TestExpectedRoutes(t *testing.T) { + t.Run("should create and update route entry", func(t *testing.T) { + ctx := context.Background() + + params, err := newParams("something:tag", testFileIngress) + if err != nil { + t.Fatal(err) + } + params.Instance.Spec.Ingress.Type = v1alpha1.IngressTypeRoute + params.Instance.Spec.Ingress.Route.Termination = v1alpha1.TLSRouteTerminationTypeInsecure + + err = expectedRoutes(ctx, params, desiredRoutes(ctx, params)) + assert.NoError(t, err) + + nns := types.NamespacedName{Namespace: params.Instance.Namespace, Name: "otlp-grpc-test-route"} + exists, err := populateObjectIfExists(t, &routev1.Route{}, nns) + assert.NoError(t, err) + assert.True(t, exists) + + // update fields + const expectHostname = "something-else.com" + params.Instance.Spec.Ingress.Annotations = map[string]string{"blub": "blob"} + params.Instance.Spec.Ingress.Hostname = expectHostname + + err = expectedRoutes(ctx, params, desiredRoutes(ctx, params)) + assert.NoError(t, err) + + got := &routev1.Route{} + err = params.Client.Get(ctx, nns, got) + assert.NoError(t, err) + + gotHostname := got.Spec.Host + if !strings.Contains(gotHostname, got.Spec.Host) { + t.Errorf("host name is not up-to-date. expect: %s, got: %s", expectHostname, gotHostname) + } + + if v, ok := got.Annotations["blub"]; !ok || v != "blob" { + t.Error("annotations are not up-to-date. Missing entry or value is invalid.") + } + }) +} + +func TestDeleteRoutes(t *testing.T) { + t.Run("should delete excess routes", func(t *testing.T) { + // create + ctx := context.Background() + + myParams, err := newParams("something:tag", testFileIngress) + if err != nil { + t.Fatal(err) + } + myParams.Instance.Spec.Ingress.Type = v1alpha1.IngressTypeRoute + + err = expectedRoutes(ctx, myParams, desiredRoutes(ctx, myParams)) + assert.NoError(t, err) + + nns := types.NamespacedName{Namespace: "default", Name: "otlp-grpc-test-route"} + exists, err := populateObjectIfExists(t, &routev1.Route{}, nns) + assert.NoError(t, err) + assert.True(t, exists) + + // delete + if err = deleteRoutes(ctx, params(), []routev1.Route{}); err != nil { + t.Error(err) + } + + // check + exists, err = populateObjectIfExists(t, &routev1.Route{}, nns) + assert.NoError(t, err) + assert.False(t, exists) + }) +} + +func TestRoutes(t *testing.T) { + t.Run("wrong mode", func(t *testing.T) { + ctx := context.Background() + err := Routes(ctx, params()) + assert.Nil(t, err) + }) + + t.Run("supported mode and service exists", func(t *testing.T) { + ctx := context.Background() + myParams := params() + err := expectedServices(context.Background(), myParams, []corev1.Service{service("test-collector", params().Instance.Spec.Ports)}) + assert.NoError(t, err) + + assert.Nil(t, Routes(ctx, myParams)) + }) + +} diff --git a/pkg/collector/reconcile/suite_test.go b/pkg/collector/reconcile/suite_test.go index a83f3bf8fb..598807dd2d 100644 --- a/pkg/collector/reconcile/suite_test.go +++ b/pkg/collector/reconcile/suite_test.go @@ -25,8 +25,10 @@ import ( "testing" "time" + routev1 "github.com/openshift/api/route/v1" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -45,6 +47,7 @@ import ( "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" "github.com/open-telemetry/opentelemetry-operator/internal/config" + "github.com/open-telemetry/opentelemetry-operator/pkg/collector/testdata" ) var ( @@ -72,6 +75,9 @@ func TestMain(m *testing.M) { testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + CRDInstallOptions: envtest.CRDInstallOptions{ + CRDs: []*apiextensionsv1.CustomResourceDefinition{testdata.OpenShiftRouteCRD}, + }, WebhookInstallOptions: envtest.WebhookInstallOptions{ Paths: []string{filepath.Join("..", "..", "..", "config", "webhook")}, }, @@ -82,6 +88,11 @@ func TestMain(m *testing.M) { os.Exit(1) } + if err = routev1.AddToScheme(testScheme); err != nil { + fmt.Printf("failed to register scheme: %v", err) + os.Exit(1) + } + if err = v1alpha1.AddToScheme(testScheme); err != nil { fmt.Printf("failed to register scheme: %v", err) os.Exit(1) diff --git a/pkg/collector/testdata/route_crd.go b/pkg/collector/testdata/route_crd.go new file mode 100644 index 0000000000..c32a7f95bd --- /dev/null +++ b/pkg/collector/testdata/route_crd.go @@ -0,0 +1,74 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// OpenShiftRouteCRD as go structure. +var OpenShiftRouteCRD = &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "routes.route.openshift.io", + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "route.openshift.io", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + XPreserveUnknownFields: func(v bool) *bool { return &v }(true), + }, + }, + AdditionalPrinterColumns: []apiextensionsv1.CustomResourceColumnDefinition{ + { + Name: "Host", + Type: "string", + JSONPath: ".status.ingress[0].host", + }, + { + Name: "Admitted", + Type: "string", + JSONPath: `.status.ingress[0].conditions[?(@.type=="Admitted")].status`, + }, + { + Name: "Service", + Type: "string", + JSONPath: ".spec.to.name", + }, + { + Name: "TLS", + Type: "string", + JSONPath: ".spec.tls.type", + }, + }, + Subresources: &apiextensionsv1.CustomResourceSubresources{ + Status: &apiextensionsv1.CustomResourceSubresourceStatus{}, + }, + }, + }, + Scope: apiextensionsv1.NamespaceScoped, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "routes", + Singular: "route", + Kind: "Route", + }, + }, +} diff --git a/pkg/naming/main.go b/pkg/naming/main.go index 59b8029a8e..ff7f7cd9a3 100644 --- a/pkg/naming/main.go +++ b/pkg/naming/main.go @@ -94,6 +94,11 @@ func Ingress(otelcol v1alpha1.OpenTelemetryCollector) string { return DNSName(Truncate("%s-ingress", 63, otelcol.Name)) } +// Route builds the route name based on the instance. +func Route(otelcol v1alpha1.OpenTelemetryCollector, prefix string) string { + return DNSName(Truncate("%s-%s-route", 63, prefix, otelcol.Name)) +} + // TAService returns the name to use for the TargetAllocator service. func TAService(otelcol v1alpha1.OpenTelemetryCollector) string { return DNSName(Truncate("%s-targetallocator", 63, otelcol.Name)) diff --git a/tests/e2e/route/00-assert.yaml b/tests/e2e/route/00-assert.yaml new file mode 100644 index 0000000000..35ee38e2a6 --- /dev/null +++ b/tests/e2e/route/00-assert.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simplest-collector +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + something.com: "true" + labels: + app.kubernetes.io/managed-by: opentelemetry-operator + app.kubernetes.io/name: otlp-grpc-simplest-route + name: otlp-grpc-simplest-route + ownerReferences: + - apiVersion: opentelemetry.io/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: OpenTelemetryCollector + name: simplest +spec: + host: otlp-grpc.example.com + path: /otlp-grpc + port: + targetPort: otlp-grpc + to: + kind: Service + name: simplest-collector + weight: null + wildcardPolicy: None diff --git a/tests/e2e/route/00-install.yaml b/tests/e2e/route/00-install.yaml new file mode 100644 index 0000000000..b2f47baafe --- /dev/null +++ b/tests/e2e/route/00-install.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: simplest +spec: + mode: "deployment" + ingress: + type: route + hostname: "example.com" + annotations: + something.com: "true" + route: + termination: "insecure" + + config: | + receivers: + otlp: + protocols: + grpc: + + exporters: + logging: + + service: + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [logging]
NameTypeDescriptionRequired
terminationenum + Termination indicates termination type. By default "edge" is used.
+
+ Enum: insecure, edge, passthrough, reencrypt
 false