From 6d31174518dd12bdf9d48941e90605a081e1a8c0 Mon Sep 17 00:00:00 2001 From: Alex Leong Date: Tue, 19 Nov 2024 19:27:23 +0000 Subject: [PATCH] Add federated service e2e test Signed-off-by: Alex Leong --- .../multicluster-traffic/federated_test.go | 150 ++++++++++++++++++ .../testdata/federated-client.yml | 33 ++++ 2 files changed, 183 insertions(+) create mode 100644 test/integration/multicluster/multicluster-traffic/federated_test.go create mode 100644 test/integration/multicluster/multicluster-traffic/testdata/federated-client.yml diff --git a/test/integration/multicluster/multicluster-traffic/federated_test.go b/test/integration/multicluster/multicluster-traffic/federated_test.go new file mode 100644 index 0000000000000..ee6d5e396d32b --- /dev/null +++ b/test/integration/multicluster/multicluster-traffic/federated_test.go @@ -0,0 +1,150 @@ +package multiclustertraffic + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/linkerd/linkerd2/pkg/k8s" + "github.com/linkerd/linkerd2/testutil" + kerrors "k8s.io/apimachinery/pkg/api/errors" +) + +// TestFederatedService deploys emojivoto to two clusters and has the web-svc +// in both clusters join a federated service. It creates a vote-bot in the +// source cluster which sends traffic to the federated service and then checks +// the logs of the web-svc in both clusters. If it has successfully issued +// requests, then we'll see log messages indicating that the web-svc can't +// reach the voting-svc (because it's not running). +// +// We verify that the federated service exists and has no endpoints in the +// source cluster. +func TestFederatedService(t *testing.T) { + if err := TestHelper.SwitchContext(contexts[testutil.TargetContextKey]); err != nil { + testutil.AnnotatedFatalf(t, + "failed to rebuild helper clientset with new context", + "failed to rebuild helper clientset with new context [%s]: %v", + contexts[testutil.TargetContextKey], err) + } + + ctx := context.Background() + // Create emojivoto in target cluster, to be deleted at the end of the test. + annotations := map[string]string{ + // "config.linkerd.io/proxy-log-level": "linkerd=debug,info", + } + TestHelper.WithDataPlaneNamespace(ctx, "emojivoto-federated", annotations, t, func(t *testing.T, ns string) { + t.Run("Deploy resources in source and target clusters", func(t *testing.T) { + // Deploy federated-client in source-cluster + o, err := TestHelper.KubectlWithContext("", contexts[testutil.SourceContextKey], "create", "ns", ns) + if err != nil { + testutil.AnnotatedFatalf(t, "failed to create ns", "failed to create ns: %s\n%s", err, o) + } + o, err = TestHelper.KubectlApplyWithContext("", contexts[testutil.SourceContextKey], "--namespace", ns, "-f", "testdata/federated-client.yml") + if err != nil { + testutil.AnnotatedFatalf(t, "failed to install federated-client", "failed to installfederated-client: %s\n%s", err, o) + } + + // Deploy emojivoto in both clusters + for _, ctx := range contexts { + out, err := TestHelper.KubectlApplyWithContext("", ctx, "--namespace", ns, "-f", "testdata/emojivoto-no-bot.yml") + if err != nil { + testutil.AnnotatedFatalf(t, "failed to install emojivoto", "failed to install emojivoto: %s\n%s", err, out) + } + + // Label the service to join the federated service. + timeout := time.Minute + err = testutil.RetryFor(timeout, func() error { + out, err = TestHelper.KubectlWithContext("", ctx, "--namespace", ns, "label", "service/web-svc", "mirror.linkerd.io/federated=member") + return err + }) + if err != nil { + testutil.AnnotatedFatalf(t, "failed to label web-svc", "%s\n%s", err, out) + } + } + }) + + t.Run("Wait until target workloads are ready", func(t *testing.T) { + // Wait until client is up and running in source cluster + voteBotDeployReplica := map[string]testutil.DeploySpec{"vote-bot": {Namespace: ns, Replicas: 1}} + TestHelper.WaitRolloutWithContext(t, voteBotDeployReplica, contexts[testutil.SourceContextKey]) + + // Wait until services and replicas are up and running. + emojiDeployReplicas := map[string]testutil.DeploySpec{ + "web": {Namespace: ns, Replicas: 1}, + "emoji": {Namespace: ns, Replicas: 1}, + "voting": {Namespace: ns, Replicas: 1}, + } + for _, ctx := range contexts { + TestHelper.WaitRolloutWithContext(t, emojiDeployReplicas, ctx) + } + + }) + + timeout := time.Minute + t.Run("Ensure federated service exists and has no endpoints", func(t *testing.T) { + err := TestHelper.SwitchContext(contexts[testutil.SourceContextKey]) + if err != nil { + testutil.AnnotatedFatal(t, "failed to switch contexts", err) + } + err = testutil.RetryFor(timeout, func() error { + svc, err := TestHelper.GetService(ctx, ns, "web-svc-federated") + if err != nil { + return err + } + remoteDiscovery, found := svc.Labels[k8s.RemoteDiscoveryAnnotation] + if !found { + testutil.AnnotatedFatal(t, "federated service missing label", "federated service missing label: "+k8s.RemoteDiscoveryLabel) + } + if remoteDiscovery != "web-svc@target" { + testutil.AnnotatedFatal(t, "federated service has incorrect remote discovery", fmt.Sprintf("federated service remote discovery was %s, expected %s", remoteDiscovery, "web-svc@target")) + } + localDiscovery, found := svc.Labels[k8s.LocalDiscoveryAnnotation] + if !found { + testutil.AnnotatedFatal(t, "federated service missing label", "federated service missing label: "+k8s.LocalDiscoveryAnnotation) + } + if localDiscovery != "web-svc" { + testutil.AnnotatedFatal(t, "federated service has incorrect local discovery", fmt.Sprintf("federated service local discovery was %s, expected %s", localDiscovery, "web-svc")) + } + + _, err = TestHelper.GetEndpoints(ctx, ns, "web-svc-federated") + if err == nil { + testutil.AnnotatedFatal(t, "federated service should not have endpoints", "federated service should not have endpoints") + } + if !kerrors.IsNotFound(err) { + testutil.AnnotatedFatalf(t, "failed to retrieve federated service endpoints", err.Error()) + } + return nil + }) + if err != nil { + testutil.AnnotatedFatal(t, "timed-out verifying federated service", err) + } + }) + + for _, ctx := range contexts { + err := testutil.RetryFor(timeout, func() error { + out, err := TestHelper.KubectlWithContext("", + ctx, + "--namespace", ns, + "logs", + "--selector", "app=web-svc", + "--container", "web-svc", + ) + if err != nil { + return fmt.Errorf("%w\n%s", err, out) + } + // Check for expected error messages + for _, row := range strings.Split(out, "\n") { + if strings.Contains(row, " /api/vote?choice=:doughnut: ") { + return nil + } + } + return fmt.Errorf("web-svc logs in %s cluster do not include voting errors\n%s", ctx, out) + }) + if err != nil { + testutil.AnnotatedFatal(t, fmt.Sprintf("timed-out waiting for traffic in %s cluster (%s)", ctx, timeout), err) + } + } + }) +} diff --git a/test/integration/multicluster/multicluster-traffic/testdata/federated-client.yml b/test/integration/multicluster/multicluster-traffic/testdata/federated-client.yml new file mode 100644 index 0000000000000..a0914196e3d02 --- /dev/null +++ b/test/integration/multicluster/multicluster-traffic/testdata/federated-client.yml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: vote-bot + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v10 + name: vote-bot +spec: + replicas: 1 + selector: + matchLabels: + app: vote-bot + version: v10 + template: + metadata: + annotations: + linkerd.io/inject: enabled + labels: + app: vote-bot + version: v10 + spec: + containers: + - command: + - emojivoto-vote-bot + env: + - name: WEB_HOST + value: web-svc-federated:80 + image: buoyantio/emojivoto-web:v10 + name: vote-bot + resources: + requests: + cpu: 10m