From ed7cb4e2f7f520d3d9cfafe668a76c2330b6454e Mon Sep 17 00:00:00 2001 From: ii2day Date: Tue, 15 Aug 2023 18:19:15 +0800 Subject: [PATCH] add schedule e2e Signed-off-by: ii2day --- docs/reference/apphttphealthy-zh_CN.md | 20 +- docs/reference/runtime-zh_CN.md | 26 ++ pkg/reportManager/worker.go | 15 +- test/docs/Runtime.md | 23 +- test/e2e/apphttphealth/apphttphealth_test.go | 67 ++- test/e2e/common/tools.go | 447 +++++++++++++++++- test/e2e/netdns/netdns_test.go | 17 +- test/e2e/netreach/netreach_suite_test.go | 3 +- test/e2e/netreach/netreach_test.go | 10 +- test/e2e/runtime/runtime_suite_test.go | 27 ++ test/e2e/runtime/runtime_test.go | 464 +++++++++++++++++++ 11 files changed, 1085 insertions(+), 34 deletions(-) create mode 100644 docs/reference/runtime-zh_CN.md create mode 100644 test/e2e/runtime/runtime_suite_test.go create mode 100644 test/e2e/runtime/runtime_test.go diff --git a/docs/reference/apphttphealthy-zh_CN.md b/docs/reference/apphttphealthy-zh_CN.md index e9341ae6..ca7a581c 100644 --- a/docs/reference/apphttphealthy-zh_CN.md +++ b/docs/reference/apphttphealthy-zh_CN.md @@ -75,16 +75,16 @@ status: #### AgentSpec -| 字段 | 描述 | 结构 | 验证 | 取值 | 默认值 | -|-------------------------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------|-----|----------------------|-----------| -| annotation | agent 工作负载的 annotation | map[string]string | 可选 | | | -| kind | agent 工作负载的类型 | string | 可选 | Deployment、DaemonSet | DaemonSet | -| deploymentReplicas | agent 工作负载类型为 deployment 时的期望副本数 | int | 可选 | 大于等于 0 | 0 | -| affinity | agent 工作负载亲和性 | [labelSelector](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L1195) | 可选 | | | -| env | agent 工作负载环境变量 | [env](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/api/core/v1/types.go#L2012) | 可选 | | | -| hostNetwork | agent 工作负载是否使用宿主机网络 | bool | 可选 | true、false | false | -| resources | agent 工作负载资源使用配置 | [resources](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/api/core/v1/types.go#L2333) | 可选 | | | -| terminationGracePeriodMinutes | agent 工作负载完成任务后多少分钟之后终止 | int | 可选 | 大于等于 0 | 60 | +| 字段 | 描述 | 结构 | 验证 | 取值 | 默认值 | +|-------------------------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------|-----|----------------------|--------------------| +| annotation | agent 工作负载的 annotation | map[string]string | 可选 | | | +| kind | agent 工作负载的类型 | string | 可选 | Deployment、DaemonSet | DaemonSet | +| deploymentReplicas | agent 工作负载类型为 deployment 时的期望副本数 | int | 可选 | 大于等于 0 | 0 | +| affinity | agent 工作负载亲和性 | [labelSelector](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L1195) | 可选 | | | +| env | agent 工作负载环境变量 | [env](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/api/core/v1/types.go#L2012) | 可选 | | | +| hostNetwork | agent 工作负载是否使用宿主机网络 | bool | 可选 | true、false | false | +| resources | agent 工作负载资源使用配置 | [resources](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/api/core/v1/types.go#L2333) | 可选 | | cpu:100m,memory:128Mi | +| terminationGracePeriodMinutes | agent 工作负载完成任务后多少分钟之后终止 | int | 可选 | 大于等于 0 | 60 | #### Schedule diff --git a/docs/reference/runtime-zh_CN.md b/docs/reference/runtime-zh_CN.md new file mode 100644 index 00000000..f41f4d3b --- /dev/null +++ b/docs/reference/runtime-zh_CN.md @@ -0,0 +1,26 @@ +## runtime + +当下发任务 CR 后，kdoctor-controller 会根据 CR 中的 AgentSpec 生成对应的任务载体（DaemonSet 或 Deployment）执行任务，每一个任务独立使用一个载体。 + +### 载体资源 + +当任务 CR 下发后，kdocotr-controller 会创建如下资源进行任务。 + +### 工作负载 + +工作负载为 DaemonSet 或 Deployment，默认为 Daemonset，负载中的每一个 Pod 根据任务配置进行的请求，并将执行结果落盘到 Pod 中，可通过 AgentSpec 中设置 +工作负载的销毁时间，默认任务执行完 60 分钟后，销毁工作负载，当删除 CR 任务时，工作负载会一并被删除。 + +### Service + +在创建工作负载时，kdoctor-controller 同时会根据 IP Family 的配置，创建对应的 service 并于工作负载的 pod 绑定。用于测试 service 网络连通性。与工作负载 +的销毁逻辑相同。 + +### Ingress + +当任务为 NetReach 时，若测试目标包含 Ingress 时，会创建一个 Ingress，用于测试 Ingress 的网络联通性，与工作负载的销毁逻辑相同。 + +### 报告收取 + +当任务 CR 下发后，kdoctor-controller 会将任务注册进 ReportManager，ReportManager 会定期去每一个任务负载中通过 GRPC 接口获取报告，并聚合 +在 kdoctor-controller 中，聚合后可通过命令 `kubectl get kdoctorreport` 获取报告结果，因此，若报告未收集完成就将工作负载删除将影响报告聚合结果。 diff --git a/pkg/reportManager/worker.go b/pkg/reportManager/worker.go index 7be54d34..f49c918a 100644 --- a/pkg/reportManager/worker.go +++ b/pkg/reportManager/worker.go @@ -87,7 +87,7 @@ func (s *reportManager) syncReportFromOneAgent(ctx context.Context, logger *zap. } -func (s *reportManager) runControllerAggregateReportOnce(ctx context.Context, logger *zap.Logger) error { +func (s *reportManager) runControllerAggregateReportOnce(ctx context.Context, logger *zap.Logger, taskName string) error { // grpc client grpcClient := grpcManager.NewGrpcClient(s.logger.Named("grpc"), true) @@ -108,6 +108,10 @@ func (s *reportManager) runControllerAggregateReportOnce(ctx context.Context, lo logger.Sugar().Debugf("task %s runtime %s status %s not created finish", m.TaskName, m.RuntimeName, m.RuntimeStatus) continue } + if m.TaskName != taskName { + logger.Sugar().Debugf("this agent %s is not ccurrent sync task %s ,skip ", m.RuntimeName, taskName) + continue + } var podIP k8sObjManager.PodIps var err error if m.RuntimeKind == types.KindDaemonSet { @@ -161,5 +165,12 @@ func (s *reportManager) syncHandler(ctx context.Context, triggerName string) err logger := s.logger.With( zap.String("triggerSource", triggerName), ) - return s.runControllerAggregateReportOnce(ctx, logger) + var taskName string + if triggerName == "periodicallyTrigger" { + taskName = triggerName + } else { + taskName = strings.Split(triggerName, ".")[1] + } + + return s.runControllerAggregateReportOnce(ctx, logger, taskName) } diff --git a/test/docs/Runtime.md b/test/docs/Runtime.md index 7507ce16..fc968264 100644 --- a/test/docs/Runtime.md +++ b/test/docs/Runtime.md @@ -1,11 +1,16 @@ # E2E Cases for Task Runtime -| Case ID | Title | Priority | Smoke | Status | Other | -|---------|----------------------------------------------------------------------------------------|----------|-------|--------|-------------| -| E00001 | Successfully testing Task Runtime DaemonSet creation | p1 | | | | -| E00002 | Successfully testing Task Runtime Deployment creation | p1 | | | | -| E00003 | Successfully testing Task Runtime Service creation | p1 | | | | -| E00004 | Successfully testing cascading deletion with Task Runtime Service | p1 | | | | -| E00005 | Successfully testing cascading deletion with Task Runtime | p1 | | | | -| E00006 | Successfully testing Task NetReach Runtime Service and Ingress creation | p1 | | | | -| E00007 | Successfully testing cascading deletion with Task NetReach Runtime Service and Ingress | p1 | | | | +| Case ID | Title | Priority | Smoke | Status | Other | +|---------|------------------------------------------------------------------------------------------------|----------|-------|--------|-------------| +| E00001 | Successfully testing Task NetReach Runtime DaemonSet Service Ingress creation | p1 | | done | | +| E00002 | Successfully testing Task NetAppHttpHealthy Runtime DaemonSet Service creation | p1 | | done | | +| E00003 | Successfully testing Task NetDns Runtime DaemonSet Service creation | p1 | | done | | +| E00004 | Successfully testing Task NetReach Runtime Deployment Service Ingress creation | p1 | | | | +| E00005 | Successfully testing Task NetAppHttpHealthy Runtime Deployment Service creation | p1 | | | | +| E00006 | Successfully testing Task NetDns Runtime Deployment Service creation | p1 | | | | +| E00007 | Successfully testing cascading deletion with Task NetReach DaemonSet Service and Ingress | p1 | | done | | +| E00008 | Successfully testing cascading deletion with Task NetAppHttpHealthy DaemonSet Service | p1 | | done | | +| E00009 | Successfully testing cascading deletion with Task NetDns DaemonSet Service | p1 | | done | | +| E00010 | Successfully testing cascading deletion with Task NetReach Deployment Service and Ingress | p1 | | | | +| E00011 | Successfully testing cascading deletion with Task NetAppHttpHealthy Deployment Service | p1 | | | | +| E00012 | Successfully testing cascading deletion with Task NetDns Deployment Service | p1 | | | | \ No newline at end of file diff --git a/test/e2e/apphttphealth/apphttphealth_test.go b/test/e2e/apphttphealth/apphttphealth_test.go index 5effa687..2d0aa0a1 100644 --- a/test/e2e/apphttphealth/apphttphealth_test.go +++ b/test/e2e/apphttphealth/apphttphealth_test.go @@ -15,9 +15,9 @@ import ( ) var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { - var termMin = int64(3) + var termMin = int64(1) - It("success http testing appHttpHealth method GET", Label("A00001", "A00011", "C00006"), func() { + It("success http testing appHttpHealth method GET", Label("A00001", "A00011", "C00006", "E00002"), func() { var e error successRate := float64(1) successMean := int64(1500) @@ -65,12 +65,18 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") success, e := common.CompareResult(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, testPodIPs, reportNum) Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("failed http testing appHttpHealth due to status code", Label("A00002"), func() { @@ -123,6 +129,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -130,6 +139,8 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeFalse(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("Failed http testing appHttpHealth due to delay ", Label("A00003"), func() { @@ -180,6 +191,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -187,6 +201,8 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeFalse(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("success https testing appHttpHealth method GET", Label("A00004"), func() { @@ -238,6 +254,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -245,6 +264,8 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("failed https testing appHttpHealth due to tls", Label("A00005"), func() { @@ -298,6 +319,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -305,6 +329,8 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeFalse(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("Successfully http testing appHttpHealth method PUT ", Label("A00006"), func() { @@ -355,6 +381,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -362,6 +391,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") + }) It("Successfully http testing appHttpHealth method POST With Body", Label("A00007"), func() { @@ -415,12 +447,18 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") success, e := common.CompareResult(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, testPodIPs, reportNum) Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("Successfully http testing appHttpHealth method HEAD", Label("A00008"), func() { @@ -471,6 +509,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -478,6 +519,8 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("Successfully http testing appHttpHealth method PATCH", Label("A00009"), func() { @@ -528,6 +571,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -535,6 +581,8 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("Successfully http testing appHttpHealth method OPTIONS", Label("A00010"), func() { @@ -585,6 +633,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -592,6 +643,8 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("Successfully http testing appHttpHealth due to success rate", Label("A00012"), func() { @@ -642,6 +695,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -649,6 +705,8 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) It("Successfully https testing appHttpHealth method GET Protocol Http2", Label("A00013"), func() { @@ -702,6 +760,9 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { e = frame.CreateResource(appHttpHealth) Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 120) Expect(e).NotTo(HaveOccurred(), "wait appHttpHealth task finish") @@ -709,5 +770,7 @@ var _ = Describe("testing appHttpHealth test ", Label("appHttpHealth"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, appHttpHealthName, pluginManager.KindNameAppHttpHealthy, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) }) diff --git a/test/e2e/common/tools.go b/test/e2e/common/tools.go index c4156381..f227004e 100644 --- a/test/e2e/common/tools.go +++ b/test/e2e/common/tools.go @@ -9,25 +9,30 @@ import ( "crypto/tls" "encoding/json" "fmt" - "github.com/kdoctor-io/kdoctor/api/v1/agentServer/models" - "github.com/onsi/ginkgo/v2" "io" + "k8s.io/apimachinery/pkg/api/errors" "math" "net" "net/http" "os/exec" + "reflect" "strings" "time" "github.com/docker/docker/api/types" docker_client "github.com/docker/docker/client" - "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/kdoctor.io/v1beta1" - kdoctor_report "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/system/v1beta1" - "github.com/kdoctor-io/kdoctor/pkg/pluginManager" + "github.com/onsi/ginkgo/v2" frame "github.com/spidernet-io/e2eframework/framework" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kdoctor-io/kdoctor/api/v1/agentServer/models" + "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/kdoctor.io/v1beta1" + kdoctor_report "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/system/v1beta1" + "github.com/kdoctor-io/kdoctor/pkg/pluginManager" + kdoctor_types "github.com/kdoctor-io/kdoctor/pkg/types" ) func WaitKdoctorTaskDone(f *frame.Framework, task client.Object, taskKind string, timeout int) error { @@ -484,3 +489,435 @@ func CreateTestApp(name, namespace string, o []string) error { return nil } + +func CheckRuntime(f *frame.Framework, task client.Object, taskKind string, timeout int) error { + interval := time.Duration(10) + switch taskKind { + case pluginManager.KindNameNetReach: + fake := &v1beta1.NetReach{ + ObjectMeta: metav1.ObjectMeta{ + Name: task.GetName(), + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.NetReach{} + after := time.After(time.Duration(timeout) * time.Second) + create := false + for !create { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime create", pluginManager.KindNameNetReach, task.GetName()) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameNetReach, task.GetName(), err) + } + if rs.Status.Resource == nil { + continue + } + if rs.Status.Resource.RuntimeStatus == v1beta1.RuntimeCreated { + create = true + break + } + time.Sleep(time.Second * interval) + } + } + if err := checkAgentSpec(f, rs, rs.Spec.AgentSpec, rs.Status, pluginManager.KindNameNetReach); err != nil { + return fmt.Errorf("check AgentSpce err,reason= %v ", err) + } + case pluginManager.KindNameAppHttpHealthy: + fake := &v1beta1.AppHttpHealthy{ + ObjectMeta: metav1.ObjectMeta{ + Name: task.GetName(), + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.AppHttpHealthy{} + after := time.After(time.Duration(timeout) * time.Second) + create := false + for !create { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime create", pluginManager.KindNameAppHttpHealthy, task.GetName()) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameAppHttpHealthy, task.GetName(), err) + } + if rs.Status.Resource == nil { + continue + } + if rs.Status.Resource.RuntimeStatus == v1beta1.RuntimeCreated { + create = true + break + } + time.Sleep(time.Second * interval) + } + } + if err := checkAgentSpec(f, rs, rs.Spec.AgentSpec, rs.Status, pluginManager.KindNameAppHttpHealthy); err != nil { + return fmt.Errorf("check AgentSpce err,reason= %v ", err) + } + case pluginManager.KindNameNetdns: + fake := &v1beta1.Netdns{ + ObjectMeta: metav1.ObjectMeta{ + Name: task.GetName(), + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.Netdns{} + after := time.After(time.Duration(timeout) * time.Second) + create := false + for !create { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime create", pluginManager.KindNameNetdns, task.GetName()) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameNetdns, task.GetName(), err) + } + if rs.Status.Resource == nil { + continue + } + if rs.Status.Resource.RuntimeStatus == v1beta1.RuntimeCreated { + create = true + break + } + time.Sleep(time.Second * interval) + } + } + if err := checkAgentSpec(f, rs, rs.Spec.AgentSpec, rs.Status, pluginManager.KindNameNetdns); err != nil { + return fmt.Errorf("check AgentSpce err,reason= %v ", err) + } + default: + return fmt.Errorf("unknown task type: %s", task.GetObjectKind().GroupVersionKind().Kind) + } + return nil +} + +// checkAgentSpec check agentSpec generate deployment or daemonSet is right +func checkAgentSpec(f *frame.Framework, task client.Object, agentSpec v1beta1.AgentSpec, taskStatus v1beta1.TaskStatus, taskKind string) error { + + switch agentSpec.Kind { + case kdoctor_types.KindDaemonSet: + if taskStatus.Resource.RuntimeType != kdoctor_types.KindDaemonSet { + return fmt.Errorf("agent spec is %s,but status reource runtimeType is %s ", kdoctor_types.KindDaemonSet, taskStatus.Resource.RuntimeType) + } + runtime, err := f.GetDaemonSet(taskStatus.Resource.RuntimeName, TestNameSpace) + if err != nil { + return fmt.Errorf("failed get runtime daemonset %s ,reason=%v ", taskStatus.Resource.RuntimeName, err) + } + // compare annotation + if !reflect.DeepEqual(runtime.Spec.Template.Annotations, agentSpec.Annotation) { + return fmt.Errorf("create runtime daemonset annotation not equal,spec: %v real: %v", agentSpec.Annotation, runtime.Spec.Template.Annotations) + } + + // TODO(ii2day): compare env resource affinity + + // compare HostNetwork + if !reflect.DeepEqual(runtime.Spec.Template.Spec.HostNetwork, agentSpec.HostNetwork) { + return fmt.Errorf("create runtime daemonset HostNetwork not equal,spec: %v real: %v", agentSpec.HostNetwork, runtime.Spec.Template.Spec.HostNetwork) + } + + case kdoctor_types.KindDeployment: + runtime, err := f.GetDeployment(taskStatus.Resource.RuntimeName, TestNameSpace) + if err != nil { + return fmt.Errorf("failed get runtime deployment %s ,reason=%v ", taskStatus.Resource.RuntimeName, err) + } + // compare annotation + if !reflect.DeepEqual(runtime.Spec.Template.Annotations, agentSpec.Annotation) { + return fmt.Errorf("create runtime deployment annotation not equal,spec: %v real: %v", agentSpec.Annotation, runtime.Spec.Template.Annotations) + } + // TODO(ii2day): compare env resource affinity + + // compare HostNetwork + if !reflect.DeepEqual(runtime.Spec.Template.Spec.HostNetwork, agentSpec.HostNetwork) { + return fmt.Errorf("create runtime deployment HostNetwork not equal,spec: %v real: %v", agentSpec.HostNetwork, runtime.Spec.Template.Spec.HostNetwork) + } + + // compare replicas + if *agentSpec.DeploymentReplicas != runtime.Status.Replicas { + return fmt.Errorf("create runtime deployment Replicas not equal,spec: %d real: %d", *agentSpec.DeploymentReplicas, runtime.Status.Replicas) + } + default: + return fmt.Errorf("unknown agent kind %s ", agentSpec.Kind) + } + + if TestIPv4 { + _, err := f.GetService(*taskStatus.Resource.ServiceNameV4, TestNameSpace) + if err != nil { + return fmt.Errorf("failed get service %s ,reason=%v ", *taskStatus.Resource.ServiceNameV4, err) + } + } + + if TestIPv6 { + _, err := f.GetService(*taskStatus.Resource.ServiceNameV6, TestNameSpace) + if err != nil { + return fmt.Errorf("failed get service %s ,reason=%v ", *taskStatus.Resource.ServiceNameV6, err) + } + } + if taskKind == kdoctor_types.KindNameNetReach { + taskNr := task.(*v1beta1.NetReach) + if taskNr.Spec.Target.Ingress { + ig := &networkingv1.Ingress{} + fake := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: *taskStatus.Resource.ServiceNameV4, + Namespace: TestNameSpace, + }, + } + key := client.ObjectKeyFromObject(fake) + if err := f.GetResource(key, ig); err != nil { + return fmt.Errorf("task %s enable ingress test,but not found %s ingress err=%v", task.GetName(), *taskStatus.Resource.ServiceNameV4, err) + } + } + } + return nil +} + +// CheckRuntimeDeadLine check terminationGracePeriodMinutes delete deployment or daemonSet service ingress. +func CheckRuntimeDeadLine(f *frame.Framework, taskName, taskKind string, timeout int) error { + interval := time.Duration(10) + var runtimeResource *v1beta1.TaskResource + var terminationGracePeriodMinutes int64 + var testIngress = false + switch taskKind { + case kdoctor_types.KindNameNetReach: + fake := &v1beta1.NetReach{ + ObjectMeta: metav1.ObjectMeta{ + Name: taskName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.NetReach{} + after := time.After(time.Duration(timeout) * time.Second) + done := false + for !done { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime deadline", pluginManager.KindNameNetReach, taskName) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameNetReach, taskName, err) + } + if rs.Status.FinishTime != nil { + done = true + runtimeResource = rs.Status.Resource + terminationGracePeriodMinutes = *rs.Spec.AgentSpec.TerminationGracePeriodMinutes + testIngress = rs.Spec.Target.Ingress + break + } + time.Sleep(time.Second * interval) + } + } + case kdoctor_types.KindNameNetdns: + fake := &v1beta1.Netdns{ + ObjectMeta: metav1.ObjectMeta{ + Name: taskName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.Netdns{} + after := time.After(time.Duration(timeout) * time.Minute) + done := false + for !done { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime deadline", pluginManager.KindNameNetdns, taskName) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameNetdns, taskName, err) + } + if rs.Status.FinishTime != nil { + done = true + runtimeResource = rs.Status.Resource + terminationGracePeriodMinutes = *rs.Spec.AgentSpec.TerminationGracePeriodMinutes + break + } + time.Sleep(time.Second * interval) + } + } + case kdoctor_types.KindNameAppHttpHealthy: + fake := &v1beta1.AppHttpHealthy{ + ObjectMeta: metav1.ObjectMeta{ + Name: taskName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.AppHttpHealthy{} + after := time.After(time.Duration(timeout) * time.Minute) + done := false + for !done { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime deadline", pluginManager.KindNameAppHttpHealthy, taskName) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameAppHttpHealthy, taskName, err) + } + if rs.Status.FinishTime != nil { + done = true + runtimeResource = rs.Status.Resource + terminationGracePeriodMinutes = *rs.Spec.AgentSpec.TerminationGracePeriodMinutes + break + } + time.Sleep(time.Second * interval) + } + } + default: + return fmt.Errorf("unknown task %s type: %s", taskName, taskKind) + } + + c := time.After(time.Duration(terminationGracePeriodMinutes) * time.Minute) + // wait delete time + <-c + + c2 := time.After(time.Minute) + + // check runtime delete + runtimeDeleted := false + for !runtimeDeleted { + select { + case <-c2: + return fmt.Errorf("timeout delete %s task %s runtime", taskKind, taskName) + default: + if runtimeResource.RuntimeType == kdoctor_types.KindDaemonSet { + _, err := f.GetDaemonSet(runtimeResource.RuntimeName, TestNameSpace) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime daemonSet %s deleted \n", runtimeResource.RuntimeName) + runtimeDeleted = true + break + } + return fmt.Errorf("failed get task %s deamonSet reason=%v", taskName, err) + } + + } else { + _, err := f.GetDeployment(runtimeResource.RuntimeName, TestNameSpace) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime deployment %s deleted \n", runtimeResource.RuntimeName) + runtimeDeleted = true + break + } + return fmt.Errorf("failed get task %s deamonSet reason=%v", taskName, err) + } + } + time.Sleep(time.Second * interval) + } + } + + // check runtime service ingress delete + c3 := time.After(time.Minute) + serviceV4Deleted := false + serviceV6Deleted := false + ingressDeleted := false + for !serviceV4Deleted || !serviceV6Deleted { + select { + case <-c3: + return fmt.Errorf("timeout delete %s task %s runtime service", taskKind, taskName) + default: + if TestIPv4 { + _, err := f.GetService(*runtimeResource.ServiceNameV4, TestNameSpace) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime service v4 %s deleted \n", *runtimeResource.ServiceNameV4) + serviceV4Deleted = true + } else { + return fmt.Errorf("failed get task %s service %s reason=%v", taskName, *runtimeResource.ServiceNameV4, err) + } + } + if testIngress && !ingressDeleted { + ig := &networkingv1.Ingress{} + fake := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: *runtimeResource.ServiceNameV4, + Namespace: TestNameSpace, + }, + } + key := client.ObjectKeyFromObject(fake) + err = f.GetResource(key, ig) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime ingress %s deleted \n", *runtimeResource.ServiceNameV4) + ingressDeleted = true + } else { + return fmt.Errorf("task %s enable ingress test,but not found %s ingress err=%v", taskName, *runtimeResource.ServiceNameV4, err) + } + } + } else { + ingressDeleted = true + } + } else { + serviceV4Deleted = true + } + if TestIPv6 { + _, err := f.GetService(*runtimeResource.ServiceNameV6, TestNameSpace) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime service v6 %s deleted \n", *runtimeResource.ServiceNameV6) + serviceV6Deleted = true + break + } + return fmt.Errorf("failed get task %s service %s reason=%v", taskName, *runtimeResource.ServiceNameV6, err) + } + } else { + serviceV6Deleted = true + } + time.Sleep(time.Second * interval) + } + } + + return nil + +} + +func GetRuntimeResource(f *frame.Framework, resource *v1beta1.TaskResource, ingress bool) error { + if resource == nil { + return fmt.Errorf("runtime resource is nil") + } + + c := time.After(time.Minute) + <-c + + switch resource.RuntimeType { + case kdoctor_types.KindDaemonSet: + _, err := f.GetDaemonSet(resource.RuntimeName, TestNameSpace) + if !errors.IsNotFound(err) { + return fmt.Errorf("after 1 min runtime daemonset %s not delete", resource.RuntimeName) + } + case kdoctor_types.KindDeployment: + _, err := f.GetDeployment(resource.RuntimeName, TestNameSpace) + if !errors.IsNotFound(err) { + return fmt.Errorf("after 1 min runtime deployment %s not delete", resource.RuntimeName) + } + } + + if resource.ServiceNameV4 != nil { + _, err := f.GetService(*resource.ServiceNameV4, TestNameSpace) + if !errors.IsNotFound(err) { + return fmt.Errorf("after 1 min runtime service %s not delete", *resource.ServiceNameV4) + } + + if ingress { + ig := &networkingv1.Ingress{} + fake := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: *resource.ServiceNameV4, + Namespace: TestNameSpace, + }, + } + key := client.ObjectKeyFromObject(fake) + err := f.GetResource(key, ig) + if !errors.IsNotFound(err) { + return fmt.Errorf("after 1 min runtime ingress %s not delete", *resource.ServiceNameV4) + } + } + } + + if resource.ServiceNameV6 != nil { + _, err := f.GetService(*resource.ServiceNameV6, TestNameSpace) + if !errors.IsNotFound(err) { + return fmt.Errorf("after 1 min runtime service %s not delete", *resource.ServiceNameV6) + } + } + + return nil +} diff --git a/test/e2e/netdns/netdns_test.go b/test/e2e/netdns/netdns_test.go index 403be8ec..6556a902 100644 --- a/test/e2e/netdns/netdns_test.go +++ b/test/e2e/netdns/netdns_test.go @@ -16,8 +16,8 @@ import ( var _ = Describe("testing netDns ", Label("netDns"), func() { var targetDomain = "%s.kubernetes.default.svc.cluster.local" - var termMin = int64(3) - It("Successfully testing Cluster Dns Server case", Label("D00001", "C00005"), func() { + var termMin = int64(1) + It("Successfully testing Cluster Dns Server case", Label("D00001", "C00005", "E00003"), func() { var e error successRate := float64(1) successMean := int64(1500) @@ -71,12 +71,18 @@ var _ = Describe("testing netDns ", Label("netDns"), func() { e = frame.CreateResource(netDns) Expect(e).NotTo(HaveOccurred(), "create netDns resource") + e = common.CheckRuntime(frame, netDns, pluginManager.KindNameNetdns, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, netDns, pluginManager.KindNameNetdns, 120) Expect(e).NotTo(HaveOccurred(), "wait netDns task finish") success, e := common.CompareResult(frame, netDnsName, pluginManager.KindNameNetdns, []string{}, reportNum) - Expect(success).NotTo(BeFalse(), "compare report and task result") Expect(e).NotTo(HaveOccurred(), "compare report and task") + Expect(success).NotTo(BeFalse(), "compare report and task result") + + e = common.CheckRuntimeDeadLine(frame, netDnsName, pluginManager.KindNameNetdns, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) @@ -133,6 +139,9 @@ var _ = Describe("testing netDns ", Label("netDns"), func() { e = frame.CreateResource(netDns) Expect(e).NotTo(HaveOccurred(), "create netDns resource") + e = common.CheckRuntime(frame, netDns, pluginManager.KindNameNetdns, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, netDns, pluginManager.KindNameNetdns, 120) Expect(e).NotTo(HaveOccurred(), "wait netDns task finish") @@ -140,5 +149,7 @@ var _ = Describe("testing netDns ", Label("netDns"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, netDnsName, pluginManager.KindNameNetdns, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) }) diff --git a/test/e2e/netreach/netreach_suite_test.go b/test/e2e/netreach/netreach_suite_test.go index 41e48d7c..3f2da60e 100644 --- a/test/e2e/netreach/netreach_suite_test.go +++ b/test/e2e/netreach/netreach_suite_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" e2e "github.com/spidernet-io/e2eframework/framework" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/runtime" "testing" ) @@ -22,7 +23,7 @@ var reportNum int var _ = BeforeSuite(func() { defer GinkgoRecover() var e error - frame, e = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{kdoctor_v1beta1.AddToScheme}) + frame, e = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{kdoctor_v1beta1.AddToScheme, networkingv1.AddToScheme}) Expect(e).NotTo(HaveOccurred()) nodeLIst, e := frame.GetNodeList() Expect(e).NotTo(HaveOccurred(), "get node list") diff --git a/test/e2e/netreach/netreach_test.go b/test/e2e/netreach/netreach_test.go index 089a6f0b..6b061816 100644 --- a/test/e2e/netreach/netreach_test.go +++ b/test/e2e/netreach/netreach_test.go @@ -13,8 +13,8 @@ import ( ) var _ = Describe("testing netReach ", Label("netReach"), func() { - var termMin = int64(3) - It("success testing netReach", Label("B00001", "C00004"), func() { + var termMin = int64(1) + It("success testing netReach", Label("B00001", "C00004", "E00001"), func() { var e error successRate := float64(1) successMean := int64(1500) @@ -68,6 +68,9 @@ var _ = Describe("testing netReach ", Label("netReach"), func() { e = frame.CreateResource(netReach) Expect(e).NotTo(HaveOccurred(), "create netReach resource") + e = common.CheckRuntime(frame, netReach, pluginManager.KindNameNetReach, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, netReach, pluginManager.KindNameNetReach, 120) Expect(e).NotTo(HaveOccurred(), "wait netReach task finish") @@ -75,5 +78,8 @@ var _ = Describe("testing netReach ", Label("netReach"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, netReachName, pluginManager.KindNameNetReach, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) + }) diff --git a/test/e2e/runtime/runtime_suite_test.go b/test/e2e/runtime/runtime_suite_test.go new file mode 100644 index 00000000..f86b8f6b --- /dev/null +++ b/test/e2e/runtime/runtime_suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2023 Authors of kdoctor-io +// SPDX-License-Identifier: Apache-2.0 + +package runtime_test + +import ( + kdoctor_v1beta1 "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/kdoctor.io/v1beta1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + e2e "github.com/spidernet-io/e2eframework/framework" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/runtime" + "testing" +) + +func TestRuntime(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Runtime Suite") +} + +var frame *e2e.Framework +var _ = BeforeSuite(func() { + defer GinkgoRecover() + var e error + frame, e = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{kdoctor_v1beta1.AddToScheme, networkingv1.AddToScheme}) + Expect(e).NotTo(HaveOccurred()) +}) diff --git a/test/e2e/runtime/runtime_test.go b/test/e2e/runtime/runtime_test.go new file mode 100644 index 00000000..cc3d054b --- /dev/null +++ b/test/e2e/runtime/runtime_test.go @@ -0,0 +1,464 @@ +// Copyright 2023 Authors of kdoctor-io +// SPDX-License-Identifier: Apache-2.0 + +package runtime_test + +import ( + "github.com/kdoctor-io/kdoctor/pkg/types" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spidernet-io/e2eframework/tools" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/kdoctor.io/v1beta1" + "github.com/kdoctor-io/kdoctor/pkg/pluginManager" + "github.com/kdoctor-io/kdoctor/test/e2e/common" +) + +var _ = Describe("testing runtime ", Label("runtime"), func() { + var termMin = int64(5) + var replicas = int32(2) + It("Successfully testing cascading deletion with Task NetReach DaemonSet Service and Ingress ", Label("E00007"), func() { + var e error + successRate := float64(1) + successMean := int64(1500) + crontab := "0 1" + netReachName := "netreach-" + tools.RandomName() + + netReach := new(v1beta1.NetReach) + netReach.Name = netReachName + + // agentSpec + agentSpec := new(v1beta1.AgentSpec) + agentSpec.TerminationGracePeriodMinutes = &termMin + netReach.Spec.AgentSpec = *agentSpec + + // successCondition + successCondition := new(v1beta1.NetSuccessCondition) + successCondition.SuccessRate = &successRate + successCondition.MeanAccessDelayInMs = &successMean + netReach.Spec.SuccessCondition = successCondition + + // target + target := new(v1beta1.NetReachTarget) + if !common.TestIPv4 && common.TestIPv6 { + target.Ingress = false + } else { + target.Ingress = true + } + target.LoadBalancer = true + target.ClusterIP = true + target.Endpoint = true + target.NodePort = true + target.MultusInterface = false + target.IPv4 = &common.TestIPv4 + target.IPv6 = &common.TestIPv6 + netReach.Spec.Target = target + + // request + request := new(v1beta1.NetHttpRequest) + request.PerRequestTimeoutInMS = 1000 + request.QPS = 10 + request.DurationInSecond = 10 + netReach.Spec.Request = request + + // Schedule + Schedule := new(v1beta1.SchedulePlan) + Schedule.Schedule = &crontab + Schedule.RoundNumber = 1 + Schedule.RoundTimeoutMinute = 1 + netReach.Spec.Schedule = Schedule + + e = frame.CreateResource(netReach) + Expect(e).NotTo(HaveOccurred(), "create netReach resource") + + e = common.CheckRuntime(frame, netReach, pluginManager.KindNameNetReach, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + + // get runtime resource + fake := &v1beta1.NetReach{ + ObjectMeta: metav1.ObjectMeta{ + Name: netReachName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.NetReach{} + e = frame.GetResource(key, rs) + Expect(e).NotTo(HaveOccurred(), "get task runtime status") + + e = frame.DeleteResource(rs) + Expect(e).NotTo(HaveOccurred(), "delete task") + // check runtime resource is deleted after deleting task + e = common.GetRuntimeResource(frame, rs.Status.Resource, true) + Expect(e).NotTo(HaveOccurred(), "get task runtime resource") + }) + + It("Successfully testing cascading deletion with Task NetAppHttpHealthy DaemonSet Service", Label("E00008"), func() { + var e error + successRate := float64(1) + successMean := int64(1500) + crontab := "0 1" + appHttpHealthName := "apphttphealth-get" + tools.RandomName() + + appHttpHealth := new(v1beta1.AppHttpHealthy) + appHttpHealth.Name = appHttpHealthName + + // agent + agentSpec := new(v1beta1.AgentSpec) + agentSpec.TerminationGracePeriodMinutes = &termMin + appHttpHealth.Spec.AgentSpec = *agentSpec + + // successCondition + successCondition := new(v1beta1.NetSuccessCondition) + successCondition.SuccessRate = &successRate + successCondition.MeanAccessDelayInMs = &successMean + appHttpHealth.Spec.SuccessCondition = successCondition + + // target + target := new(v1beta1.AppHttpHealthyTarget) + target.Method = "GET" + target.Host = "www.baidu.com" + appHttpHealth.Spec.Target = target + + // request + request := new(v1beta1.NetHttpRequest) + request.PerRequestTimeoutInMS = 2000 + request.QPS = 10 + request.DurationInSecond = 10 + appHttpHealth.Spec.Request = request + + // Schedule + Schedule := new(v1beta1.SchedulePlan) + Schedule.Schedule = &crontab + Schedule.RoundNumber = 1 + Schedule.RoundTimeoutMinute = 1 + appHttpHealth.Spec.Schedule = Schedule + + e = frame.CreateResource(appHttpHealth) + Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + + // get runtime resource + fake := &v1beta1.AppHttpHealthy{ + ObjectMeta: metav1.ObjectMeta{ + Name: appHttpHealthName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.AppHttpHealthy{} + e = frame.GetResource(key, rs) + Expect(e).NotTo(HaveOccurred(), "get task runtime status") + + e = frame.DeleteResource(rs) + Expect(e).NotTo(HaveOccurred(), "delete task") + // check runtime resource is deleted after deleting task + e = common.GetRuntimeResource(frame, rs.Status.Resource, false) + Expect(e).NotTo(HaveOccurred(), "get task runtime resource") + }) + + It("Successfully testing cascading deletion with Task NetDns DaemonSet Service ", Label("E00009"), func() { + var e error + successRate := float64(1) + successMean := int64(1500) + crontab := "0 1" + netDnsName := "netdns-e2e-" + tools.RandomName() + + netDns := new(v1beta1.Netdns) + netDns.Name = netDnsName + + // agentSpec + agentSpec := new(v1beta1.AgentSpec) + agentSpec.TerminationGracePeriodMinutes = &termMin + netDns.Spec.AgentSpec = *agentSpec + + // successCondition + successCondition := new(v1beta1.NetSuccessCondition) + successCondition.SuccessRate = &successRate + successCondition.MeanAccessDelayInMs = &successMean + netDns.Spec.SuccessCondition = successCondition + + // target + target := new(v1beta1.NetDnsTarget) + targetDns := new(v1beta1.NetDnsTargetDnsSpec) + targetDns.TestIPv4 = &common.TestIPv4 + targetDns.TestIPv6 = &common.TestIPv6 + ServiceName := "kube-dns" + ServiceNamespace := "kube-system" + targetDns.ServiceName = &ServiceName + targetDns.ServiceNamespace = &ServiceNamespace + target.NetDnsTargetDns = targetDns + netDns.Spec.Target = target + + // request + request := new(v1beta1.NetdnsRequest) + var perRequestTimeoutInMS = uint64(1000) + var qps = uint64(10) + var durationInSecond = uint64(10) + request.PerRequestTimeoutInMS = &perRequestTimeoutInMS + request.QPS = &qps + request.DurationInSecond = &durationInSecond + request.Domain = "www.baidu.com" + protocol := "udp" + request.Protocol = &protocol + netDns.Spec.Request = request + + // Schedule + Schedule := new(v1beta1.SchedulePlan) + Schedule.Schedule = &crontab + Schedule.RoundNumber = 1 + Schedule.RoundTimeoutMinute = 1 + netDns.Spec.Schedule = Schedule + + e = frame.CreateResource(netDns) + Expect(e).NotTo(HaveOccurred(), "create netDns resource") + + e = common.CheckRuntime(frame, netDns, pluginManager.KindNameNetdns, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + + // get runtime resource + fake := &v1beta1.Netdns{ + ObjectMeta: metav1.ObjectMeta{ + Name: netDnsName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.Netdns{} + e = frame.GetResource(key, rs) + Expect(e).NotTo(HaveOccurred(), "get task runtime status") + + e = frame.DeleteResource(rs) + Expect(e).NotTo(HaveOccurred(), "delete task") + // check runtime resource is deleted after deleting task + e = common.GetRuntimeResource(frame, rs.Status.Resource, false) + Expect(e).NotTo(HaveOccurred(), "get task runtime resource") + + }) + + It("Successfully testing cascading deletion with Task NetReach Deployment Service and Ingress ", Label("E00010"), func() { + var e error + successRate := float64(1) + successMean := int64(1500) + crontab := "0 1" + netReachName := "netreach-" + tools.RandomName() + + netReach := new(v1beta1.NetReach) + netReach.Name = netReachName + + // agentSpec + agentSpec := new(v1beta1.AgentSpec) + agentSpec.TerminationGracePeriodMinutes = &termMin + agentSpec.Kind = types.KindDeployment + agentSpec.DeploymentReplicas = &replicas + netReach.Spec.AgentSpec = *agentSpec + + // successCondition + successCondition := new(v1beta1.NetSuccessCondition) + successCondition.SuccessRate = &successRate + successCondition.MeanAccessDelayInMs = &successMean + netReach.Spec.SuccessCondition = successCondition + + // target + target := new(v1beta1.NetReachTarget) + if !common.TestIPv4 && common.TestIPv6 { + target.Ingress = false + } else { + target.Ingress = true + } + target.LoadBalancer = true + target.ClusterIP = true + target.Endpoint = true + target.NodePort = true + target.MultusInterface = false + target.IPv4 = &common.TestIPv4 + target.IPv6 = &common.TestIPv6 + netReach.Spec.Target = target + + // request + request := new(v1beta1.NetHttpRequest) + request.PerRequestTimeoutInMS = 1000 + request.QPS = 10 + request.DurationInSecond = 10 + netReach.Spec.Request = request + + // Schedule + Schedule := new(v1beta1.SchedulePlan) + Schedule.Schedule = &crontab + Schedule.RoundNumber = 1 + Schedule.RoundTimeoutMinute = 1 + netReach.Spec.Schedule = Schedule + + e = frame.CreateResource(netReach) + Expect(e).NotTo(HaveOccurred(), "create netReach resource") + + e = common.CheckRuntime(frame, netReach, pluginManager.KindNameNetReach, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + + // get runtime resource + fake := &v1beta1.NetReach{ + ObjectMeta: metav1.ObjectMeta{ + Name: netReachName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.NetReach{} + e = frame.GetResource(key, rs) + Expect(e).NotTo(HaveOccurred(), "get task runtime status") + + e = frame.DeleteResource(rs) + Expect(e).NotTo(HaveOccurred(), "delete task") + // check runtime resource is deleted after deleting task + e = common.GetRuntimeResource(frame, rs.Status.Resource, true) + Expect(e).NotTo(HaveOccurred(), "get task runtime resource") + }) + + It("Successfully testing cascading deletion with Task NetAppHttpHealthy Deployment Service", Label("E00011"), func() { + var e error + successRate := float64(1) + successMean := int64(1500) + crontab := "0 1" + appHttpHealthName := "apphttphealth-get" + tools.RandomName() + + appHttpHealth := new(v1beta1.AppHttpHealthy) + appHttpHealth.Name = appHttpHealthName + + // agent + agentSpec := new(v1beta1.AgentSpec) + agentSpec.TerminationGracePeriodMinutes = &termMin + agentSpec.Kind = types.KindDeployment + agentSpec.DeploymentReplicas = &replicas + appHttpHealth.Spec.AgentSpec = *agentSpec + + // successCondition + successCondition := new(v1beta1.NetSuccessCondition) + successCondition.SuccessRate = &successRate + successCondition.MeanAccessDelayInMs = &successMean + appHttpHealth.Spec.SuccessCondition = successCondition + + // target + target := new(v1beta1.AppHttpHealthyTarget) + target.Method = "GET" + target.Host = "www.baidu.com" + appHttpHealth.Spec.Target = target + + // request + request := new(v1beta1.NetHttpRequest) + request.PerRequestTimeoutInMS = 2000 + request.QPS = 10 + request.DurationInSecond = 10 + appHttpHealth.Spec.Request = request + + // Schedule + Schedule := new(v1beta1.SchedulePlan) + Schedule.Schedule = &crontab + Schedule.RoundNumber = 1 + Schedule.RoundTimeoutMinute = 1 + appHttpHealth.Spec.Schedule = Schedule + + e = frame.CreateResource(appHttpHealth) + Expect(e).NotTo(HaveOccurred(), "create appHttpHealth resource") + + e = common.CheckRuntime(frame, appHttpHealth, pluginManager.KindNameAppHttpHealthy, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + + // get runtime resource + fake := &v1beta1.AppHttpHealthy{ + ObjectMeta: metav1.ObjectMeta{ + Name: appHttpHealthName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.AppHttpHealthy{} + e = frame.GetResource(key, rs) + Expect(e).NotTo(HaveOccurred(), "get task runtime status") + + e = frame.DeleteResource(rs) + Expect(e).NotTo(HaveOccurred(), "delete task") + // check runtime resource is deleted after deleting task + e = common.GetRuntimeResource(frame, rs.Status.Resource, false) + Expect(e).NotTo(HaveOccurred(), "get task runtime resource") + }) + + It("Successfully testing cascading deletion with Task NetDns Deployment Service ", Label("E00012"), func() { + var e error + successRate := float64(1) + successMean := int64(1500) + crontab := "0 1" + netDnsName := "netdns-e2e-" + tools.RandomName() + + netDns := new(v1beta1.Netdns) + netDns.Name = netDnsName + + // agentSpec + agentSpec := new(v1beta1.AgentSpec) + agentSpec.TerminationGracePeriodMinutes = &termMin + agentSpec.Kind = types.KindDeployment + agentSpec.DeploymentReplicas = &replicas + netDns.Spec.AgentSpec = *agentSpec + + // successCondition + successCondition := new(v1beta1.NetSuccessCondition) + successCondition.SuccessRate = &successRate + successCondition.MeanAccessDelayInMs = &successMean + netDns.Spec.SuccessCondition = successCondition + + // target + target := new(v1beta1.NetDnsTarget) + targetDns := new(v1beta1.NetDnsTargetDnsSpec) + targetDns.TestIPv4 = &common.TestIPv4 + targetDns.TestIPv6 = &common.TestIPv6 + ServiceName := "kube-dns" + ServiceNamespace := "kube-system" + targetDns.ServiceName = &ServiceName + targetDns.ServiceNamespace = &ServiceNamespace + target.NetDnsTargetDns = targetDns + netDns.Spec.Target = target + + // request + request := new(v1beta1.NetdnsRequest) + var perRequestTimeoutInMS = uint64(1000) + var qps = uint64(10) + var durationInSecond = uint64(10) + request.PerRequestTimeoutInMS = &perRequestTimeoutInMS + request.QPS = &qps + request.DurationInSecond = &durationInSecond + request.Domain = "www.baidu.com" + protocol := "udp" + request.Protocol = &protocol + netDns.Spec.Request = request + + // Schedule + Schedule := new(v1beta1.SchedulePlan) + Schedule.Schedule = &crontab + Schedule.RoundNumber = 1 + Schedule.RoundTimeoutMinute = 1 + netDns.Spec.Schedule = Schedule + + e = frame.CreateResource(netDns) + Expect(e).NotTo(HaveOccurred(), "create netDns resource") + + e = common.CheckRuntime(frame, netDns, pluginManager.KindNameNetdns, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + + // get runtime resource + fake := &v1beta1.Netdns{ + ObjectMeta: metav1.ObjectMeta{ + Name: netDnsName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.Netdns{} + e = frame.GetResource(key, rs) + Expect(e).NotTo(HaveOccurred(), "get task runtime status") + + e = frame.DeleteResource(rs) + Expect(e).NotTo(HaveOccurred(), "delete task") + // check runtime resource is deleted after deleting task + e = common.GetRuntimeResource(frame, rs.Status.Resource, false) + Expect(e).NotTo(HaveOccurred(), "get task runtime resource") + + }) + +})