diff --git a/docs/reference/apphttphealthy-zh_CN.md b/docs/reference/apphttphealthy-zh_CN.md index e9341ae6..ca7a581c 100644 --- a/docs/reference/apphttphealthy-zh_CN.md +++ b/docs/reference/apphttphealthy-zh_CN.md @@ -75,16 +75,16 @@ status: #### AgentSpec -| 字段 | 描述 | 结构 | 验证 | 取值 | 默认值 | -|-------------------------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------|-----|----------------------|-----------| -| annotation | agent 工作负载的 annotation | map[string]string | 可选 | | | -| kind | agent 工作负载的类型 | string | 可选 | Deployment、DaemonSet | DaemonSet | -| deploymentReplicas | agent 工作负载类型为 deployment 时的期望副本数 | int | 可选 | 大于等于 0 | 0 | -| affinity | agent 工作负载亲和性 | [labelSelector](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L1195) | 可选 | | | -| env | agent 工作负载环境变量 | [env](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/api/core/v1/types.go#L2012) | 可选 | | | -| hostNetwork | agent 工作负载是否使用宿主机网络 | bool | 可选 | true、false | false | -| resources | agent 工作负载资源使用配置 | [resources](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/api/core/v1/types.go#L2333) | 可选 | | | -| terminationGracePeriodMinutes | agent 工作负载完成任务后多少分钟之后终止 | int | 可选 | 大于等于 0 | 60 | +| 字段 | 描述 | 结构 | 验证 | 取值 | 默认值 | +|-------------------------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------|-----|----------------------|--------------------| +| annotation | agent 工作负载的 annotation | map[string]string | 可选 | | | +| kind | agent 工作负载的类型 | string | 可选 | Deployment、DaemonSet | DaemonSet | +| deploymentReplicas | agent 工作负载类型为 deployment 时的期望副本数 | int | 可选 | 大于等于 0 | 0 | +| affinity | agent 工作负载亲和性 | [labelSelector](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go#L1195) | 可选 | | | +| env | agent 工作负载环境变量 | [env](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/api/core/v1/types.go#L2012) | 可选 | | | +| hostNetwork | agent 工作负载是否使用宿主机网络 | bool | 可选 | true、false | false | +| resources | agent 工作负载资源使用配置 | [resources](https://github.com/kubernetes/kubernetes/blob/v1.27.0/staging/src/k8s.io/api/core/v1/types.go#L2333) | 可选 | | cpu:100m,memory:128Mi | +| terminationGracePeriodMinutes | agent 工作负载完成任务后多少分钟之后终止 | int | 可选 | 大于等于 0 | 60 | #### Schedule diff --git a/test/e2e/common/tools.go b/test/e2e/common/tools.go index c4156381..8d319b59 100644 --- a/test/e2e/common/tools.go +++ b/test/e2e/common/tools.go @@ -9,25 +9,30 @@ import ( "crypto/tls" "encoding/json" "fmt" - "github.com/kdoctor-io/kdoctor/api/v1/agentServer/models" - "github.com/onsi/ginkgo/v2" "io" + "k8s.io/apimachinery/pkg/api/errors" "math" "net" "net/http" "os/exec" + "reflect" "strings" "time" "github.com/docker/docker/api/types" docker_client "github.com/docker/docker/client" - "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/kdoctor.io/v1beta1" - kdoctor_report "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/system/v1beta1" - "github.com/kdoctor-io/kdoctor/pkg/pluginManager" + "github.com/onsi/ginkgo/v2" frame "github.com/spidernet-io/e2eframework/framework" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kdoctor-io/kdoctor/api/v1/agentServer/models" + "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/kdoctor.io/v1beta1" + kdoctor_report "github.com/kdoctor-io/kdoctor/pkg/k8s/apis/system/v1beta1" + "github.com/kdoctor-io/kdoctor/pkg/pluginManager" + kdoctor_types "github.com/kdoctor-io/kdoctor/pkg/types" ) func WaitKdoctorTaskDone(f *frame.Framework, task client.Object, taskKind string, timeout int) error { @@ -484,3 +489,382 @@ func CreateTestApp(name, namespace string, o []string) error { return nil } + +func CheckRuntime(f *frame.Framework, task client.Object, taskKind string, timeout int) error { + interval := time.Duration(10) + switch taskKind { + case pluginManager.KindNameNetReach: + fake := &v1beta1.NetReach{ + ObjectMeta: metav1.ObjectMeta{ + Name: task.GetName(), + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.NetReach{} + after := time.After(time.Duration(timeout) * time.Second) + create := false + for !create { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime create", pluginManager.KindNameNetReach, task.GetName()) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameNetReach, task.GetName(), err) + } + if rs.Status.Resource == nil { + continue + } + if rs.Status.Resource.RuntimeStatus == v1beta1.RuntimeCreated { + create = true + break + } + time.Sleep(time.Second * interval) + } + } + if err := checkAgentSpec(f, rs, rs.Spec.AgentSpec, rs.Status, pluginManager.KindNameNetReach); err != nil { + return fmt.Errorf("check AgentSpce err,reason= %v ", err) + } + case pluginManager.KindNameAppHttpHealthy: + fake := &v1beta1.AppHttpHealthy{ + ObjectMeta: metav1.ObjectMeta{ + Name: task.GetName(), + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.AppHttpHealthy{} + after := time.After(time.Duration(timeout) * time.Second) + create := false + for !create { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime create", pluginManager.KindNameAppHttpHealthy, task.GetName()) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameAppHttpHealthy, task.GetName(), err) + } + if rs.Status.Resource == nil { + continue + } + if rs.Status.Resource.RuntimeStatus == v1beta1.RuntimeCreated { + create = true + break + } + time.Sleep(time.Second * interval) + } + } + if err := checkAgentSpec(f, rs, rs.Spec.AgentSpec, rs.Status, pluginManager.KindNameAppHttpHealthy); err != nil { + return fmt.Errorf("check AgentSpce err,reason= %v ", err) + } + case pluginManager.KindNameNetdns: + fake := &v1beta1.Netdns{ + ObjectMeta: metav1.ObjectMeta{ + Name: task.GetName(), + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.Netdns{} + after := time.After(time.Duration(timeout) * time.Second) + create := false + for !create { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime create", pluginManager.KindNameNetdns, task.GetName()) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameNetdns, task.GetName(), err) + } + if rs.Status.Resource == nil { + continue + } + if rs.Status.Resource.RuntimeStatus == v1beta1.RuntimeCreated { + create = true + break + } + time.Sleep(time.Second * interval) + } + } + if err := checkAgentSpec(f, rs, rs.Spec.AgentSpec, rs.Status, pluginManager.KindNameNetdns); err != nil { + return fmt.Errorf("check AgentSpce err,reason= %v ", err) + } + default: + return fmt.Errorf("unknown task type: %s", task.GetObjectKind().GroupVersionKind().Kind) + } + return nil +} + +// checkAgentSpec check agentSpec generate deployment or daemonSet is right +func checkAgentSpec(f *frame.Framework, task client.Object, agentSpec v1beta1.AgentSpec, taskStatus v1beta1.TaskStatus, taskKind string) error { + + switch agentSpec.Kind { + case kdoctor_types.KindDaemonSet: + if taskStatus.Resource.RuntimeType != kdoctor_types.KindDaemonSet { + return fmt.Errorf("agent spec is %s,but status reource runtimeType is %s ", kdoctor_types.KindDaemonSet, taskStatus.Resource.RuntimeType) + } + runtime, err := f.GetDaemonSet(taskStatus.Resource.RuntimeName, TestNameSpace) + if err != nil { + return fmt.Errorf("failed get runtime daemonset %s ,reason=%v ", taskStatus.Resource.RuntimeName, err) + } + // compare annotation + if !reflect.DeepEqual(runtime.Spec.Template.Annotations, agentSpec.Annotation) { + return fmt.Errorf("create runtime daemonset annotation not equal,spec: %v real: %v", agentSpec.Annotation, runtime.Spec.Template.Annotations) + } + + // TODO(ii2day): compare env resource affinity + + // compare HostNetwork + if !reflect.DeepEqual(runtime.Spec.Template.Spec.HostNetwork, agentSpec.HostNetwork) { + return fmt.Errorf("create runtime daemonset HostNetwork not equal,spec: %v real: %v", agentSpec.HostNetwork, runtime.Spec.Template.Spec.HostNetwork) + } + + case kdoctor_types.KindDeployment: + runtime, err := f.GetDeployment(taskStatus.Resource.RuntimeName, TestNameSpace) + if err != nil { + return fmt.Errorf("failed get runtime deployment %s ,reason=%v ", taskStatus.Resource.RuntimeName, err) + } + // compare annotation + if !reflect.DeepEqual(runtime.Spec.Template.Annotations, agentSpec.Annotation) { + return fmt.Errorf("create runtime deployment annotation not equal,spec: %v real: %v", agentSpec.Annotation, runtime.Spec.Template.Annotations) + } + // TODO(ii2day): compare env resource affinity + + // compare HostNetwork + if !reflect.DeepEqual(runtime.Spec.Template.Spec.HostNetwork, agentSpec.HostNetwork) { + return fmt.Errorf("create runtime deployment HostNetwork not equal,spec: %v real: %v", agentSpec.HostNetwork, runtime.Spec.Template.Spec.HostNetwork) + } + + // compare replicas + if *agentSpec.DeploymentReplicas != runtime.Status.Replicas { + return fmt.Errorf("create runtime deployment Replicas not equal,spec: %d real: %d", *agentSpec.DeploymentReplicas, runtime.Status.Replicas) + } + default: + return fmt.Errorf("unknown agent kind %s ", agentSpec.Kind) + } + + if TestIPv4 { + _, err := f.GetService(*taskStatus.Resource.ServiceNameV4, TestNameSpace) + if err != nil { + return fmt.Errorf("failed get service %s ,reason=%v ", *taskStatus.Resource.ServiceNameV4, err) + } + } + + if TestIPv6 { + _, err := f.GetService(*taskStatus.Resource.ServiceNameV6, TestNameSpace) + if err != nil { + return fmt.Errorf("failed get service %s ,reason=%v ", *taskStatus.Resource.ServiceNameV6, err) + } + } + if taskKind == kdoctor_types.KindNameNetReach { + taskNr := task.(*v1beta1.NetReach) + if taskNr.Spec.Target.Ingress { + ig := &networkingv1.Ingress{} + fake := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: *taskStatus.Resource.ServiceNameV4, + Namespace: TestNameSpace, + }, + } + key := client.ObjectKeyFromObject(fake) + if err := f.GetResource(key, ig); err != nil { + return fmt.Errorf("task %s enable ingress test,but not found %s ingress err=%v", task.GetName(), *taskStatus.Resource.ServiceNameV4, err) + } + } + } + return nil +} + +// CheckRuntimeDeadLine check terminationGracePeriodMinutes delete deployment or daemonSet service ingress. +func CheckRuntimeDeadLine(f *frame.Framework, taskName, taskKind string, timeout int) error { + interval := time.Duration(10) + var runtimeResource *v1beta1.TaskResource + var terminationGracePeriodMinutes int64 + var testIngress = false + switch taskKind { + case kdoctor_types.KindNameNetReach: + fake := &v1beta1.NetReach{ + ObjectMeta: metav1.ObjectMeta{ + Name: taskName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.NetReach{} + after := time.After(time.Duration(timeout) * time.Second) + done := false + for !done { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime deadline", pluginManager.KindNameNetReach, taskName) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameNetReach, taskName, err) + } + if rs.Status.FinishTime != nil { + done = true + runtimeResource = rs.Status.Resource + terminationGracePeriodMinutes = *rs.Spec.AgentSpec.TerminationGracePeriodMinutes + testIngress = rs.Spec.Target.Ingress + break + } + time.Sleep(time.Second * interval) + } + } + case kdoctor_types.KindNameNetdns: + fake := &v1beta1.Netdns{ + ObjectMeta: metav1.ObjectMeta{ + Name: taskName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.Netdns{} + after := time.After(time.Duration(timeout) * time.Minute) + done := false + for !done { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime deadline", pluginManager.KindNameNetdns, taskName) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameNetdns, taskName, err) + } + if rs.Status.FinishTime != nil { + done = true + runtimeResource = rs.Status.Resource + terminationGracePeriodMinutes = *rs.Spec.AgentSpec.TerminationGracePeriodMinutes + break + } + time.Sleep(time.Second * interval) + } + } + case kdoctor_types.KindNameAppHttpHealthy: + fake := &v1beta1.AppHttpHealthy{ + ObjectMeta: metav1.ObjectMeta{ + Name: taskName, + }, + } + key := client.ObjectKeyFromObject(fake) + rs := &v1beta1.AppHttpHealthy{} + after := time.After(time.Duration(timeout) * time.Minute) + done := false + for !done { + select { + case <-after: + return fmt.Errorf("timeout wait task %s %s runtime deadline", pluginManager.KindNameAppHttpHealthy, taskName) + default: + if err := f.GetResource(key, rs); err != nil { + return fmt.Errorf("failed get resource %s %s, err: %v", pluginManager.KindNameAppHttpHealthy, taskName, err) + } + if rs.Status.FinishTime != nil { + done = true + runtimeResource = rs.Status.Resource + terminationGracePeriodMinutes = *rs.Spec.AgentSpec.TerminationGracePeriodMinutes + break + } + time.Sleep(time.Second * interval) + } + } + default: + return fmt.Errorf("unknown task %s type: %s", taskName, taskKind) + } + + c := time.After(time.Duration(terminationGracePeriodMinutes) * time.Minute) + // wait delete time + <-c + + c2 := time.After(time.Minute) + + // check runtime delete + runtimeDeleted := false + for !runtimeDeleted { + select { + case <-c2: + return fmt.Errorf("timeout delete %s task %s runtime", taskKind, taskName) + default: + if runtimeResource.RuntimeType == kdoctor_types.KindDaemonSet { + _, err := f.GetDaemonSet(runtimeResource.RuntimeName, TestNameSpace) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime daemonSet %s deleted \n", runtimeResource.RuntimeName) + runtimeDeleted = true + break + } + return fmt.Errorf("failed get task %s deamonSet reason=%v", taskName, err) + } + + } else { + _, err := f.GetDeployment(runtimeResource.RuntimeName, TestNameSpace) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime deployment %s deleted \n", runtimeResource.RuntimeName) + runtimeDeleted = true + break + } + return fmt.Errorf("failed get task %s deamonSet reason=%v", taskName, err) + } + } + time.Sleep(time.Second * interval) + } + } + + // check runtime service ingress delete + c3 := time.After(time.Minute) + serviceV4Deleted := false + serviceV6Deleted := false + ingressDeleted := false + for !serviceV4Deleted || !serviceV6Deleted { + select { + case <-c3: + return fmt.Errorf("timeout delete %s task %s runtime service", taskKind, taskName) + default: + if TestIPv4 { + _, err := f.GetService(*runtimeResource.ServiceNameV4, TestNameSpace) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime service v4 %s deleted \n", *runtimeResource.ServiceNameV4) + serviceV4Deleted = true + } else { + return fmt.Errorf("failed get task %s service %s reason=%v", taskName, *runtimeResource.ServiceNameV4, err) + } + } + if testIngress && !ingressDeleted { + ig := &networkingv1.Ingress{} + fake := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: *runtimeResource.ServiceNameV4, + Namespace: TestNameSpace, + }, + } + key := client.ObjectKeyFromObject(fake) + err = f.GetResource(key, ig) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime ingress %s deleted \n", *runtimeResource.ServiceNameV4) + ingressDeleted = true + } else { + return fmt.Errorf("task %s enable ingress test,but not found %s ingress err=%v", taskName, *runtimeResource.ServiceNameV4, err) + } + } + } else { + ingressDeleted = true + } + } else { + serviceV4Deleted = true + } + if TestIPv6 { + _, err := f.GetService(*runtimeResource.ServiceNameV6, TestNameSpace) + if err != nil { + if errors.IsNotFound(err) { + ginkgo.GinkgoWriter.Printf("task runtime service v6 %s deleted \n", *runtimeResource.ServiceNameV6) + serviceV6Deleted = true + break + } + return fmt.Errorf("failed get task %s service %s reason=%v", taskName, *runtimeResource.ServiceNameV6, err) + } + } else { + serviceV6Deleted = true + } + time.Sleep(time.Second * interval) + } + } + + return nil + +} diff --git a/test/e2e/netdns/netdns_test.go b/test/e2e/netdns/netdns_test.go index 403be8ec..cda9bc83 100644 --- a/test/e2e/netdns/netdns_test.go +++ b/test/e2e/netdns/netdns_test.go @@ -16,7 +16,7 @@ import ( var _ = Describe("testing netDns ", Label("netDns"), func() { var targetDomain = "%s.kubernetes.default.svc.cluster.local" - var termMin = int64(3) + var termMin = int64(1) It("Successfully testing Cluster Dns Server case", Label("D00001", "C00005"), func() { var e error successRate := float64(1) @@ -133,6 +133,9 @@ var _ = Describe("testing netDns ", Label("netDns"), func() { e = frame.CreateResource(netDns) Expect(e).NotTo(HaveOccurred(), "create netDns resource") + e = common.CheckRuntime(frame, netDns, pluginManager.KindNameNetdns, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, netDns, pluginManager.KindNameNetdns, 120) Expect(e).NotTo(HaveOccurred(), "wait netDns task finish") @@ -140,5 +143,7 @@ var _ = Describe("testing netDns ", Label("netDns"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, netDnsName, pluginManager.KindNameNetdns, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) }) diff --git a/test/e2e/netreach/netreach_suite_test.go b/test/e2e/netreach/netreach_suite_test.go index 41e48d7c..3f2da60e 100644 --- a/test/e2e/netreach/netreach_suite_test.go +++ b/test/e2e/netreach/netreach_suite_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" e2e "github.com/spidernet-io/e2eframework/framework" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/runtime" "testing" ) @@ -22,7 +23,7 @@ var reportNum int var _ = BeforeSuite(func() { defer GinkgoRecover() var e error - frame, e = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{kdoctor_v1beta1.AddToScheme}) + frame, e = e2e.NewFramework(GinkgoT(), []func(*runtime.Scheme) error{kdoctor_v1beta1.AddToScheme, networkingv1.AddToScheme}) Expect(e).NotTo(HaveOccurred()) nodeLIst, e := frame.GetNodeList() Expect(e).NotTo(HaveOccurred(), "get node list") diff --git a/test/e2e/netreach/netreach_test.go b/test/e2e/netreach/netreach_test.go index 089a6f0b..b7dd3272 100644 --- a/test/e2e/netreach/netreach_test.go +++ b/test/e2e/netreach/netreach_test.go @@ -13,7 +13,7 @@ import ( ) var _ = Describe("testing netReach ", Label("netReach"), func() { - var termMin = int64(3) + var termMin = int64(1) It("success testing netReach", Label("B00001", "C00004"), func() { var e error successRate := float64(1) @@ -68,6 +68,9 @@ var _ = Describe("testing netReach ", Label("netReach"), func() { e = frame.CreateResource(netReach) Expect(e).NotTo(HaveOccurred(), "create netReach resource") + e = common.CheckRuntime(frame, netReach, pluginManager.KindNameNetReach, 60) + Expect(e).NotTo(HaveOccurred(), "check task runtime spec") + e = common.WaitKdoctorTaskDone(frame, netReach, pluginManager.KindNameNetReach, 120) Expect(e).NotTo(HaveOccurred(), "wait netReach task finish") @@ -75,5 +78,7 @@ var _ = Describe("testing netReach ", Label("netReach"), func() { Expect(e).NotTo(HaveOccurred(), "compare report and task") Expect(success).To(BeTrue(), "compare report and task result") + e = common.CheckRuntimeDeadLine(frame, netReachName, pluginManager.KindNameNetReach, 120) + Expect(e).NotTo(HaveOccurred(), "check task runtime resource delete") }) })