From 28bcebf37bca2557bef73b2367127078eb0b05e5 Mon Sep 17 00:00:00 2001 From: piglei Date: Tue, 19 Mar 2024 11:24:22 +0800 Subject: [PATCH] Finish re-organize operator module --- operator/api/v1alpha1/bkapp_conversion.go | 10 +- operator/api/v1alpha2/bkapp_webhook.go | 4 +- operator/api/v1alpha2/bkapp_webhook_test.go | 10 +- operator/api/v1alpha2/compat.go | 4 +- operator/api/v1alpha2/compat_test.go | 4 +- operator/controllers/bkapp_controller_test.go | 4 +- operator/main.go | 4 +- .../bkapp/autoscaling/autoscaling.go | 3 +- .../pkg/controllers/bkapp/autoscaling/gpa.go | 24 ++++ .../controllers/bkapp/autoscaling/gpa_test.go | 96 +++++++++++++ .../pkg/controllers/bkapp/envs/env_overlay.go | 4 +- .../bkapp/envs/env_overlay_test.go | 4 +- operator/pkg/controllers/bkapp/hooks/hooks.go | 4 +- .../bkapp/hooks/resources/hooks.go | 6 +- operator/pkg/controllers/bkapp/predicates.go | 10 +- .../bkapp/processes/deployments.go | 8 +- .../bkapp/processes/resources/deployment.go | 6 +- .../processes/resources/deployment_test.go | 6 +- .../kubestatus => health}/deployment.go | 22 +-- .../kubestatus => health}/deployment_test.go | 4 +- .../health_suite_test.go} | 6 +- .../pkg/{utils/kubestatus => health}/pod.go | 18 +-- .../{utils/kubestatus => health}/pod_test.go | 6 +- .../kubestatus/health.go => health/types.go} | 2 +- operator/pkg/kubeutil/{utils.go => client.go} | 32 +---- .../{utils_test.go => client_test.go} | 13 +- .../deployment.go} | 22 +-- .../kubetypes/common.go => kubeutil/misc.go} | 22 ++- .../common_test.go => kubeutil/misc_test.go} | 2 +- .../client.go => kubeutil/traced_client.go} | 30 ++-- operator/pkg/utils/kubestatus/gpa.go | 50 ------- operator/pkg/utils/kubestatus/gpa_test.go | 129 ------------------ 32 files changed, 251 insertions(+), 318 deletions(-) rename operator/pkg/{utils/kubestatus => health}/deployment.go (87%) rename operator/pkg/{utils/kubestatus => health}/deployment_test.go (98%) rename operator/pkg/{utils/kubetypes/kubetypes_suite_test.go => health/health_suite_test.go} (91%) rename operator/pkg/{utils/kubestatus => health}/pod.go (90%) rename operator/pkg/{utils/kubestatus => health}/pod_test.go (97%) rename operator/pkg/{utils/kubestatus/health.go => health/types.go} (98%) rename operator/pkg/kubeutil/{utils.go => client.go} (74%) rename operator/pkg/kubeutil/{utils_test.go => client_test.go} (91%) rename operator/pkg/{utils/kubestatus/kubestatus_suite_test.go => kubeutil/deployment.go} (70%) rename operator/pkg/{utils/kubetypes/common.go => kubeutil/misc.go} (84%) rename operator/pkg/{utils/kubetypes/common_test.go => kubeutil/misc_test.go} (99%) rename operator/pkg/{client/client.go => kubeutil/traced_client.go} (84%) delete mode 100644 operator/pkg/utils/kubestatus/gpa.go delete mode 100644 operator/pkg/utils/kubestatus/gpa_test.go diff --git a/operator/api/v1alpha1/bkapp_conversion.go b/operator/api/v1alpha1/bkapp_conversion.go index 5f67fe4274..d629c4563b 100644 --- a/operator/api/v1alpha1/bkapp_conversion.go +++ b/operator/api/v1alpha1/bkapp_conversion.go @@ -8,7 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/conversion" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) var _ conversion.Convertible = &BkApp{} @@ -61,10 +61,10 @@ func (src *BkApp) ConvertTo(dstRaw conversion.Hub) error { } // Save legacy proc image and resource configs to annotations - if err := kubetypes.SetJsonAnnotation(dst, LegacyProcImageAnnoKey, legacyProcImageConfig); err != nil { + if err := kubeutil.SetJsonAnnotation(dst, LegacyProcImageAnnoKey, legacyProcImageConfig); err != nil { return err } - if err := kubetypes.SetJsonAnnotation(dst, LegacyProcResAnnoKey, legacyProcResConfig); err != nil { + if err := kubeutil.SetJsonAnnotation(dst, LegacyProcResAnnoKey, legacyProcResConfig); err != nil { return err } @@ -145,8 +145,8 @@ func (dst *BkApp) ConvertFrom(srcRaw conversion.Hub) error { ) // Handle Processes field - legacyProcImageConfig, _ := kubetypes.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](src, LegacyProcImageAnnoKey) - legacyProcResConfig, _ := kubetypes.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](src, LegacyProcResAnnoKey) + legacyProcImageConfig, _ := kubeutil.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](src, LegacyProcImageAnnoKey) + legacyProcResConfig, _ := kubeutil.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](src, LegacyProcResAnnoKey) for _, proc := range src.Spec.Processes { dstProc := Process{ Name: proc.Name, diff --git a/operator/api/v1alpha2/bkapp_webhook.go b/operator/api/v1alpha2/bkapp_webhook.go index f8d940ad82..833d0ae8ea 100644 --- a/operator/api/v1alpha2/bkapp_webhook.go +++ b/operator/api/v1alpha2/bkapp_webhook.go @@ -38,7 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "bk.tencent.com/paas-app-operator/pkg/config" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" "bk.tencent.com/paas-app-operator/pkg/utils/quota" "bk.tencent.com/paas-app-operator/pkg/utils/stringx" ) @@ -184,7 +184,7 @@ func (r *BkApp) validateAnnotations() *field.Error { } // 通过注解配置的进程资源信息，也需要校验是否合法 - legacyProcResConfig, err := kubetypes.GetJsonAnnotation[LegacyProcConfig]( + legacyProcResConfig, err := kubeutil.GetJsonAnnotation[LegacyProcConfig]( r, LegacyProcResAnnoKey, ) // 获取进程中的资源配额注解成功，才需要进行检查 diff --git a/operator/api/v1alpha2/bkapp_webhook_test.go b/operator/api/v1alpha2/bkapp_webhook_test.go index ee5df06530..b5687a4790 100644 --- a/operator/api/v1alpha2/bkapp_webhook_test.go +++ b/operator/api/v1alpha2/bkapp_webhook_test.go @@ -32,7 +32,7 @@ import ( paasv1alpha1 "bk.tencent.com/paas-app-operator/api/v1alpha1" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" "bk.tencent.com/paas-app-operator/pkg/config" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" "bk.tencent.com/paas-app-operator/pkg/utils/stringx" ) @@ -647,7 +647,7 @@ var _ = Describe("test webhook.Validator", func() { It("Normal", func() { legacyProcResConfig := make(paasv1alpha2.LegacyProcConfig) legacyProcResConfig["web"] = map[string]string{"cpu": "2", "memory": "2G"} - _ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig) + _ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig) err := bkapp.ValidateCreate() Expect(err).To(BeNil()) @@ -655,7 +655,7 @@ var _ = Describe("test webhook.Validator", func() { It("Invalid unset", func() { legacyProcResConfig := make(paasv1alpha2.LegacyProcConfig) legacyProcResConfig["web"] = map[string]string{"cpu": "", "memory": "2G"} - _ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig) + _ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig) err := bkapp.ValidateCreate() Expect(err).NotTo(BeNil()) @@ -663,7 +663,7 @@ var _ = Describe("test webhook.Validator", func() { It("Invalid exceed cpu max limit", func() { legacyProcResConfig := make(paasv1alpha2.LegacyProcConfig) legacyProcResConfig["web"] = map[string]string{"cpu": "6", "memory": "2G"} - _ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig) + _ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig) err := bkapp.ValidateCreate() Expect(err).NotTo(BeNil()) @@ -671,7 +671,7 @@ var _ = Describe("test webhook.Validator", func() { It("Invalid exceed memory max limit", func() { legacyProcResConfig := make(paasv1alpha2.LegacyProcConfig) legacyProcResConfig["web"] = map[string]string{"cpu": "2", "memory": "8G"} - _ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig) + _ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig) err := bkapp.ValidateCreate() Expect(err).NotTo(BeNil()) diff --git a/operator/api/v1alpha2/compat.go b/operator/api/v1alpha2/compat.go index 9ce04767ab..3c4dbe18c7 100644 --- a/operator/api/v1alpha2/compat.go +++ b/operator/api/v1alpha2/compat.go @@ -24,7 +24,7 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) // ProcImageGetter help getting container image from bkapp @@ -44,7 +44,7 @@ func NewProcImageGetter(bkapp *BkApp) *ProcImageGetter { // - return: , , func (r *ProcImageGetter) Get(name string) (string, corev1.PullPolicy, error) { // Legacy API version: read image configs from annotations - legacyProcImageConfig, _ := kubetypes.GetJsonAnnotation[LegacyProcConfig]( + legacyProcImageConfig, _ := kubeutil.GetJsonAnnotation[LegacyProcConfig]( r.bkapp, LegacyProcImageAnnoKey, ) diff --git a/operator/api/v1alpha2/compat_test.go b/operator/api/v1alpha2/compat_test.go index 067bdd7db0..faed28830b 100644 --- a/operator/api/v1alpha2/compat_test.go +++ b/operator/api/v1alpha2/compat_test.go @@ -25,7 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) var _ = Describe("test compat", func() { @@ -67,7 +67,7 @@ var _ = Describe("test compat", func() { Context("Test ProcImageGetter", func() { It("Get Legacy", func() { - _ = kubetypes.SetJsonAnnotation( + _ = kubeutil.SetJsonAnnotation( bkapp, paasv1alpha2.LegacyProcImageAnnoKey, paasv1alpha2.LegacyProcConfig{ "web": {"image": "busybox:1.0.0", "policy": "Never"}, }, diff --git a/operator/controllers/bkapp_controller_test.go b/operator/controllers/bkapp_controller_test.go index 1377c5b9bc..18ecfc7db3 100644 --- a/operator/controllers/bkapp_controller_test.go +++ b/operator/controllers/bkapp_controller_test.go @@ -42,9 +42,9 @@ import ( paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" "bk.tencent.com/paas-app-operator/pkg/platform/external" "bk.tencent.com/paas-app-operator/pkg/testing" - "bk.tencent.com/paas-app-operator/pkg/utils/kubestatus" ) var _ = Describe("", func() { @@ -222,7 +222,7 @@ var _ = Describe("", func() { createdDeployment.Status.ObservedGeneration = createdDeployment.Generation createdDeployment.Status.Replicas = *createdDeployment.Spec.Replicas createdDeployment.Status.UpdatedReplicas = *createdDeployment.Spec.Replicas - cond := kubestatus.FindDeploymentStatusCondition( + cond := kubeutil.FindDeploymentStatusCondition( createdDeployment.Status.Conditions, appsv1.DeploymentAvailable, ) diff --git a/operator/main.go b/operator/main.go index 7983d2ad96..c77f2cfa6c 100644 --- a/operator/main.go +++ b/operator/main.go @@ -48,9 +48,9 @@ import ( paasv1alpha1 "bk.tencent.com/paas-app-operator/api/v1alpha1" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" "bk.tencent.com/paas-app-operator/controllers" - "bk.tencent.com/paas-app-operator/pkg/client" "bk.tencent.com/paas-app-operator/pkg/config" dgmingress "bk.tencent.com/paas-app-operator/pkg/controllers/dgroupmapping/ingress" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" "bk.tencent.com/paas-app-operator/pkg/platform/external" //+kubebuilder:scaffold:imports @@ -136,7 +136,7 @@ func main() { setupLog.Error(err, "unable to start manager") os.Exit(1) } - mgrCli := client.New(mgr.GetClient()) + mgrCli := kubeutil.NewTracedClient(mgr.GetClient()) mgrScheme := mgr.GetScheme() bkappMgrOpts := genGroupKindMgrOpts(paasv1alpha1.GroupKindBkApp, projConf.Controller) diff --git a/operator/pkg/controllers/bkapp/autoscaling/autoscaling.go b/operator/pkg/controllers/bkapp/autoscaling/autoscaling.go index f17596e3d6..237bf78152 100644 --- a/operator/pkg/controllers/bkapp/autoscaling/autoscaling.go +++ b/operator/pkg/controllers/bkapp/autoscaling/autoscaling.go @@ -32,7 +32,6 @@ import ( "bk.tencent.com/paas-app-operator/pkg/config" "bk.tencent.com/paas-app-operator/pkg/kubeutil" "bk.tencent.com/paas-app-operator/pkg/metrics" - "bk.tencent.com/paas-app-operator/pkg/utils/kubestatus" autoscaling "github.com/Tencent/bk-bcs/bcs-runtime/bcs-k8s/bcs-component/bcs-general-pod-autoscaler/pkg/apis/autoscaling/v1alpha1" ) @@ -140,7 +139,7 @@ func (r *AutoscalingReconciler) updateCondition(ctx context.Context, bkapp *paas } for _, gpa := range current { - healthStatus := kubestatus.GenGPAHealthStatus(gpa) + healthStatus := GenGPAHealthStatus(gpa) if healthStatus.Phase != paasv1alpha2.HealthHealthy { apimeta.SetStatusCondition(&bkapp.Status.Conditions, metav1.Condition{ Type: paasv1alpha2.AutoscalingAvailable, diff --git a/operator/pkg/controllers/bkapp/autoscaling/gpa.go b/operator/pkg/controllers/bkapp/autoscaling/gpa.go index 57c9d01a86..ea51275a00 100644 --- a/operator/pkg/controllers/bkapp/autoscaling/gpa.go +++ b/operator/pkg/controllers/bkapp/autoscaling/gpa.go @@ -27,6 +27,7 @@ import ( paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/envs" + "bk.tencent.com/paas-app-operator/pkg/health" autoscaling "github.com/Tencent/bk-bcs/bcs-runtime/bcs-k8s/bcs-component/bcs-general-pod-autoscaler/pkg/apis/autoscaling/v1alpha1" ) @@ -99,3 +100,26 @@ func buildMetricSpecs(policy paasv1alpha2.ScalingPolicy) (metrics []autoscaling. return metrics } + +// GenGPAHealthStatus check if the GPA is healthy +// For a deployment: +// +// healthy means the GPA is available, ready to scale workloads with policy. +// unhealthy means the GPA is failed when reconciled. +func GenGPAHealthStatus(gpa *autoscaling.GeneralPodAutoscaler) *health.HealthStatus { + for _, condition := range gpa.Status.Conditions { + if condition.Status == v1.ConditionFalse { + return &health.HealthStatus{ + Phase: paasv1alpha2.HealthUnhealthy, + Reason: condition.Reason, + Message: condition.Message, + } + } + } + + return &health.HealthStatus{ + Phase: paasv1alpha2.HealthHealthy, + Reason: paasv1alpha2.AutoscalingAvailable, + Message: "", + } +} diff --git a/operator/pkg/controllers/bkapp/autoscaling/gpa_test.go b/operator/pkg/controllers/bkapp/autoscaling/gpa_test.go index ba173677c2..94e680e5ee 100644 --- a/operator/pkg/controllers/bkapp/autoscaling/gpa_test.go +++ b/operator/pkg/controllers/bkapp/autoscaling/gpa_test.go @@ -122,3 +122,99 @@ var _ = Describe("GPAResources", func() { }) }) }) + +var _ = Describe("Test gpa", func() { + var gpa *autoscaling.GeneralPodAutoscaler + var builder *fake.ClientBuilder + var scheme *runtime.Scheme + + BeforeEach(func() { + gpa = &autoscaling.GeneralPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "autoscaling.tkex.tencent.com/v1alpha1", + Kind: "GeneralPodAutoscaler", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "default-web-gpa", + }, + Spec: autoscaling.GeneralPodAutoscalerSpec{ + MinReplicas: lo.ToPtr(int32(2)), + MaxReplicas: int32(5), + ScaleTargetRef: autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "default-web", + }, + AutoScalingDrivenMode: autoscaling.AutoScalingDrivenMode{ + MetricMode: &autoscaling.MetricMode{ + Metrics: []autoscaling.MetricSpec{}, + }, + }, + }, + } + + builder = fake.NewClientBuilder() + scheme = runtime.NewScheme() + Expect(autoscaling.AddToScheme(scheme)).NotTo(HaveOccurred()) + builder.WithScheme(scheme) + }) + + Context("test GenGPAHealthStatus", func() { + It("test healthy", func() { + gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{ + { + Type: autoscaling.AbleToScale, + Status: v1.ConditionTrue, + Reason: "ReadyForNewScale", + Message: "recommended size matches current size.", + }, + { + Type: autoscaling.ScalingActive, + Status: v1.ConditionTrue, + Reason: "ValidMetricFound", + Message: "the GPA was able to successfully calculate a replica count from.", + }, + } + status := GenGPAHealthStatus(gpa) + Expect(status.Phase).To(Equal(paasv1alpha2.HealthHealthy)) + Expect(status.Reason).To(Equal("AutoscalingAvailable")) + }) + It("test ScalingActive unhealthy", func() { + gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{ + { + Type: autoscaling.AbleToScale, + Status: v1.ConditionTrue, + Reason: "ReadyForNewScale", + Message: "recommended size matches current size.", + }, + { + Type: autoscaling.ScalingActive, + Status: v1.ConditionFalse, + Reason: "FailedGetResourceMetric", + Message: "the GPA was unable to compute the replica count: unable to get metrics for resource cpu.", + }, + } + status := GenGPAHealthStatus(gpa) + Expect(status.Phase).To(Equal(paasv1alpha2.HealthUnhealthy)) + Expect(status.Reason).To(Equal("FailedGetResourceMetric")) + }) + It("test AbleToScale unhealthy", func() { + gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{ + { + Type: autoscaling.AbleToScale, + Status: v1.ConditionFalse, + Reason: "FailedGetScale", + }, + { + Type: autoscaling.ScalingActive, + Status: v1.ConditionTrue, + Reason: "ValidMetricFound", + Message: "the GPA was able to successfully calculate a replica count from.", + }, + } + status := GenGPAHealthStatus(gpa) + Expect(status.Phase).To(Equal(paasv1alpha2.HealthUnhealthy)) + Expect(status.Reason).To(Equal("FailedGetScale")) + }) + }) +}) diff --git a/operator/pkg/controllers/bkapp/envs/env_overlay.go b/operator/pkg/controllers/bkapp/envs/env_overlay.go index a677236ea9..6310dbb14f 100644 --- a/operator/pkg/controllers/bkapp/envs/env_overlay.go +++ b/operator/pkg/controllers/bkapp/envs/env_overlay.go @@ -24,7 +24,7 @@ import ( paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" "bk.tencent.com/paas-app-operator/pkg/config" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" "bk.tencent.com/paas-app-operator/pkg/utils/quota" ) @@ -217,7 +217,7 @@ func (r *ProcResourcesGetter) Default() corev1.ResourceRequirements { // - return: , func (r *ProcResourcesGetter) GetByProc(name string) (result corev1.ResourceRequirements, err error) { // Legacy version: try to read resources configs from legacy annotation - legacyProcResourcesConfig, _ := kubetypes.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig]( + legacyProcResourcesConfig, _ := kubeutil.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig]( r.bkapp, paasv1alpha2.LegacyProcResAnnoKey, ) diff --git a/operator/pkg/controllers/bkapp/envs/env_overlay_test.go b/operator/pkg/controllers/bkapp/envs/env_overlay_test.go index d5463ee479..df815fb012 100644 --- a/operator/pkg/controllers/bkapp/envs/env_overlay_test.go +++ b/operator/pkg/controllers/bkapp/envs/env_overlay_test.go @@ -30,7 +30,7 @@ import ( . "github.com/onsi/gomega" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) var _ = Describe("Environment overlay related functions", func() { @@ -236,7 +236,7 @@ var _ = Describe("Environment overlay related functions", func() { }) It("Get Legacy", func() { - _ = kubetypes.SetJsonAnnotation( + _ = kubeutil.SetJsonAnnotation( bkapp, paasv1alpha2.LegacyProcResAnnoKey, paasv1alpha2.LegacyProcConfig{ "web": {"cpu": "2", "memory": "2Gi"}, }, diff --git a/operator/pkg/controllers/bkapp/hooks/hooks.go b/operator/pkg/controllers/bkapp/hooks/hooks.go index 0df7bb9191..d4ab27d6d7 100644 --- a/operator/pkg/controllers/bkapp/hooks/hooks.go +++ b/operator/pkg/controllers/bkapp/hooks/hooks.go @@ -39,8 +39,8 @@ import ( "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/labels" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names" hookres "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/hooks/resources" + "bk.tencent.com/paas-app-operator/pkg/health" "bk.tencent.com/paas-app-operator/pkg/metrics" - "bk.tencent.com/paas-app-operator/pkg/utils/kubestatus" ) // HookPodsHistoryLimit 最大保留的 Hook Pod 数量（单种类型） @@ -159,7 +159,7 @@ func (r *HookReconciler) getCurrentState(ctx context.Context, bkapp *paasv1alpha }) } - healthStatus := kubestatus.CheckPodHealthStatus(&pod) + healthStatus := health.CheckPodHealthStatus(&pod) return hookres.HookInstance{ Pod: &pod, Status: &paasv1alpha2.HookStatus{ diff --git a/operator/pkg/controllers/bkapp/hooks/resources/hooks.go b/operator/pkg/controllers/bkapp/hooks/resources/hooks.go index fc9bde85d8..40514f9a17 100644 --- a/operator/pkg/controllers/bkapp/hooks/resources/hooks.go +++ b/operator/pkg/controllers/bkapp/hooks/resources/hooks.go @@ -32,7 +32,7 @@ import ( "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/labels" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/envs" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) const ( @@ -146,8 +146,8 @@ func BuildPreReleaseHook(bkapp *paasv1alpha2.BkApp, status *paasv1alpha2.HookSta Containers: []corev1.Container{ { Image: image, - Command: kubetypes.ReplaceCommandEnvVariables(command), - Args: kubetypes.ReplaceCommandEnvVariables(args), + Command: kubeutil.ReplaceCommandEnvVariables(command), + Args: kubeutil.ReplaceCommandEnvVariables(args), Env: common.GetAppEnvs(bkapp), Name: "hook", ImagePullPolicy: pullPolicy, diff --git a/operator/pkg/controllers/bkapp/predicates.go b/operator/pkg/controllers/bkapp/predicates.go index 04c77ae39a..8647975279 100644 --- a/operator/pkg/controllers/bkapp/predicates.go +++ b/operator/pkg/controllers/bkapp/predicates.go @@ -8,7 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" - "bk.tencent.com/paas-app-operator/pkg/utils/kubestatus" + "bk.tencent.com/paas-app-operator/pkg/health" ) // NewHookSuccessPredicate create an GenericHookPredicate instance which will handle hook run successful event. @@ -22,8 +22,8 @@ func NewHookSuccessPredicate() predicate.Predicate { return &GenericHookPredicate{ Logger: logf.Log, updateFunc: func(oldPod, newPod *corev1.Pod) bool { - oldHealthStatus := kubestatus.CheckPodHealthStatus(oldPod) - newHealthStatus := kubestatus.CheckPodHealthStatus(newPod) + oldHealthStatus := health.CheckPodHealthStatus(oldPod) + newHealthStatus := health.CheckPodHealthStatus(newPod) // the pod state is changing to ready return (oldHealthStatus.Phase != paasv1alpha2.HealthHealthy) && @@ -43,8 +43,8 @@ func NewHookFailedPredicate() predicate.Predicate { return &GenericHookPredicate{ Logger: logf.Log, updateFunc: func(oldPod, newPod *corev1.Pod) bool { - oldHealthStatus := kubestatus.CheckPodHealthStatus(oldPod) - newHealthStatus := kubestatus.CheckPodHealthStatus(newPod) + oldHealthStatus := health.CheckPodHealthStatus(oldPod) + newHealthStatus := health.CheckPodHealthStatus(newPod) // the pod state is changing to not-ready return (oldHealthStatus.Phase != paasv1alpha2.HealthUnhealthy) && diff --git a/operator/pkg/controllers/bkapp/processes/deployments.go b/operator/pkg/controllers/bkapp/processes/deployments.go index d0ba7ee1f7..96f4537b26 100644 --- a/operator/pkg/controllers/bkapp/processes/deployments.go +++ b/operator/pkg/controllers/bkapp/processes/deployments.go @@ -39,9 +39,9 @@ import ( "bk.tencent.com/paas-app-operator/controllers/base" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/processes/resources" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/svcdisc" + "bk.tencent.com/paas-app-operator/pkg/health" "bk.tencent.com/paas-app-operator/pkg/kubeutil" "bk.tencent.com/paas-app-operator/pkg/metrics" - "bk.tencent.com/paas-app-operator/pkg/utils/kubestatus" ) // NewDeploymentReconciler will return a DeploymentReconciler with given k8s client @@ -226,14 +226,14 @@ func (r *DeploymentReconciler) updateCondition(ctx context.Context, bkapp *paasv availableCount := 0 anyFailed := false for _, deployment := range current { - healthStatus := kubestatus.CheckDeploymentHealthStatus(deployment) + healthStatus := health.CheckDeploymentHealthStatus(deployment) if healthStatus.Phase == paasv1alpha2.HealthHealthy { availableCount += 1 continue } - failMessage, err := kubestatus.GetDeploymentDirectFailMessage(ctx, r.Client, deployment) - if errors.Is(err, kubestatus.ErrDeploymentStillProgressing) { + failMessage, err := health.GetDeploymentDirectFailMessage(ctx, r.Client, deployment) + if errors.Is(err, health.ErrDeploymentStillProgressing) { continue } if healthStatus.Phase == paasv1alpha2.HealthUnhealthy { diff --git a/operator/pkg/controllers/bkapp/processes/resources/deployment.go b/operator/pkg/controllers/bkapp/processes/resources/deployment.go index a98738a27d..1acdb6f8a4 100644 --- a/operator/pkg/controllers/bkapp/processes/resources/deployment.go +++ b/operator/pkg/controllers/bkapp/processes/resources/deployment.go @@ -38,7 +38,7 @@ import ( "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/envs" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/processes/volumes" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) const ( @@ -187,8 +187,8 @@ func buildContainers( Resources: resRequirements, ImagePullPolicy: pullPolicy, Env: envs, - Command: kubetypes.ReplaceCommandEnvVariables(command), - Args: kubetypes.ReplaceCommandEnvVariables(args), + Command: kubeutil.ReplaceCommandEnvVariables(command), + Args: kubeutil.ReplaceCommandEnvVariables(args), } if proc.TargetPort != 0 { container.Ports = []corev1.ContainerPort{{ContainerPort: proc.TargetPort}} diff --git a/operator/pkg/controllers/bkapp/processes/resources/deployment_test.go b/operator/pkg/controllers/bkapp/processes/resources/deployment_test.go index fde6e27d1a..b35ca37ed6 100644 --- a/operator/pkg/controllers/bkapp/processes/resources/deployment_test.go +++ b/operator/pkg/controllers/bkapp/processes/resources/deployment_test.go @@ -30,7 +30,7 @@ import ( paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" "bk.tencent.com/paas-app-operator/pkg/config" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/labels" - "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) var _ = Describe("Test build deployments from BkApp", func() { @@ -175,7 +175,7 @@ var _ = Describe("Test build deployments from BkApp", func() { }) It("legacy version", func() { - _ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcImageAnnoKey, paasv1alpha2.LegacyProcConfig{ + _ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcImageAnnoKey, paasv1alpha2.LegacyProcConfig{ "web": {"image": "busybox:1.0.0", "policy": "Never"}, "worker": {"image": "busybox:2.0.0", "policy": "Always"}, }) @@ -237,7 +237,7 @@ var _ = Describe("Test build deployments from BkApp", func() { }) It("legacy version", func() { - _ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, paasv1alpha2.LegacyProcConfig{ + _ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, paasv1alpha2.LegacyProcConfig{ "web": {"cpu": "1", "memory": "1Gi"}, "worker": {"cpu": "2", "memory": "2Gi"}, }) diff --git a/operator/pkg/utils/kubestatus/deployment.go b/operator/pkg/health/deployment.go similarity index 87% rename from operator/pkg/utils/kubestatus/deployment.go rename to operator/pkg/health/deployment.go index 846ca6f020..c0d56fe07b 100644 --- a/operator/pkg/utils/kubestatus/deployment.go +++ b/operator/pkg/health/deployment.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubestatus +package health import ( "context" @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) // ErrDeploymentStillProgressing indicates the deployment is progressing @@ -48,14 +49,14 @@ func CheckDeploymentHealthStatus(deployment *appsv1.Deployment) *HealthStatus { Message: "Waiting for rollout to finish: observed deployment generation less then desired generation", } } else { - failureCond := FindDeploymentStatusCondition(deployment.Status.Conditions, appsv1.DeploymentReplicaFailure) + failureCond := kubeutil.FindDeploymentStatusCondition(deployment.Status.Conditions, appsv1.DeploymentReplicaFailure) if failureCond != nil && failureCond.Status == corev1.ConditionTrue { return makeStatusFromDeploymentCondition(paasv1alpha2.HealthUnhealthy, failureCond) } replicas := *deployment.Spec.Replicas - progressingCond := FindDeploymentStatusCondition(deployment.Status.Conditions, appsv1.DeploymentProgressing) + progressingCond := kubeutil.FindDeploymentStatusCondition(deployment.Status.Conditions, appsv1.DeploymentProgressing) if progressingCond != nil { if progressingCond.Status == corev1.ConditionFalse { return makeStatusFromDeploymentCondition(paasv1alpha2.HealthUnhealthy, progressingCond) @@ -67,7 +68,7 @@ func CheckDeploymentHealthStatus(deployment *appsv1.Deployment) *HealthStatus { } if deployment.Status.UpdatedReplicas == replicas { - availableCond := FindDeploymentStatusCondition(deployment.Status.Conditions, appsv1.DeploymentAvailable) + availableCond := kubeutil.FindDeploymentStatusCondition(deployment.Status.Conditions, appsv1.DeploymentAvailable) if availableCond != nil { if availableCond.Status != corev1.ConditionTrue { return makeStatusFromDeploymentCondition(paasv1alpha2.HealthUnhealthy, availableCond) @@ -130,19 +131,6 @@ func GetDeploymentDirectFailMessage( return "", errors.WithStack(ErrDeploymentStillProgressing) } -// FindDeploymentStatusCondition finds the conditionType in conditions. -func FindDeploymentStatusCondition( - conditions []appsv1.DeploymentCondition, - conditionType appsv1.DeploymentConditionType, -) *appsv1.DeploymentCondition { - for i := range conditions { - if conditions[i].Type == conditionType { - return &conditions[i] - } - } - return nil -} - // a shortcut for making a HealthStatus with given status and condition func makeStatusFromDeploymentCondition( phase paasv1alpha2.HealthPhase, diff --git a/operator/pkg/utils/kubestatus/deployment_test.go b/operator/pkg/health/deployment_test.go similarity index 98% rename from operator/pkg/utils/kubestatus/deployment_test.go rename to operator/pkg/health/deployment_test.go index 3785b453f0..4247c74a01 100644 --- a/operator/pkg/utils/kubestatus/deployment_test.go +++ b/operator/pkg/health/deployment_test.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubestatus +package health import ( "context" @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -var _ = Describe("Test kubestatus/deployment", func() { +var _ = Describe("Test deployment", func() { var deployment *appsv1.Deployment var builder *fake.ClientBuilder var scheme *runtime.Scheme diff --git a/operator/pkg/utils/kubetypes/kubetypes_suite_test.go b/operator/pkg/health/health_suite_test.go similarity index 91% rename from operator/pkg/utils/kubetypes/kubetypes_suite_test.go rename to operator/pkg/health/health_suite_test.go index d30fc7c002..3b82c7663c 100644 --- a/operator/pkg/utils/kubetypes/kubetypes_suite_test.go +++ b/operator/pkg/health/health_suite_test.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubetypes +package health_test import ( "testing" @@ -25,7 +25,7 @@ import ( . "github.com/onsi/gomega" ) -func TestKubetypes(t *testing.T) { +func TestHealth(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "utils/kubetypes Suite") + RunSpecs(t, "pkg/health Suite") } diff --git a/operator/pkg/utils/kubestatus/pod.go b/operator/pkg/health/pod.go similarity index 90% rename from operator/pkg/utils/kubestatus/pod.go rename to operator/pkg/health/pod.go index 720771c260..f9ba37fade 100644 --- a/operator/pkg/utils/kubestatus/pod.go +++ b/operator/pkg/health/pod.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubestatus +package health import ( "fmt" @@ -61,13 +61,13 @@ func CheckPodHealthStatus(pod *corev1.Pod) *HealthStatus { switch pod.Spec.RestartPolicy { case corev1.RestartPolicyAlways: // if pod is ready, k8s will set PodReady to True - condReady := FindPodStatusCondition(pod.Status.Conditions, corev1.PodReady) + condReady := findPodStatusCondition(pod.Status.Conditions, corev1.PodReady) if condReady != nil && condReady.Status == corev1.ConditionTrue { return healthyStatus } // if it's not ready, check to see if any container terminated, if so, it's unhealthy for _, ctr := range pod.Status.ContainerStatuses { - if failMessage := GetContainerFailMessage(ctr); failMessage != "" && failMessage != ContainerCreating { + if failMessage := getContainerFailMessage(ctr); failMessage != "" && failMessage != ContainerCreating { return unhealthyStatus.withMessage(failMessage) } } @@ -88,7 +88,7 @@ func CheckPodHealthStatus(pod *corev1.Pod) *HealthStatus { } // try to get fail message from ContainerStatuses for _, ctr := range pod.Status.ContainerStatuses { - if failMessage := GetContainerFailMessage(ctr); failMessage != "" { + if failMessage := getContainerFailMessage(ctr); failMessage != "" { return unhealthyStatus.withMessage(failMessage) } } @@ -97,7 +97,7 @@ func CheckPodHealthStatus(pod *corev1.Pod) *HealthStatus { case corev1.PodPending: // check if failed to start container for _, ctr := range pod.Status.ContainerStatuses { - if failMessage := GetContainerFailMessage(ctr); failMessage != "" && failMessage != ContainerCreating { + if failMessage := getContainerFailMessage(ctr); failMessage != "" && failMessage != ContainerCreating { return unhealthyStatus.withMessage(failMessage) } } @@ -113,8 +113,8 @@ func CheckPodHealthStatus(pod *corev1.Pod) *HealthStatus { } } -// GetContainerFailMessage 获取容器的失败信息 -func GetContainerFailMessage(ctr corev1.ContainerStatus) string { +// getContainerFailMessage 获取容器的失败信息 +func getContainerFailMessage(ctr corev1.ContainerStatus) string { if failMessage := getContainerFailedMessageFromState(ctr.LastTerminationState); failMessage != "" { return failMessage } @@ -146,8 +146,8 @@ func getContainerFailedMessageFromState(state corev1.ContainerState) string { return "" } -// FindPodStatusCondition finds the conditionType in conditions. -func FindPodStatusCondition( +// findPodStatusCondition finds the conditionType in conditions. +func findPodStatusCondition( conditions []corev1.PodCondition, conditionType corev1.PodConditionType, ) *corev1.PodCondition { diff --git a/operator/pkg/utils/kubestatus/pod_test.go b/operator/pkg/health/pod_test.go similarity index 97% rename from operator/pkg/utils/kubestatus/pod_test.go rename to operator/pkg/health/pod_test.go index 06d83633d9..4d3825a9dd 100644 --- a/operator/pkg/utils/kubestatus/pod_test.go +++ b/operator/pkg/health/pod_test.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubestatus +package health import ( . "github.com/onsi/ginkgo/v2" @@ -26,7 +26,7 @@ import ( paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" ) -var _ = Describe("Test kubestatus/pod", func() { +var _ = Describe("Test pod", func() { DescribeTable( "test CheckPodHealthStatus", func(pod *corev1.Pod, phase paasv1alpha2.HealthPhase, reason, message string) { @@ -125,7 +125,7 @@ var _ = Describe("Test kubestatus/pod", func() { DescribeTable( "test GetContainerFailMessage", func(ctr corev1.ContainerStatus, expected string) { - Expect(GetContainerFailMessage(ctr)).To(Equal(expected)) + Expect(getContainerFailMessage(ctr)).To(Equal(expected)) }, Entry( "terminated container with message", diff --git a/operator/pkg/utils/kubestatus/health.go b/operator/pkg/health/types.go similarity index 98% rename from operator/pkg/utils/kubestatus/health.go rename to operator/pkg/health/types.go index d7c3284e45..bed2c90066 100644 --- a/operator/pkg/utils/kubestatus/health.go +++ b/operator/pkg/health/types.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubestatus +package health import paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" diff --git a/operator/pkg/kubeutil/utils.go b/operator/pkg/kubeutil/client.go similarity index 74% rename from operator/pkg/kubeutil/utils.go rename to operator/pkg/kubeutil/client.go index c93d335960..25b6869908 100644 --- a/operator/pkg/kubeutil/utils.go +++ b/operator/pkg/kubeutil/client.go @@ -22,35 +22,15 @@ import ( "context" "github.com/pkg/errors" - "github.com/samber/lo" apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) -type nameAccessor interface { - GetName() string -} - -// FindExtraByName filter the `input` slice, take items whose "name(by GetName() -// method)" can't be found in base slice. -func FindExtraByName[T nameAccessor](input []T, base []T) []T { - // Make an index - names := make(map[string]struct{}) - for _, obj := range base { - names[obj.GetName()] = struct{}{} - } - - return lo.Filter(input, func(item T, _ int) bool { - _, ok := names[item.GetName()] - return !ok - }) -} - -// updateHandler should implement the object update policy -type updateHandler[T client.Object] func(ctx context.Context, cli client.Client, current T, want T) error +// UpdateHandler should implement the object update policy +type UpdateHandler[T client.Object] func(ctx context.Context, cli client.Client, current T, want T) error -// alwaysUpdate will always update the current object -func alwaysUpdate[T client.Object](ctx context.Context, cli client.Client, current T, want T) error { +// AlwaysUpdate will always update the current object +func AlwaysUpdate[T client.Object](ctx context.Context, cli client.Client, current T, want T) error { if err := cli.Update(ctx, want); err != nil { return errors.Wrapf( err, "failed to update %s(%s)", want.GetObjectKind().GroupVersionKind().String(), want.GetName(), @@ -71,7 +51,7 @@ func UpsertObject[T any, PT interface { ctx context.Context, cli client.Client, obj PT, - updateHandler updateHandler[PT], + updateHandler UpdateHandler[PT], ) error { exists := PT(new(T)) if err := cli.Get(ctx, client.ObjectKeyFromObject(obj), exists); err != nil { @@ -87,7 +67,7 @@ func UpsertObject[T any, PT interface { } } else { if updateHandler == nil { - updateHandler = alwaysUpdate[PT] + updateHandler = AlwaysUpdate[PT] } // 集群资源存在, 且命中更新策略时, 更新资源 return updateHandler(ctx, cli, exists, obj) diff --git a/operator/pkg/kubeutil/utils_test.go b/operator/pkg/kubeutil/client_test.go similarity index 91% rename from operator/pkg/kubeutil/utils_test.go rename to operator/pkg/kubeutil/client_test.go index 719cc856cc..ea6105ee0a 100644 --- a/operator/pkg/kubeutil/utils_test.go +++ b/operator/pkg/kubeutil/client_test.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubeutil +package kubeutil_test import ( "context" @@ -37,6 +37,7 @@ import ( "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/labels" "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names" procres "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/processes/resources" + "bk.tencent.com/paas-app-operator/pkg/kubeutil" ) var _ = Describe("Test utils", func() { @@ -109,7 +110,7 @@ var _ = Describe("Test utils", func() { webDeploy, _ := procres.BuildProcDeployment(bkapp, "web") want := []*appsv1.Deployment{webDeploy} - outdated := FindExtraByName(current, want) + outdated := kubeutil.FindExtraByName(current, want) Expect(len(outdated)).To(Equal(1)) Expect(outdated[0].Name).To(Equal("bkapp-sample--foo")) }) @@ -117,7 +118,7 @@ var _ = Describe("Test utils", func() { Context("test UpsertObject", func() { DescribeTable( "test update with different handleUpdate", - func(strategy updateHandler[*appsv1.Deployment], updated bool) { + func(strategy kubeutil.UpdateHandler[*appsv1.Deployment], updated bool) { a := fakeDeploy b := fakeDeploy.DeepCopy() got := appsv1.Deployment{} @@ -125,13 +126,13 @@ var _ = Describe("Test utils", func() { b.Labels["foo"] = "bar" cli := builder.WithObjects(bkapp, &a).Build() - Expect(UpsertObject(ctx, cli, b, strategy)).NotTo(HaveOccurred()) + Expect(kubeutil.UpsertObject(ctx, cli, b, strategy)).NotTo(HaveOccurred()) _ = cli.Get(ctx, client.ObjectKeyFromObject(&a), &got) Expect(equality.Semantic.DeepEqual(got.Labels, b.Labels)).To(Equal(updated)) }, - Entry("always update", alwaysUpdate[*appsv1.Deployment], true), + Entry("always update", kubeutil.AlwaysUpdate[*appsv1.Deployment], true), Entry( "always forbid update", func(ctx context.Context, cli client.Client, current *appsv1.Deployment, want *appsv1.Deployment) error { @@ -146,7 +147,7 @@ var _ = Describe("Test utils", func() { cli := builder.WithObjects(bkapp).Build() Expect(apierrors.IsNotFound(cli.Get(ctx, client.ObjectKeyFromObject(&fakeDeploy), &got))).To(BeTrue()) - Expect(UpsertObject(ctx, cli, &fakeDeploy, nil)).NotTo(HaveOccurred()) + Expect(kubeutil.UpsertObject(ctx, cli, &fakeDeploy, nil)).NotTo(HaveOccurred()) _ = cli.Get(ctx, client.ObjectKeyFromObject(&fakeDeploy), &got) diff --git a/operator/pkg/utils/kubestatus/kubestatus_suite_test.go b/operator/pkg/kubeutil/deployment.go similarity index 70% rename from operator/pkg/utils/kubestatus/kubestatus_suite_test.go rename to operator/pkg/kubeutil/deployment.go index 4202b878a3..7dfc4ef8ce 100644 --- a/operator/pkg/utils/kubestatus/kubestatus_suite_test.go +++ b/operator/pkg/kubeutil/deployment.go @@ -15,17 +15,21 @@ * We undertake not to change the open source license (MIT license) applicable * to the current version of the project delivered to anyone in the future. */ - -package kubestatus_test +package kubeutil import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" ) -func TestKubeStatus(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "utils/kubestatus Suite") +// FindDeploymentStatusCondition finds the conditionType in conditions. +func FindDeploymentStatusCondition( + conditions []appsv1.DeploymentCondition, + conditionType appsv1.DeploymentConditionType, +) *appsv1.DeploymentCondition { + for i := range conditions { + if conditions[i].Type == conditionType { + return &conditions[i] + } + } + return nil } diff --git a/operator/pkg/utils/kubetypes/common.go b/operator/pkg/kubeutil/misc.go similarity index 84% rename from operator/pkg/utils/kubetypes/common.go rename to operator/pkg/kubeutil/misc.go index 3b165ca96b..8609be5df1 100644 --- a/operator/pkg/utils/kubetypes/common.go +++ b/operator/pkg/kubeutil/misc.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubetypes +package kubeutil import ( "encoding/json" @@ -24,10 +24,30 @@ import ( "strings" "github.com/pkg/errors" + "github.com/samber/lo" "sigs.k8s.io/controller-runtime/pkg/client" ) +type nameAccessor interface { + GetName() string +} + +// FindExtraByName filter the `input` slice, take items whose "name(by GetName() +// method)" can't be found in base slice. +func FindExtraByName[T nameAccessor](input []T, base []T) []T { + // Make an index + names := make(map[string]struct{}) + for _, obj := range base { + names[obj.GetName()] = struct{}{} + } + + return lo.Filter(input, func(item T, _ int) bool { + _, ok := names[item.GetName()] + return !ok + }) +} + // GetJsonAnnotation gets the value of given key in the annotations, the data will be unmarshaled // to the given type provided by the type parameter. func GetJsonAnnotation[T any](obj client.Object, key string) (T, error) { diff --git a/operator/pkg/utils/kubetypes/common_test.go b/operator/pkg/kubeutil/misc_test.go similarity index 99% rename from operator/pkg/utils/kubetypes/common_test.go rename to operator/pkg/kubeutil/misc_test.go index 85af6e10fd..3b91052140 100644 --- a/operator/pkg/utils/kubetypes/common_test.go +++ b/operator/pkg/kubeutil/misc_test.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package kubetypes +package kubeutil import ( . "github.com/onsi/ginkgo/v2" diff --git a/operator/pkg/client/client.go b/operator/pkg/kubeutil/traced_client.go similarity index 84% rename from operator/pkg/client/client.go rename to operator/pkg/kubeutil/traced_client.go index 0405978068..01ec00ed77 100644 --- a/operator/pkg/client/client.go +++ b/operator/pkg/kubeutil/traced_client.go @@ -16,7 +16,7 @@ * to the current version of the project delivered to anyone in the future. */ -package client +package kubeutil import ( "context" @@ -29,54 +29,54 @@ import ( // stackTraceClient wrap client.Client error with stack type stackTraceClient struct { - cli client.Client + client client.Client } var _ client.Client = &stackTraceClient{} -// New stackTraceClient -func New(cli client.Client) client.Client { - return &stackTraceClient{cli: cli} +// NewTracedClient creates a new client object with stack trace enabled +func NewTracedClient(client client.Client) client.Client { + return &stackTraceClient{client: client} } // Scheme returns the scheme this client is using. func (c *stackTraceClient) Scheme() *runtime.Scheme { - return c.cli.Scheme() + return c.client.Scheme() } // RESTMapper returns the rest this client is using. func (c *stackTraceClient) RESTMapper() apimeta.RESTMapper { - return c.cli.RESTMapper() + return c.client.RESTMapper() } // Get retrieves an obj for the given object key from the Kubernetes Cluster. // obj must be a struct pointer so that obj can be updated with the response // returned by the Server. func (c *stackTraceClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { - return errors.WithStack(c.cli.Get(ctx, key, obj)) + return errors.WithStack(c.client.Get(ctx, key, obj)) } // List retrieves list of objects for a given namespace and list options. On a // successful call, Items field in the list will be populated with the // result returned from the server. func (c *stackTraceClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - return errors.WithStack(c.cli.List(ctx, list, opts...)) + return errors.WithStack(c.client.List(ctx, list, opts...)) } // Create saves the object obj in the Kubernetes cluster. func (c *stackTraceClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { - return errors.WithStack(c.cli.Create(ctx, obj, opts...)) + return errors.WithStack(c.client.Create(ctx, obj, opts...)) } // Delete deletes the given obj from Kubernetes cluster. func (c *stackTraceClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { - return errors.WithStack(c.cli.Delete(ctx, obj, opts...)) + return errors.WithStack(c.client.Delete(ctx, obj, opts...)) } // Update updates the given obj in the Kubernetes cluster. obj must be a // struct pointer so that obj can be updated with the content returned by the Server. func (c *stackTraceClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { - return errors.WithStack(c.cli.Update(ctx, obj, opts...)) + return errors.WithStack(c.client.Update(ctx, obj, opts...)) } // Patch patches the given obj in the Kubernetes cluster. obj must be a @@ -84,19 +84,19 @@ func (c *stackTraceClient) Update(ctx context.Context, obj client.Object, opts . func (c *stackTraceClient) Patch( ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption, ) error { - return errors.WithStack(c.cli.Patch(ctx, obj, patch, opts...)) + return errors.WithStack(c.client.Patch(ctx, obj, patch, opts...)) } // DeleteAllOf deletes all objects of the given type matching the given options. func (c *stackTraceClient) DeleteAllOf( ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption, ) error { - return errors.WithStack(c.cli.DeleteAllOf(ctx, obj, opts...)) + return errors.WithStack(c.client.DeleteAllOf(ctx, obj, opts...)) } // StatusClient knows how to create a client which can update status subresource for kubernetes objects. func (c *stackTraceClient) Status() client.StatusWriter { - return &stackTraceStatusWriter{sw: c.cli.Status()} + return &stackTraceStatusWriter{sw: c.client.Status()} } // stackTraceStatusWriter wrap client.StatusWriter error with stack diff --git a/operator/pkg/utils/kubestatus/gpa.go b/operator/pkg/utils/kubestatus/gpa.go deleted file mode 100644 index b19a204dbb..0000000000 --- a/operator/pkg/utils/kubestatus/gpa.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - * TencentBlueKing is pleased to support the open source community by making - * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. - * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. - * Licensed under the MIT License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://opensource.org/licenses/MIT - * - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. See the License for the specific language governing permissions and - * limitations under the License. - * - * We undertake not to change the open source license (MIT license) applicable - * to the current version of the project delivered to anyone in the future. - */ - -package kubestatus - -import ( - corev1 "k8s.io/api/core/v1" - - paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" - - autoscaling "github.com/Tencent/bk-bcs/bcs-runtime/bcs-k8s/bcs-component/bcs-general-pod-autoscaler/pkg/apis/autoscaling/v1alpha1" -) - -// GenGPAHealthStatus check if the GPA is healthy -// For a deployment: -// -// healthy means the GPA is available, ready to scale workloads with policy. -// unhealthy means the GPA is failed when reconciled. -func GenGPAHealthStatus(gpa *autoscaling.GeneralPodAutoscaler) *HealthStatus { - for _, condition := range gpa.Status.Conditions { - if condition.Status == corev1.ConditionFalse { - return &HealthStatus{ - Phase: paasv1alpha2.HealthUnhealthy, - Reason: condition.Reason, - Message: condition.Message, - } - } - } - - return &HealthStatus{ - Phase: paasv1alpha2.HealthHealthy, - Reason: paasv1alpha2.AutoscalingAvailable, - Message: "", - } -} diff --git a/operator/pkg/utils/kubestatus/gpa_test.go b/operator/pkg/utils/kubestatus/gpa_test.go deleted file mode 100644 index 77ac60567e..0000000000 --- a/operator/pkg/utils/kubestatus/gpa_test.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - * TencentBlueKing is pleased to support the open source community by making - * 蓝鲸智云 - PaaS 平台 (BlueKing - PaaS System) available. - * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. - * Licensed under the MIT License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://opensource.org/licenses/MIT - * - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. See the License for the specific language governing permissions and - * limitations under the License. - * - * We undertake not to change the open source license (MIT license) applicable - * to the current version of the project delivered to anyone in the future. - */ - -package kubestatus - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" - - autoscaling "github.com/Tencent/bk-bcs/bcs-runtime/bcs-k8s/bcs-component/bcs-general-pod-autoscaler/pkg/apis/autoscaling/v1alpha1" -) - -var _ = Describe("Test kubestatus/gpa", func() { - var gpa *autoscaling.GeneralPodAutoscaler - var builder *fake.ClientBuilder - var scheme *runtime.Scheme - - BeforeEach(func() { - gpa = &autoscaling.GeneralPodAutoscaler{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "autoscaling.tkex.tencent.com/v1alpha1", - Kind: "GeneralPodAutoscaler", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "default-web-gpa", - }, - Spec: autoscaling.GeneralPodAutoscalerSpec{ - MinReplicas: lo.ToPtr(int32(2)), - MaxReplicas: int32(5), - ScaleTargetRef: autoscaling.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "default-web", - }, - AutoScalingDrivenMode: autoscaling.AutoScalingDrivenMode{ - MetricMode: &autoscaling.MetricMode{ - Metrics: []autoscaling.MetricSpec{}, - }, - }, - }, - } - - builder = fake.NewClientBuilder() - scheme = runtime.NewScheme() - Expect(autoscaling.AddToScheme(scheme)).NotTo(HaveOccurred()) - builder.WithScheme(scheme) - }) - - Context("test GenGPAHealthStatus", func() { - It("test healthy", func() { - gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{ - { - Type: autoscaling.AbleToScale, - Status: v1.ConditionTrue, - Reason: "ReadyForNewScale", - Message: "recommended size matches current size.", - }, - { - Type: autoscaling.ScalingActive, - Status: v1.ConditionTrue, - Reason: "ValidMetricFound", - Message: "the GPA was able to successfully calculate a replica count from.", - }, - } - status := GenGPAHealthStatus(gpa) - Expect(status.Phase).To(Equal(paasv1alpha2.HealthHealthy)) - Expect(status.Reason).To(Equal("AutoscalingAvailable")) - }) - It("test ScalingActive unhealthy", func() { - gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{ - { - Type: autoscaling.AbleToScale, - Status: v1.ConditionTrue, - Reason: "ReadyForNewScale", - Message: "recommended size matches current size.", - }, - { - Type: autoscaling.ScalingActive, - Status: v1.ConditionFalse, - Reason: "FailedGetResourceMetric", - Message: "the GPA was unable to compute the replica count: unable to get metrics for resource cpu.", - }, - } - status := GenGPAHealthStatus(gpa) - Expect(status.Phase).To(Equal(paasv1alpha2.HealthUnhealthy)) - Expect(status.Reason).To(Equal("FailedGetResourceMetric")) - }) - It("test AbleToScale unhealthy", func() { - gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{ - { - Type: autoscaling.AbleToScale, - Status: v1.ConditionFalse, - Reason: "FailedGetScale", - }, - { - Type: autoscaling.ScalingActive, - Status: v1.ConditionTrue, - Reason: "ValidMetricFound", - Message: "the GPA was able to successfully calculate a replica count from.", - }, - } - status := GenGPAHealthStatus(gpa) - Expect(status.Phase).To(Equal(paasv1alpha2.HealthUnhealthy)) - Expect(status.Reason).To(Equal("FailedGetScale")) - }) - }) -})