Finish re-organize operator module

operator/api/v1alpha1/bkapp_conversion.go

@@ -8,7 +8,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/conversion"
 
 	paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubetypes"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 )
 
 var _ conversion.Convertible = &BkApp{}
@@ -61,10 +61,10 @@ func (src *BkApp) ConvertTo(dstRaw conversion.Hub) error {
 	}
 
 	// Save legacy proc image and resource configs to annotations
-	if err := kubetypes.SetJsonAnnotation(dst, LegacyProcImageAnnoKey, legacyProcImageConfig); err != nil {
+	if err := kubeutil.SetJsonAnnotation(dst, LegacyProcImageAnnoKey, legacyProcImageConfig); err != nil {
 		return err
 	}
-	if err := kubetypes.SetJsonAnnotation(dst, LegacyProcResAnnoKey, legacyProcResConfig); err != nil {
+	if err := kubeutil.SetJsonAnnotation(dst, LegacyProcResAnnoKey, legacyProcResConfig); err != nil {
 		return err
 	}
 
@@ -145,8 +145,8 @@ func (dst *BkApp) ConvertFrom(srcRaw conversion.Hub) error {
 	)
 
 	// Handle Processes field
-	legacyProcImageConfig, _ := kubetypes.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](src, LegacyProcImageAnnoKey)
-	legacyProcResConfig, _ := kubetypes.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](src, LegacyProcResAnnoKey)
+	legacyProcImageConfig, _ := kubeutil.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](src, LegacyProcImageAnnoKey)
+	legacyProcResConfig, _ := kubeutil.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](src, LegacyProcResAnnoKey)
 	for _, proc := range src.Spec.Processes {
 		dstProc := Process{
 			Name:         proc.Name,

operator/api/v1alpha2/bkapp_webhook.go

@@ -38,7 +38,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"bk.tencent.com/paas-app-operator/pkg/config"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubetypes"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 	"bk.tencent.com/paas-app-operator/pkg/utils/quota"
 	"bk.tencent.com/paas-app-operator/pkg/utils/stringx"
 )
@@ -184,7 +184,7 @@ func (r *BkApp) validateAnnotations() *field.Error {
 	}
 
 	// 通过注解配置的进程资源信息，也需要校验是否合法
-	legacyProcResConfig, err := kubetypes.GetJsonAnnotation[LegacyProcConfig](
+	legacyProcResConfig, err := kubeutil.GetJsonAnnotation[LegacyProcConfig](
 		r, LegacyProcResAnnoKey,
 	)
 	// 获取进程中的资源配额注解成功，才需要进行检查

operator/api/v1alpha2/bkapp_webhook_test.go

@@ -32,7 +32,7 @@ import (
 	paasv1alpha1 "bk.tencent.com/paas-app-operator/api/v1alpha1"
 	paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2"
 	"bk.tencent.com/paas-app-operator/pkg/config"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubetypes"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 	"bk.tencent.com/paas-app-operator/pkg/utils/stringx"
 )
 
@@ -647,31 +647,31 @@ var _ = Describe("test webhook.Validator", func() {
 		It("Normal", func() {
 			legacyProcResConfig := make(paasv1alpha2.LegacyProcConfig)
 			legacyProcResConfig["web"] = map[string]string{"cpu": "2", "memory": "2G"}
-			_ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig)
+			_ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig)
 
 			err := bkapp.ValidateCreate()
 			Expect(err).To(BeNil())
 		})
 		It("Invalid unset", func() {
 			legacyProcResConfig := make(paasv1alpha2.LegacyProcConfig)
 			legacyProcResConfig["web"] = map[string]string{"cpu": "", "memory": "2G"}
-			_ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig)
+			_ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig)
 
 			err := bkapp.ValidateCreate()
 			Expect(err).NotTo(BeNil())
 		})
 		It("Invalid exceed cpu max limit", func() {
 			legacyProcResConfig := make(paasv1alpha2.LegacyProcConfig)
 			legacyProcResConfig["web"] = map[string]string{"cpu": "6", "memory": "2G"}
-			_ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig)
+			_ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig)
 
 			err := bkapp.ValidateCreate()
 			Expect(err).NotTo(BeNil())
 		})
 		It("Invalid exceed memory max limit", func() {
 			legacyProcResConfig := make(paasv1alpha2.LegacyProcConfig)
 			legacyProcResConfig["web"] = map[string]string{"cpu": "2", "memory": "8G"}
-			_ = kubetypes.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig)
+			_ = kubeutil.SetJsonAnnotation(bkapp, paasv1alpha2.LegacyProcResAnnoKey, legacyProcResConfig)
 
 			err := bkapp.ValidateCreate()
 			Expect(err).NotTo(BeNil())

operator/api/v1alpha2/compat.go

@@ -24,7 +24,7 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubetypes"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 )
 
 // ProcImageGetter help getting container image from bkapp
@@ -44,7 +44,7 @@ func NewProcImageGetter(bkapp *BkApp) *ProcImageGetter {
 // - return: <image>, <imagePullPolicy>, <error>
 func (r *ProcImageGetter) Get(name string) (string, corev1.PullPolicy, error) {
 	// Legacy API version: read image configs from annotations
-	legacyProcImageConfig, _ := kubetypes.GetJsonAnnotation[LegacyProcConfig](
+	legacyProcImageConfig, _ := kubeutil.GetJsonAnnotation[LegacyProcConfig](
 		r.bkapp,
 		LegacyProcImageAnnoKey,
 	)

operator/api/v1alpha2/compat_test.go

@@ -25,7 +25,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubetypes"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 )
 
 var _ = Describe("test compat", func() {
@@ -67,7 +67,7 @@ var _ = Describe("test compat", func() {
 
 	Context("Test ProcImageGetter", func() {
 		It("Get Legacy", func() {
-			_ = kubetypes.SetJsonAnnotation(
+			_ = kubeutil.SetJsonAnnotation(
 				bkapp, paasv1alpha2.LegacyProcImageAnnoKey, paasv1alpha2.LegacyProcConfig{
 					"web": {"image": "busybox:1.0.0", "policy": "Never"},
 				},

operator/controllers/bkapp_controller_test.go

@@ -42,9 +42,9 @@ import (
 
 	paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2"
 	"bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 	"bk.tencent.com/paas-app-operator/pkg/platform/external"
 	"bk.tencent.com/paas-app-operator/pkg/testing"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubestatus"
 )
 
 var _ = Describe("", func() {
@@ -222,7 +222,7 @@ var _ = Describe("", func() {
 				createdDeployment.Status.ObservedGeneration = createdDeployment.Generation
 				createdDeployment.Status.Replicas = *createdDeployment.Spec.Replicas
 				createdDeployment.Status.UpdatedReplicas = *createdDeployment.Spec.Replicas
-				cond := kubestatus.FindDeploymentStatusCondition(
+				cond := kubeutil.FindDeploymentStatusCondition(
 					createdDeployment.Status.Conditions,
 					appsv1.DeploymentAvailable,
 				)

operator/main.go

@@ -48,9 +48,9 @@ import (
 	paasv1alpha1 "bk.tencent.com/paas-app-operator/api/v1alpha1"
 	paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2"
 	"bk.tencent.com/paas-app-operator/controllers"
-	"bk.tencent.com/paas-app-operator/pkg/client"
 	"bk.tencent.com/paas-app-operator/pkg/config"
 	dgmingress "bk.tencent.com/paas-app-operator/pkg/controllers/dgroupmapping/ingress"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 	"bk.tencent.com/paas-app-operator/pkg/platform/external"
 
 	//+kubebuilder:scaffold:imports
@@ -136,7 +136,7 @@ func main() {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
-	mgrCli := client.New(mgr.GetClient())
+	mgrCli := kubeutil.NewTracedClient(mgr.GetClient())
 	mgrScheme := mgr.GetScheme()
 
 	bkappMgrOpts := genGroupKindMgrOpts(paasv1alpha1.GroupKindBkApp, projConf.Controller)

operator/pkg/controllers/bkapp/autoscaling/autoscaling.go

@@ -32,7 +32,6 @@ import (
 	"bk.tencent.com/paas-app-operator/pkg/config"
 	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 	"bk.tencent.com/paas-app-operator/pkg/metrics"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubestatus"
 
 	autoscaling "github.com/Tencent/bk-bcs/bcs-runtime/bcs-k8s/bcs-component/bcs-general-pod-autoscaler/pkg/apis/autoscaling/v1alpha1"
 )
@@ -140,7 +139,7 @@ func (r *AutoscalingReconciler) updateCondition(ctx context.Context, bkapp *paas
 	}
 
 	for _, gpa := range current {
-		healthStatus := kubestatus.GenGPAHealthStatus(gpa)
+		healthStatus := GenGPAHealthStatus(gpa)
 		if healthStatus.Phase != paasv1alpha2.HealthHealthy {
 			apimeta.SetStatusCondition(&bkapp.Status.Conditions, metav1.Condition{
 				Type:               paasv1alpha2.AutoscalingAvailable,

operator/pkg/controllers/bkapp/autoscaling/gpa.go

@@ -27,6 +27,7 @@ import (
 	paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2"
 	"bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names"
 	"bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/envs"
+	"bk.tencent.com/paas-app-operator/pkg/health"
 
 	autoscaling "github.com/Tencent/bk-bcs/bcs-runtime/bcs-k8s/bcs-component/bcs-general-pod-autoscaler/pkg/apis/autoscaling/v1alpha1"
 )
@@ -99,3 +100,26 @@ func buildMetricSpecs(policy paasv1alpha2.ScalingPolicy) (metrics []autoscaling.
 
 	return metrics
 }
+
+// GenGPAHealthStatus check if the GPA is healthy
+// For a deployment:
+//
+//	healthy means the GPA is available, ready to scale workloads with policy.
+//	unhealthy means the GPA is failed when reconciled.
+func GenGPAHealthStatus(gpa *autoscaling.GeneralPodAutoscaler) *health.HealthStatus {
+	for _, condition := range gpa.Status.Conditions {
+		if condition.Status == v1.ConditionFalse {
+			return &health.HealthStatus{
+				Phase:   paasv1alpha2.HealthUnhealthy,
+				Reason:  condition.Reason,
+				Message: condition.Message,
+			}
+		}
+	}
+
+	return &health.HealthStatus{
+		Phase:   paasv1alpha2.HealthHealthy,
+		Reason:  paasv1alpha2.AutoscalingAvailable,
+		Message: "",
+	}
+}

operator/pkg/controllers/bkapp/autoscaling/gpa_test.go

@@ -122,3 +122,99 @@ var _ = Describe("GPAResources", func() {
 		})
 	})
 })
+
+var _ = Describe("Test gpa", func() {
+	var gpa *autoscaling.GeneralPodAutoscaler
+	var builder *fake.ClientBuilder
+	var scheme *runtime.Scheme
+
+	BeforeEach(func() {
+		gpa = &autoscaling.GeneralPodAutoscaler{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: "autoscaling.tkex.tencent.com/v1alpha1",
+				Kind:       "GeneralPodAutoscaler",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "default-web-gpa",
+			},
+			Spec: autoscaling.GeneralPodAutoscalerSpec{
+				MinReplicas: lo.ToPtr(int32(2)),
+				MaxReplicas: int32(5),
+				ScaleTargetRef: autoscaling.CrossVersionObjectReference{
+					APIVersion: "apps/v1",
+					Kind:       "Deployment",
+					Name:       "default-web",
+				},
+				AutoScalingDrivenMode: autoscaling.AutoScalingDrivenMode{
+					MetricMode: &autoscaling.MetricMode{
+						Metrics: []autoscaling.MetricSpec{},
+					},
+				},
+			},
+		}
+
+		builder = fake.NewClientBuilder()
+		scheme = runtime.NewScheme()
+		Expect(autoscaling.AddToScheme(scheme)).NotTo(HaveOccurred())
+		builder.WithScheme(scheme)
+	})
+
+	Context("test GenGPAHealthStatus", func() {
+		It("test healthy", func() {
+			gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{
+				{
+					Type:    autoscaling.AbleToScale,
+					Status:  v1.ConditionTrue,
+					Reason:  "ReadyForNewScale",
+					Message: "recommended size matches current size.",
+				},
+				{
+					Type:    autoscaling.ScalingActive,
+					Status:  v1.ConditionTrue,
+					Reason:  "ValidMetricFound",
+					Message: "the GPA was able to successfully calculate a replica count from.",
+				},
+			}
+			status := GenGPAHealthStatus(gpa)
+			Expect(status.Phase).To(Equal(paasv1alpha2.HealthHealthy))
+			Expect(status.Reason).To(Equal("AutoscalingAvailable"))
+		})
+		It("test ScalingActive unhealthy", func() {
+			gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{
+				{
+					Type:    autoscaling.AbleToScale,
+					Status:  v1.ConditionTrue,
+					Reason:  "ReadyForNewScale",
+					Message: "recommended size matches current size.",
+				},
+				{
+					Type:    autoscaling.ScalingActive,
+					Status:  v1.ConditionFalse,
+					Reason:  "FailedGetResourceMetric",
+					Message: "the GPA was unable to compute the replica count: unable to get metrics for resource cpu.",
+				},
+			}
+			status := GenGPAHealthStatus(gpa)
+			Expect(status.Phase).To(Equal(paasv1alpha2.HealthUnhealthy))
+			Expect(status.Reason).To(Equal("FailedGetResourceMetric"))
+		})
+		It("test AbleToScale unhealthy", func() {
+			gpa.Status.Conditions = []autoscaling.GeneralPodAutoscalerCondition{
+				{
+					Type:   autoscaling.AbleToScale,
+					Status: v1.ConditionFalse,
+					Reason: "FailedGetScale",
+				},
+				{
+					Type:    autoscaling.ScalingActive,
+					Status:  v1.ConditionTrue,
+					Reason:  "ValidMetricFound",
+					Message: "the GPA was able to successfully calculate a replica count from.",
+				},
+			}
+			status := GenGPAHealthStatus(gpa)
+			Expect(status.Phase).To(Equal(paasv1alpha2.HealthUnhealthy))
+			Expect(status.Reason).To(Equal("FailedGetScale"))
+		})
+	})
+})

operator/pkg/controllers/bkapp/envs/env_overlay.go

@@ -24,7 +24,7 @@ import (
 
 	paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2"
 	"bk.tencent.com/paas-app-operator/pkg/config"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubetypes"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 	"bk.tencent.com/paas-app-operator/pkg/utils/quota"
 )
 
@@ -217,7 +217,7 @@ func (r *ProcResourcesGetter) Default() corev1.ResourceRequirements {
 // - return: <resources requirements>, <error>
 func (r *ProcResourcesGetter) GetByProc(name string) (result corev1.ResourceRequirements, err error) {
 	// Legacy version: try to read resources configs from legacy annotation
-	legacyProcResourcesConfig, _ := kubetypes.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](
+	legacyProcResourcesConfig, _ := kubeutil.GetJsonAnnotation[paasv1alpha2.LegacyProcConfig](
 		r.bkapp,
 		paasv1alpha2.LegacyProcResAnnoKey,
 	)

operator/pkg/controllers/bkapp/envs/env_overlay_test.go

@@ -30,7 +30,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubetypes"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 )
 
 var _ = Describe("Environment overlay related functions", func() {
@@ -236,7 +236,7 @@ var _ = Describe("Environment overlay related functions", func() {
 		})
 
 		It("Get Legacy", func() {
-			_ = kubetypes.SetJsonAnnotation(
+			_ = kubeutil.SetJsonAnnotation(
 				bkapp, paasv1alpha2.LegacyProcResAnnoKey, paasv1alpha2.LegacyProcConfig{
 					"web": {"cpu": "2", "memory": "2Gi"},
 				},

operator/pkg/controllers/bkapp/hooks/hooks.go

@@ -39,8 +39,8 @@ import (
 	"bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/labels"
 	"bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names"
 	hookres "bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/hooks/resources"
+	"bk.tencent.com/paas-app-operator/pkg/health"
 	"bk.tencent.com/paas-app-operator/pkg/metrics"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubestatus"
 )
 
 // HookPodsHistoryLimit 最大保留的 Hook Pod 数量（单种类型）
@@ -159,7 +159,7 @@ func (r *HookReconciler) getCurrentState(ctx context.Context, bkapp *paasv1alpha
 		})
 	}
 
-	healthStatus := kubestatus.CheckPodHealthStatus(&pod)
+	healthStatus := health.CheckPodHealthStatus(&pod)
 	return hookres.HookInstance{
 		Pod: &pod,
 		Status: &paasv1alpha2.HookStatus{

operator/pkg/controllers/bkapp/hooks/resources/hooks.go

@@ -32,7 +32,7 @@ import (
 	"bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/labels"
 	"bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/common/names"
 	"bk.tencent.com/paas-app-operator/pkg/controllers/bkapp/envs"
-	"bk.tencent.com/paas-app-operator/pkg/utils/kubetypes"
+	"bk.tencent.com/paas-app-operator/pkg/kubeutil"
 )
 
 const (
@@ -146,8 +146,8 @@ func BuildPreReleaseHook(bkapp *paasv1alpha2.BkApp, status *paasv1alpha2.HookSta
 				Containers: []corev1.Container{
 					{
 						Image:           image,
-						Command:         kubetypes.ReplaceCommandEnvVariables(command),
-						Args:            kubetypes.ReplaceCommandEnvVariables(args),
+						Command:         kubeutil.ReplaceCommandEnvVariables(command),
+						Args:            kubeutil.ReplaceCommandEnvVariables(args),
 						Env:             common.GetAppEnvs(bkapp),
 						Name:            "hook",
 						ImagePullPolicy: pullPolicy,