chore: Upgrade to use autoscaling/v2 for HPA (#403)

The 'autoscaling/v2beta2' API for HPA was deprecated in Kubernetes version 1.23
and is no longer available in Kubernetes version 1.26.

Changes:
- Update Go code to use 'autoscaling/v2'
- Use setup-envtest to force K8s v1.23+ for unit tests to ensure
  the 'autoscaling/v2' API is available
- Update FVT to use Kubernetes version 1.26+

Closes #402

Signed-off-by: Rafael Vasquez <raf.vasquez@ibm.com>
Signed-off-by: Christian Kadner <ckadner@us.ibm.com>
Co-authored-by: Christian Kadner <ckadner@us.ibm.com>
(cherry picked from commit ef26b75)

.github/workflows/fvt.yml

@@ -45,12 +45,12 @@ jobs:
           go-version: '1.19'
 
       - name: Start Minikube
-        uses: medyagh/setup-minikube@v0.0.11
+        uses: medyagh/setup-minikube@v0.0.13
         id: minikube
         with:
-          minikube-version: 1.27.1
+          minikube-version: 1.31.0
           container-runtime: docker
-          kubernetes-version: v1.25.2
+          kubernetes-version: v1.26.1
           cpus: max
           memory: max
 

Dockerfile.develop

@@ -25,7 +25,7 @@ ARG TARGETARCH
 
 ARG OPENSHIFT_VERSION=4.9
 ARG KUSTOMIZE_VERSION=4.5.2
-ARG KUBEBUILDER_VERSION=v3.3.0
+ARG KUBEBUILDER_VERSION=v3.11.0
 ARG CONTROLLER_GEN_VERSION=v0.11.4
 
 ENV PATH=/usr/local/go/bin:$PATH:/usr/local/kubebuilder/bin:
@@ -105,6 +105,12 @@ RUN true \
     && ginkgo version \
     && true
 
+# Use setup-envtest for kubebuilder to use K8s version 1.23+ for autoscaling/v2 (HPA)
+RUN true \
+    && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest \
+    && setup-envtest use 1.23 \
+    && true
+
 # For GitHub Action 'lint', work around error "detected dubious ownership in repository at '/workspace'"
 RUN git config --system --add safe.directory /workspace
 

Makefile

@@ -25,11 +25,18 @@ ENGINE ?= "docker"
 
 # Image URL to use all building/pushing image targets
 IMG ?= kserve/modelmesh-controller:latest
+
 # Namespace to deploy model-serve into
 NAMESPACE ?= "model-serving"
 
 CONTROLLER_GEN_VERSION ?= "v0.11.4"
 
+# Kubernetes version needs to be 1.23 or newer for autoscaling/v2 (HPA)
+# https://github.com/kubernetes-sigs/controller-runtime/tree/main/tools/setup-envtest
+# install with `go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest`
+# find available versions by running `setup-envtest list`
+KUBERNETES_VERSION ?= 1.23
+
 CRD_OPTIONS ?= "crd:maxDescLen=0"
 
 # Model Mesh gRPC API Proto Generation
@@ -46,17 +53,18 @@ endif
 .PHONY: all
 all: manager
 
-# Run unit tests
+# Run unit tests, requires kubebuilder, etcd, kube-apiserver, envtest
 .PHONY: test
 test:
+	KUBEBUILDER_ASSETS="$$(setup-envtest use $(KUBERNETES_VERSION) -p path)" \
+	KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT=120s \
 	go test -coverprofile cover.out `go list ./... | grep -v fvt`
 
 # Run fvt tests. This requires an etcd, kubernetes connection, and model serving installation. Ginkgo CLI is used to run them in parallel
 .PHONY: fvt
 fvt:
 	ginkgo -v -procs=2 --fail-fast fvt/predictor fvt/scaleToZero fvt/storage fvt/hpa --timeout=50m
 
-
 # Command to regenerate the grpc go files from the proto files
 .PHONY: fvt-protoc
 fvt-protoc:

controllers/hpa/hpa_reconciler.go

@@ -22,7 +22,7 @@ import (
 	"github.com/kserve/kserve/pkg/constants"
 	"github.com/kserve/kserve/pkg/utils"
 	mmcontstant "github.com/kserve/modelmesh-serving/pkg/constants"
-	v2beta2 "k8s.io/api/autoscaling/v2beta2"
+	hpav2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierr "k8s.io/apimachinery/pkg/api/errors"
@@ -39,7 +39,7 @@ var log = logf.Log.WithName("HPAReconciler")
 type HPAReconciler struct {
 	client client.Client
 	scheme *runtime.Scheme
-	HPA    *v2beta2.HorizontalPodAutoscaler
+	HPA    *hpav2.HorizontalPodAutoscaler
 }
 
 func NewHPAReconciler(client client.Client,
@@ -51,8 +51,8 @@ func NewHPAReconciler(client client.Client,
 	}
 }
 
-func getHPAMetrics(metadata metav1.ObjectMeta) []v2beta2.MetricSpec {
-	var metrics []v2beta2.MetricSpec
+func getHPAMetrics(metadata metav1.ObjectMeta) []hpav2.MetricSpec {
+	var metrics []hpav2.MetricSpec
 	var utilization int32 = constants.DefaultCPUUtilization
 
 	annotations := metadata.Annotations
@@ -67,14 +67,14 @@ func getHPAMetrics(metadata metav1.ObjectMeta) []v2beta2.MetricSpec {
 		resourceName = corev1.ResourceName(value)
 	}
 
-	metricTarget := v2beta2.MetricTarget{
+	metricTarget := hpav2.MetricTarget{
 		Type:               "Utilization",
 		AverageUtilization: &utilization,
 	}
 
-	ms := v2beta2.MetricSpec{
-		Type: v2beta2.ResourceMetricSourceType,
-		Resource: &v2beta2.ResourceMetricSource{
+	ms := hpav2.MetricSpec{
+		Type: hpav2.ResourceMetricSourceType,
+		Resource: &hpav2.ResourceMetricSource{
 			Name:   resourceName,
 			Target: metricTarget,
 		},
@@ -84,7 +84,7 @@ func getHPAMetrics(metadata metav1.ObjectMeta) []v2beta2.MetricSpec {
 	return metrics
 }
 
-func createHPA(runtimeMeta metav1.ObjectMeta, mmDeploymentName string, mmNamespace string) *v2beta2.HorizontalPodAutoscaler {
+func createHPA(runtimeMeta metav1.ObjectMeta, mmDeploymentName string, mmNamespace string) *hpav2.HorizontalPodAutoscaler {
 	minReplicas := int32(constants.DefaultMinReplicas)
 	maxReplicas := int32(constants.DefaultMinReplicas)
 	annotations := runtimeMeta.Annotations
@@ -115,10 +115,10 @@ func createHPA(runtimeMeta metav1.ObjectMeta, mmDeploymentName string, mmNamespa
 		Annotations: runtimeMeta.Annotations,
 	}
 
-	hpa := &v2beta2.HorizontalPodAutoscaler{
+	hpa := &hpav2.HorizontalPodAutoscaler{
 		ObjectMeta: hpaObjectMeta,
-		Spec: v2beta2.HorizontalPodAutoscalerSpec{
-			ScaleTargetRef: v2beta2.CrossVersionObjectReference{
+		Spec: hpav2.HorizontalPodAutoscalerSpec{
+			ScaleTargetRef: hpav2.CrossVersionObjectReference{
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
 				Name:       hpaObjectMeta.Name,
@@ -127,15 +127,15 @@ func createHPA(runtimeMeta metav1.ObjectMeta, mmDeploymentName string, mmNamespa
 			MaxReplicas: maxReplicas,
 
 			Metrics:  metrics,
-			Behavior: &v2beta2.HorizontalPodAutoscalerBehavior{},
+			Behavior: &hpav2.HorizontalPodAutoscalerBehavior{},
 		},
 	}
 	return hpa
 }
 
 // checkHPAExist checks if the hpa exists?
-func (r *HPAReconciler) checkHPAExist(client client.Client) (constants.CheckResultType, *v2beta2.HorizontalPodAutoscaler, error) {
-	existingHPA := &v2beta2.HorizontalPodAutoscaler{}
+func (r *HPAReconciler) checkHPAExist(client client.Client) (constants.CheckResultType, *hpav2.HorizontalPodAutoscaler, error) {
+	existingHPA := &hpav2.HorizontalPodAutoscaler{}
 	err := client.Get(context.TODO(), types.NamespacedName{
 		Namespace: r.HPA.ObjectMeta.Namespace,
 		Name:      r.HPA.ObjectMeta.Name,
@@ -154,14 +154,14 @@ func (r *HPAReconciler) checkHPAExist(client client.Client) (constants.CheckResu
 	return constants.CheckResultUpdate, existingHPA, nil
 }
 
-func semanticHPAEquals(desired, existing *v2beta2.HorizontalPodAutoscaler) bool {
+func semanticHPAEquals(desired, existing *hpav2.HorizontalPodAutoscaler) bool {
 	return equality.Semantic.DeepEqual(desired.Spec.Metrics, existing.Spec.Metrics) &&
 		equality.Semantic.DeepEqual(desired.Spec.MaxReplicas, existing.Spec.MaxReplicas) &&
 		equality.Semantic.DeepEqual(*desired.Spec.MinReplicas, *existing.Spec.MinReplicas)
 }
 
 // Reconcile ...
-func (r *HPAReconciler) Reconcile(scaleToZero bool) (*v2beta2.HorizontalPodAutoscaler, error) {
+func (r *HPAReconciler) Reconcile(scaleToZero bool) (*hpav2.HorizontalPodAutoscaler, error) {
 	//reconcile
 	checkResult, existingHPA, err := r.checkHPAExist(r.client)
 	log.Info("service reconcile", "checkResult", checkResult, "scaleToZero", scaleToZero, "err", err)

fvt/fvtclient.go

@@ -40,7 +40,7 @@ import (
 	"google.golang.org/grpc/credentials"
 
 	appsv1 "k8s.io/api/apps/v1"
-	hpav2beta2 "k8s.io/api/autoscaling/v2beta2"
+	hpav2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -251,7 +251,7 @@ var (
 	}
 	gvrHPA = schema.GroupVersionResource{
 		Group:    "autoscaling",
-		Version:  "v2beta2",
+		Version:  "v2",
 		Resource: "horizontalpodautoscalers", // this must be the plural form
 	}
 )
@@ -818,16 +818,16 @@ func (fvt *FVTClient) StartWatchingDeploys() watch.Interface {
 	return deployWatcher
 }
 
-func (fvt *FVTClient) ListHPAs() hpav2beta2.HorizontalPodAutoscalerList {
+func (fvt *FVTClient) ListHPAs() hpav2.HorizontalPodAutoscalerList {
 	var err error
 
 	listOptions := metav1.ListOptions{LabelSelector: "app.kubernetes.io/managed-by=modelmesh-controller", TimeoutSeconds: &DefaultTimeout}
 	u, err := fvt.Resource(gvrHPA).Namespace(fvt.namespace).List(context.TODO(), listOptions)
 	Expect(err).ToNot(HaveOccurred())
 
-	var hpaList hpav2beta2.HorizontalPodAutoscalerList
+	var hpaList hpav2.HorizontalPodAutoscalerList
 	for _, uh := range u.Items {
-		var h hpav2beta2.HorizontalPodAutoscaler
+		var h hpav2.HorizontalPodAutoscaler
 		err = runtime.DefaultUnstructuredConverter.FromUnstructured(uh.Object, &h)
 		Expect(err).ToNot(HaveOccurred())
 		hpaList.Items = append(hpaList.Items, h)

fvt/hpa/hpa_test.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/kserve/kserve/pkg/constants"
 	mmcontstant "github.com/kserve/modelmesh-serving/pkg/constants"
-	hpav2beta2 "k8s.io/api/autoscaling/v2beta2"
+	hpav2 "k8s.io/api/autoscaling/v2"
 
 	. "github.com/kserve/modelmesh-serving/fvt"
 	. "github.com/onsi/ginkgo/v2"
@@ -61,10 +61,10 @@ var _ = Describe("Scaling of runtime deployments with HPA Autoscaler", Ordered,
 		Expect(replicas).To(BeEquivalentTo(int32(0)))
 	}
 
-	checkHPAState := func() *hpav2beta2.HorizontalPodAutoscaler {
+	checkHPAState := func() *hpav2.HorizontalPodAutoscaler {
 		hpaList := FVTClientInstance.ListHPAs()
 
-		var hpa *hpav2beta2.HorizontalPodAutoscaler
+		var hpa *hpav2.HorizontalPodAutoscaler
 		if len(hpaList.Items) == 0 {
 			hpa = nil
 		} else {