Merge pull request #45 from Charlie17Li/feat/module_scheduing_strategy

Feature: support module scheduling strategy :scatter, stacking

module-controller/api/v1alpha1/module_types.go

@@ -71,11 +71,8 @@ type ModuleInfo struct {
 
 // ModuleSpec defines the desired state of Module
 type ModuleSpec struct {
-	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
 	Selector metav1.LabelSelector `json:"selector,omitempty"`
-	// Foo is an example field of Module. Edit module_types.go to remove/update
-	Module ModuleInfo `json:"module"`
+	Module   ModuleInfo           `json:"module"`
 }
 
 // ModuleStatus defines the observed state of Module

module-controller/api/v1alpha1/moduledeployment_types.go

@@ -58,6 +58,13 @@ const (
 	ModuleDeploymentDeployTypeAsymmetric DeployType = "asymmetric"
 )
 
+type ModuleSchedulingType string
+
+const (
+	Scatter  ModuleSchedulingType = "Scatter"
+	Stacking ModuleSchedulingType = "Stacking"
+)
+
 type ReleaseStatus struct {
 	// Records the latest revision.
 	// +optional
@@ -92,6 +99,11 @@ type ModuleDeploymentCondition struct {
 	Message string `json:"message,omitempty"`
 }
 
+type ModuleSchedulingStrategy struct {
+	SchedulingType ModuleSchedulingType `json:"schedulingType"`
+	MaxModuleCount int                  `json:"maxModuleCount"`
+}
+
 type ModuleDeploymentStrategy struct {
 	UpgradeType string `json:"upgradeType"`
 
@@ -131,7 +143,9 @@ type ModuleDeploymentSpec struct {
 	// +optional
 	Pause bool `json:"pause,omitempty"`
 
-	Strategy ModuleDeploymentStrategy `json:"strategy,omitempty"`
+	OperationStrategy ModuleDeploymentStrategy `json:"strategy,omitempty"`
+
+	SchedulingStrategy ModuleSchedulingStrategy `json:"schedulingStrategy,omitempty"`
 }
 
 // ModuleDeploymentStatus defines the observed state of ModuleDeployment

module-controller/config/crd/bases/serverless.alipay.com_moduledeployments.yaml

@@ -60,6 +60,16 @@ spec:
               revisionHistoryLimit:
                 format: int32
                 type: integer
+              schedulingStrategy:
+                properties:
+                  maxModuleCount:
+                    type: integer
+                  schedulingType:
+                    type: string
+                required:
+                - maxModuleCount
+                - schedulingType
+                type: object
               strategy:
                 properties:
                   batchCount:
@@ -91,8 +101,6 @@ spec:
                     description: Specification of the desired behavior of the module
                     properties:
                       module:
-                        description: Foo is an example field of Module. Edit module_types.go
-                          to remove/update
                         properties:
                           md5:
                             type: string
@@ -113,9 +121,10 @@ spec:
                         - version
                         type: object
                       selector:
-                        description: 'INSERT ADDITIONAL SPEC FIELDS - desired state
-                          of cluster Important: Run "make" to regenerate code after
-                          modifying this file'
+                        description: A label selector is a label query over a set
+                          of resources. The result of matchLabels and matchExpressions
+                          are ANDed. An empty label selector matches all objects.
+                          A null label selector matches no objects.
                         properties:
                           matchExpressions:
                             description: matchExpressions is a list of label selector

module-controller/config/crd/bases/serverless.alipay.com_modulereplicasets.yaml

@@ -95,8 +95,6 @@ spec:
                     description: Specification of the desired behavior of the module
                     properties:
                       module:
-                        description: Foo is an example field of Module. Edit module_types.go
-                          to remove/update
                         properties:
                           md5:
                             type: string
@@ -117,9 +115,10 @@ spec:
                         - version
                         type: object
                       selector:
-                        description: 'INSERT ADDITIONAL SPEC FIELDS - desired state
-                          of cluster Important: Run "make" to regenerate code after
-                          modifying this file'
+                        description: A label selector is a label query over a set
+                          of resources. The result of matchLabels and matchExpressions
+                          are ANDed. An empty label selector matches all objects.
+                          A null label selector matches no objects.
                         properties:
                           matchExpressions:
                             description: matchExpressions is a list of label selector

module-controller/config/crd/bases/serverless.alipay.com_modules.yaml

@@ -35,8 +35,6 @@ spec:
             description: ModuleSpec defines the desired state of Module
             properties:
               module:
-                description: Foo is an example field of Module. Edit module_types.go
-                  to remove/update
                 properties:
                   md5:
                     type: string
@@ -57,8 +55,10 @@ spec:
                 - version
                 type: object
               selector:
-                description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
-                  Important: Run "make" to regenerate code after modifying this file'
+                description: A label selector is a label query over a set of resources.
+                  The result of matchLabels and matchExpressions are ANDed. An empty
+                  label selector matches all objects. A null label selector matches
+                  no objects.
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.

module-controller/config/crd/bases/serverless.alipay.com_moduletemplates.yaml

@@ -43,8 +43,6 @@ spec:
                 description: Specification of the desired behavior of the module
                 properties:
                   module:
-                    description: Foo is an example field of Module. Edit module_types.go
-                      to remove/update
                     properties:
                       md5:
                         type: string
@@ -65,9 +63,10 @@ spec:
                     - version
                     type: object
                   selector:
-                    description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of
-                      cluster Important: Run "make" to regenerate code after modifying
-                      this file'
+                    description: A label selector is a label query over a set of resources.
+                      The result of matchLabels and matchExpressions are ANDed. An
+                      empty label selector matches all objects. A null label selector
+                      matches no objects.
                     properties:
                       matchExpressions:
                         description: matchExpressions is a list of label selector

module-controller/config/rbac/role.yaml

@@ -4,6 +4,24 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - serverless.alipay.com
   resources:
@@ -82,23 +100,3 @@ rules:
   - get
   - patch
   - update
-- apiGroups:
-    - apps
-  resources:
-    - deployments
-  verbs:
-    - get
-    - list
-    - watch
-- apiGroups: [""]
-  resources:
-    - pods
-  verbs:
-    - create
-    - delete
-    - get
-    - list
-    - patch
-    - update
-    - watch
-

module-controller/go.mod

@@ -35,6 +35,8 @@ require (
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect

module-controller/go.sum

@@ -26,6 +26,7 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -74,14 +75,20 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -103,6 +110,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
 github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=

module-controller/internal/constants/label/well_known_labels.go

@@ -16,4 +16,8 @@ const (
 	DeleteModuleLabel = "serverless.alipay.com/delete-module"
 
 	ModuleInstanceCount = "serverless.alipay.com/module-instance-count"
+
+	ModuleSchedulingStrategy = "serverless.alipay.com/module-scheduling-strategy"
+
+	MaxModuleCount = "serverless.alipay.com/max-module-count"
 )

module-controller/internal/controller/module_controller.go

@@ -19,11 +19,8 @@ package controller
 import (
 	"context"
 	"fmt"
-	v1alpha1 "github.com/sofastack/sofa-serverless/api/v1alpha1"
-	"github.com/sofastack/sofa-serverless/internal/arklet"
-	"github.com/sofastack/sofa-serverless/internal/constants/finalizer"
-	"github.com/sofastack/sofa-serverless/internal/constants/label"
-	"github.com/sofastack/sofa-serverless/internal/utils"
+	"strconv"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -36,7 +33,16 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-	"strconv"
+
+	"github.com/sofastack/sofa-serverless/api/v1alpha1"
+	"github.com/sofastack/sofa-serverless/internal/arklet"
+	"github.com/sofastack/sofa-serverless/internal/constants/finalizer"
+	"github.com/sofastack/sofa-serverless/internal/constants/label"
+	"github.com/sofastack/sofa-serverless/internal/utils"
+)
+
+const (
+	ProtectModuleFinalizer = "module-installed"
 )
 
 // ModuleReconciler reconciles a Module object
@@ -49,6 +55,9 @@ type ModuleReconciler struct {
 //+kubebuilder:rbac:groups=serverless.alipay.com,resources=modules/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=serverless.alipay.com,resources=modules/finalizers,verbs=update
 
+//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch
+//+kubebuilder:rbac:groups=,resources=pods,verbs=create;delete;get;list;patch;update;watch
+
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 // TODO(user): Modify the Reconcile function to compare the state specified by

module-controller/internal/controller/moduledeployment_controller.go

@@ -19,10 +19,13 @@ package controller
 import (
 	"context"
 	"fmt"
+	"strconv"
+
 	moduledeploymentv1alpha1 "github.com/sofastack/sofa-serverless/api/v1alpha1"
 	"github.com/sofastack/sofa-serverless/internal/constants/finalizer"
 	"github.com/sofastack/sofa-serverless/internal/constants/label"
 	"github.com/sofastack/sofa-serverless/internal/utils"
+
 	v1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -45,6 +48,9 @@ type ModuleDeploymentReconciler struct {
 //+kubebuilder:rbac:groups=serverless.alipay.com,resources=moduledeployments/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=serverless.alipay.com,resources=moduledeployments/finalizers,verbs=update
 
+//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch
+//+kubebuilder:rbac:groups=,resources=pods,verbs=create;delete;get;list;patch;update;watch
+
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 // TODO(user): Modify the Reconcile function to compare the state specified by
@@ -223,6 +229,8 @@ func (r *ModuleDeploymentReconciler) generateModuleReplicas(moduleDeployment *mo
 	newLabels := moduleDeployment.Labels
 	newLabels[label.ModuleNameLabel] = moduleDeployment.Spec.Template.Spec.Module.Name
 	newLabels[label.ModuleDeploymentLabel] = moduleDeployment.Name
+	newLabels[label.ModuleSchedulingStrategy] = string(moduleDeployment.Spec.SchedulingStrategy.SchedulingType)
+	newLabels[label.MaxModuleCount] = strconv.Itoa(moduleDeployment.Spec.SchedulingStrategy.MaxModuleCount)
 	moduleReplicaSet := &moduledeploymentv1alpha1.ModuleReplicaSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Annotations: map[string]string{},