diff --git a/operator/api/v1alpha1/projectconfig_types.go b/operator/api/v1alpha1/projectconfig_types.go index bc37133465..3e73ebd0e9 100644 --- a/operator/api/v1alpha1/projectconfig_types.go +++ b/operator/api/v1alpha1/projectconfig_types.go @@ -54,18 +54,27 @@ type PaaSAnalysisConfig struct { Enabled bool `json:"enabled"` } -// ResLimitConfig contains bkapp resource limit -type ResLimitConfig struct { - // ProcDefaultCPULimits is process's default cpu quota - ProcDefaultCPULimits string `json:"procDefaultCPULimits"` +// ResLimitsConfig contains bkapp resource limits +type ResLimitsConfig struct { + // ProcDefaultCPULimit is process's default cpu quota + ProcDefaultCPULimit string `json:"procDefaultCPULimit"` - // ProcDefaultMemLimits is process's default memory quota - ProcDefaultMemLimits string `json:"procDefaultMemLimits"` + // ProcDefaultMemLimit is process's default memory quota + ProcDefaultMemLimit string `json:"procDefaultMemLimit"` // MaxReplicas is single instance max replica num MaxReplicas int32 `json:"maxReplicas"` } +// ResRequestsConfig contains bkapp resource requests +type ResRequestsConfig struct { + // ProcDefaultCPURequest is process's default cpu request + ProcDefaultCPURequest string `json:"procDefaultCPURequest"` + + // ProcDefaultMemRequest is process's default memory request + ProcDefaultMemRequest string `json:"procDefaultMemRequest"` +} + // AutoscalingConfig contains the config for autoscaling type AutoscalingConfig struct { // Enabled indicates whether autoscaling is enabled @@ -84,7 +93,8 @@ type ProjectConfig struct { Platform PlatformConfig `json:"platform"` IngressPlugin IngressPluginConfig `json:"ingressPlugin"` - ResLimit ResLimitConfig `json:"resLimit"` + ResLimits ResLimitsConfig `json:"resLimits"` + ResRequests ResRequestsConfig `json:"resRequests"` Autoscaling AutoscalingConfig `json:"autoscaling"` MaxProcesses int32 `json:"maxProcesses"` } @@ -102,9 +112,13 @@ func NewProjectConfig() *ProjectConfig { } // 资源预设默认值 - conf.ResLimit.ProcDefaultCPULimits = "4000m" - conf.ResLimit.ProcDefaultMemLimits = "1024Mi" - conf.ResLimit.MaxReplicas = 5 + conf.ResLimits.ProcDefaultCPULimit = "4000m" + conf.ResLimits.ProcDefaultMemLimit = "1024Mi" + conf.ResLimits.MaxReplicas = 5 + + // 资源请求默认值 + conf.ResRequests.ProcDefaultMemRequest = "" + conf.ResRequests.ProcDefaultCPURequest = "" conf.MaxProcesses = 8 @@ -124,17 +138,27 @@ func (p *ProjectConfig) GetMaxProcesses() int32 { // GetProcMaxReplicas returns the max replicas of a process func (p *ProjectConfig) GetProcMaxReplicas() int32 { - return p.ResLimit.MaxReplicas + return p.ResLimits.MaxReplicas +} + +// GetProcDefaultCpuLimit returns the default cpu limit of a process +func (p *ProjectConfig) GetProcDefaultCpuLimit() string { + return p.ResLimits.ProcDefaultCPULimit +} + +// GetProcDefaultMemLimit returns the default memory limit of a process +func (p *ProjectConfig) GetProcDefaultMemLimit() string { + return p.ResLimits.ProcDefaultMemLimit } -// GetProcDefaultCpuLimits returns the default cpu limits of a process -func (p *ProjectConfig) GetProcDefaultCpuLimits() string { - return p.ResLimit.ProcDefaultCPULimits +// GetProcDefaultCpuRequest returns the default cpu request of a process +func (p *ProjectConfig) GetProcDefaultCpuRequest() string { + return p.ResRequests.ProcDefaultCPURequest } -// GetProcDefaultMemLimits returns the default memory limits of a process -func (p *ProjectConfig) GetProcDefaultMemLimits() string { - return p.ResLimit.ProcDefaultMemLimits +// GetProcDefaultMemRequest returns the default cpu limit of a process +func (p *ProjectConfig) GetProcDefaultMemRequest() string { + return p.ResRequests.ProcDefaultMemRequest } // GetIngressClassName returns the ingress class name diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 8e2771145e..73e150fb0d 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -694,7 +694,8 @@ func (in *ProjectConfig) DeepCopyInto(out *ProjectConfig) { in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) out.Platform = in.Platform in.IngressPlugin.DeepCopyInto(&out.IngressPlugin) - out.ResLimit = in.ResLimit + out.ResLimits = in.ResLimits + out.ResRequests = in.ResRequests out.Autoscaling = in.Autoscaling } @@ -732,16 +733,16 @@ func (in *ReplicasOverlay) DeepCopy() *ReplicasOverlay { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ResLimitConfig) DeepCopyInto(out *ResLimitConfig) { +func (in *ResLimitsConfig) DeepCopyInto(out *ResLimitsConfig) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResLimitConfig. -func (in *ResLimitConfig) DeepCopy() *ResLimitConfig { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResLimitsConfig. +func (in *ResLimitsConfig) DeepCopy() *ResLimitsConfig { if in == nil { return nil } - out := new(ResLimitConfig) + out := new(ResLimitsConfig) in.DeepCopyInto(out) return out } @@ -761,6 +762,21 @@ func (in *ResQuotaOverlay) DeepCopy() *ResQuotaOverlay { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResRequestsConfig) DeepCopyInto(out *ResRequestsConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResRequestsConfig. +func (in *ResRequestsConfig) DeepCopy() *ResRequestsConfig { + if in == nil { + return nil + } + out := new(ResRequestsConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Revision) DeepCopyInto(out *Revision) { *out = *in diff --git a/operator/config/crd/bases/paas.bk.tencent.com_projectconfigs.yaml b/operator/config/crd/bases/paas.bk.tencent.com_projectconfigs.yaml index 78a53ab949..357a22d772 100644 --- a/operator/config/crd/bases/paas.bk.tencent.com_projectconfigs.yaml +++ b/operator/config/crd/bases/paas.bk.tencent.com_projectconfigs.yaml @@ -208,23 +208,36 @@ spec: - ingressClassName - sentryDSN type: object - resLimit: - description: ResLimitConfig contains bkapp resource limit + resLimits: + description: ResLimitsConfig contains bkapp resource limits properties: maxReplicas: description: MaxReplicas is single instance max replica num format: int32 type: integer - procDefaultCPULimits: - description: ProcDefaultCPULimits is process's default cpu quota + procDefaultCPULimit: + description: ProcDefaultCPULimit is process's default cpu quota type: string - procDefaultMemLimits: - description: ProcDefaultMemLimits is process's default memory quota + procDefaultMemLimit: + description: ProcDefaultMemLimit is process's default memory quota type: string required: - maxReplicas - - procDefaultCPULimits - - procDefaultMemLimits + - procDefaultCPULimit + - procDefaultMemLimit + type: object + resRequests: + description: ResRequestsConfig contains bkapp resource requests + properties: + procDefaultCPURequest: + description: ProcDefaultCPURequest is process's default cpu request + type: string + procDefaultMemRequest: + description: ProcDefaultMemRequest is process's default memory request + type: string + required: + - procDefaultCPURequest + - procDefaultMemRequest type: object syncPeriod: description: SyncPeriod determines the minimum frequency at which watched @@ -259,7 +272,8 @@ spec: - ingressPlugin - maxProcesses - platform - - resLimit + - resLimits + - resRequests type: object served: true storage: true diff --git a/operator/config/samples/paas_v1alpha1_projectconfig.yaml b/operator/config/samples/paas_v1alpha1_projectconfig.yaml index f71e286f84..ebca55c65a 100644 --- a/operator/config/samples/paas_v1alpha1_projectconfig.yaml +++ b/operator/config/samples/paas_v1alpha1_projectconfig.yaml @@ -18,10 +18,13 @@ platformConfig: sentryDSN: "" ## when there are multiple ingress controllers in your cluster, setting the ingressClassName is necessary. ingressClassName: "" -resLimitConfig: +resLimitsConfig: procDefaultCPULimits: "500m" procDefaultMemLimits: "256Mi" maxReplicas: 5 +resRequestsConfig: + procDefaultCPURequest: "" + procDefaultMemRequest: "" ## leaderElectionReleaseOnCancel defines if the leader should step down volume ## when the Manager ends. This requires the binary to immediately end when the diff --git a/operator/pkg/config/config.go b/operator/pkg/config/config.go index 83540e4a32..2e3cf37552 100644 --- a/operator/pkg/config/config.go +++ b/operator/pkg/config/config.go @@ -24,8 +24,10 @@ type ProjectConfigReader interface { // Process related methods GetMaxProcesses() int32 GetProcMaxReplicas() int32 - GetProcDefaultCpuLimits() string - GetProcDefaultMemLimits() string + GetProcDefaultCpuLimit() string + GetProcDefaultMemLimit() string + GetProcDefaultCpuRequest() string + GetProcDefaultMemRequest() string // Platform related methods GetIngressClassName() string @@ -44,14 +46,22 @@ func (d defaultConfig) GetProcMaxReplicas() int32 { return 5 } -func (d defaultConfig) GetProcDefaultCpuLimits() string { +func (d defaultConfig) GetProcDefaultCpuLimit() string { return "4" } -func (d defaultConfig) GetProcDefaultMemLimits() string { +func (d defaultConfig) GetProcDefaultMemLimit() string { return "1Gi" } +func (d defaultConfig) GetProcDefaultCpuRequest() string { + return "" +} + +func (d defaultConfig) GetProcDefaultMemRequest() string { + return "" +} + func (d defaultConfig) GetIngressClassName() string { return "nginx" } diff --git a/operator/pkg/controllers/resources/deployment_test.go b/operator/pkg/controllers/resources/deployment_test.go index 7d59aa45f5..009d12e400 100644 --- a/operator/pkg/controllers/resources/deployment_test.go +++ b/operator/pkg/controllers/resources/deployment_test.go @@ -232,8 +232,8 @@ var _ = Describe("Test build deployments from BkApp", func() { // The resource requirements should be the default value defined in project config // TODO: enhance below tests to check real plans - Expect(cWebRes.Limits.Cpu().String()).To(Equal(config.Global.GetProcDefaultCpuLimits())) - Expect(cWebRes.Limits.Memory().String()).To(Equal(config.Global.GetProcDefaultMemLimits())) + Expect(cWebRes.Limits.Cpu().String()).To(Equal(config.Global.GetProcDefaultCpuLimit())) + Expect(cWebRes.Limits.Memory().String()).To(Equal(config.Global.GetProcDefaultMemLimit())) }) It("legacy version", func() { diff --git a/operator/pkg/controllers/resources/env_overlay.go b/operator/pkg/controllers/resources/env_overlay.go index f670a38b91..88c252598a 100644 --- a/operator/pkg/controllers/resources/env_overlay.go +++ b/operator/pkg/controllers/resources/env_overlay.go @@ -21,6 +21,7 @@ package resources import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + logf "sigs.k8s.io/controller-runtime/pkg/log" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" "bk.tencent.com/paas-app-operator/pkg/config" @@ -28,6 +29,8 @@ import ( "bk.tencent.com/paas-app-operator/pkg/utils/quota" ) +var log_env_overlay = logf.Log.WithName("env_overlay") + // ReplicasGetter get replicas from BkApp object type ReplicasGetter struct { bkapp *paasv1alpha2.BkApp @@ -253,7 +256,7 @@ func (r *ProcResourcesGetter) fromQuotaPlan(plan paasv1alpha2.ResQuotaPlan) core case paasv1alpha2.ResQuotaPlan4C4G: cpuRaw, memRaw = "4000m", "4096Mi" default: - cpuRaw, memRaw = config.Global.GetProcDefaultCpuLimits(), config.Global.GetProcDefaultMemLimits() + cpuRaw, memRaw = config.Global.GetProcDefaultCpuLimit(), config.Global.GetProcDefaultMemLimit() } return r.calculateResources(cpuRaw, memRaw) } @@ -265,18 +268,41 @@ func (r *ProcResourcesGetter) calculateResources(cpu, memory string) corev1.Reso cpuQuota, _ := quota.NewQuantity(cpu, quota.CPU) memQuota, _ := quota.NewQuantity(memory, quota.Memory) + // 配置 cpu request + // 当配置了 ProcDefaultCpuRequest， 优先使用该值作为 CPU Request 配额 minCpuQuota, _ := quota.NewQuantity("200m", quota.CPU) + procDefaultCpuRequest := config.Global.GetProcDefaultCpuRequest() + if procDefaultCpuRequest != "" { + cpuRequestOverlay, err := quota.NewQuantity(procDefaultCpuRequest, quota.CPU) + if err != nil { + log_env_overlay.Error(err, "Fail to set cpu request", "DefaultCpuRequest", procDefaultCpuRequest) + } else { + minCpuQuota = cpuRequestOverlay + } + } + // 配置 mem request var divisor int64 = 2 medMemQuota, _ := quota.NewQuantity("2048Mi", quota.Memory) if memQuota.Cmp(*medMemQuota) == -1 { divisor = 4 } + minMemQuota := quota.Div(memQuota, divisor) + // 当配置了 ProcDefaultMemLimit，优先使用该值作为 Memory Request 配额 + procDefaultMemRequest := config.Global.GetProcDefaultMemRequest() + if procDefaultMemRequest != "" { + memoryRequestOverlay, err := quota.NewQuantity(procDefaultMemRequest, quota.Memory) + if err != nil { + log_env_overlay.Error(err, "Fail to set memory request", "DefaultMemRequest", procDefaultMemRequest) + } else { + minMemQuota = memoryRequestOverlay + } + } return corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: *minCpuQuota, - corev1.ResourceMemory: *quota.Div(memQuota, divisor), + corev1.ResourceMemory: *minMemQuota, }, Limits: corev1.ResourceList{ corev1.ResourceCPU: *cpuQuota, diff --git a/operator/pkg/controllers/resources/env_overlay_test.go b/operator/pkg/controllers/resources/env_overlay_test.go index efece7f907..34101f99da 100644 --- a/operator/pkg/controllers/resources/env_overlay_test.go +++ b/operator/pkg/controllers/resources/env_overlay_test.go @@ -29,7 +29,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + paasv1alpha1 "bk.tencent.com/paas-app-operator/api/v1alpha1" paasv1alpha2 "bk.tencent.com/paas-app-operator/api/v1alpha2" + "bk.tencent.com/paas-app-operator/pkg/config" "bk.tencent.com/paas-app-operator/pkg/utils/kubetypes" ) @@ -265,6 +267,28 @@ var _ = Describe("Environment overlay related functions", func() { Expect(resReq.Limits.Memory().Equal(resource.MustParse("1Gi"))).To(BeTrue()) }) + It("Get Default Requests", func() { + originalConfig := config.Global + projConf := paasv1alpha1.NewProjectConfig() + projConf.ResRequests.ProcDefaultCPURequest = "100m" + projConf.ResRequests.ProcDefaultMemRequest = "128Mi" + config.SetConfig(projConf) + defer config.SetConfig(originalConfig) + + bkapp.SetAnnotations(map[string]string{paasv1alpha2.EnvironmentKey: "stag"}) + bkapp.Spec.EnvOverlay = &paasv1alpha2.AppEnvOverlay{ + ResQuotas: []paasv1alpha2.ResQuotaOverlay{ + {EnvName: "stag", Process: "web", Plan: paasv1alpha2.ResQuotaPlan4C1G}, + }, + } + getter := NewProcResourcesGetter(bkapp) + resReq, _ := getter.GetByProc("web") + Expect(resReq.Requests.Cpu().Equal(resource.MustParse("100m"))).To(BeTrue()) + Expect(resReq.Requests.Memory().Equal(resource.MustParse("128Mi"))).To(BeTrue()) + Expect(resReq.Limits.Cpu().Equal(resource.MustParse("4"))).To(BeTrue()) + Expect(resReq.Limits.Memory().Equal(resource.MustParse("1Gi"))).To(BeTrue()) + }) + It("Get Standard", func() { bkapp.Spec.Processes[1].ResQuotaPlan = paasv1alpha2.ResQuotaPlan4C2G getter := NewProcResourcesGetter(bkapp) diff --git a/operator/pkg/controllers/resources/hooks_test.go b/operator/pkg/controllers/resources/hooks_test.go index 7b4ff0656d..b8ab4f0649 100644 --- a/operator/pkg/controllers/resources/hooks_test.go +++ b/operator/pkg/controllers/resources/hooks_test.go @@ -103,10 +103,10 @@ var _ = Describe("HookUtils", func() { Expect(len(hook.Pod.Spec.Containers[0].Env)).To(Equal(0)) // 容器资源配额 hookRes := hook.Pod.Spec.Containers[0].Resources - Expect(hookRes.Limits.Cpu().String()).To(Equal(config.Global.GetProcDefaultCpuLimits())) + Expect(hookRes.Limits.Cpu().String()).To(Equal(config.Global.GetProcDefaultCpuLimit())) Expect( hookRes.Limits.Memory().String(), - ).To(Equal(config.Global.GetProcDefaultMemLimits())) + ).To(Equal(config.Global.GetProcDefaultMemLimit())) // 镜像拉取密钥 Expect(