diff --git a/api/v1/runtimecomponent_types.go b/api/v1/runtimecomponent_types.go index cd05e1f9..590e3cbd 100644 --- a/api/v1/runtimecomponent_types.go +++ b/api/v1/runtimecomponent_types.go @@ -23,6 +23,7 @@ import ( routev1 "github.com/openshift/api/route/v1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -225,6 +226,16 @@ type RuntimeComponentAutoScaling struct { // Target average CPU utilization, represented as a percentage of requested CPU, over all the pods. // +operator-sdk:csv:customresourcedefinitions:order=3,type=spec,displayName="Target CPU Utilization Percentage",xDescriptors="urn:alm:descriptor:com.tectonic.ui:number" TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"` + + // Target average Memory utilization, represented as a percentage of requested memory, over all the pods. + // +operator-sdk:csv:customresourcedefinitions:order=4,type=spec,displayName="Target Memory Utilization Percentage",xDescriptors="urn:alm:descriptor:com.tectonic.ui:number" + TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"` + + // Specifications used for replica count calculation + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` + + // Scaling behavior of the target. If not set, the default HPAScalingRules for scale up and scale down are used. + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` } // Configures parameters for the network service of pods. @@ -762,6 +773,21 @@ func (a *RuntimeComponentAutoScaling) GetTargetCPUUtilizationPercentage() *int32 return a.TargetCPUUtilizationPercentage } +// GetTargetMemoryUtilizationPercentage returns target memory usage +func (a *RuntimeComponentAutoScaling) GetTargetMemoryUtilizationPercentage() *int32 { + return a.TargetMemoryUtilizationPercentage +} + +// GetMetrics returns metrics for resource utilization +func (a *RuntimeComponentAutoScaling) GetMetrics() []autoscalingv2.MetricSpec { + return a.Metrics +} + +// GetHorizontalPodAutoscalerBehavior returns behavior configures the scaling behavior of the target +func (a *RuntimeComponentAutoScaling) GetHorizontalPodAutoscalerBehavior() *autoscalingv2.HorizontalPodAutoscalerBehavior { + return a.Behavior +} + // GetSize returns persistent volume size func (s *RuntimeComponentStorage) GetSize() string { return s.Size diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 8c2f7381..9108430d 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -26,6 +26,7 @@ import ( routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -132,6 +133,23 @@ func (in *RuntimeComponentAutoScaling) DeepCopyInto(out *RuntimeComponentAutoSca *out = new(int32) **out = **in } + if in.TargetMemoryUtilizationPercentage != nil { + in, out := &in.TargetMemoryUtilizationPercentage, &out.TargetMemoryUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeComponentAutoScaling. diff --git a/api/v1beta2/runtimecomponent_types.go b/api/v1beta2/runtimecomponent_types.go index 6be32cb5..04d7b97a 100644 --- a/api/v1beta2/runtimecomponent_types.go +++ b/api/v1beta2/runtimecomponent_types.go @@ -23,6 +23,7 @@ import ( routev1 "github.com/openshift/api/route/v1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -666,6 +667,21 @@ func (a *RuntimeComponentAutoScaling) GetTargetCPUUtilizationPercentage() *int32 return a.TargetCPUUtilizationPercentage } +// GetTargetMemoryUtilizationPercentage returns target memory usage +func (a *RuntimeComponentAutoScaling) GetTargetMemoryUtilizationPercentage() *int32 { + return nil +} + +// GetMetrics returns metrics for resource utilization +func (a *RuntimeComponentAutoScaling) GetMetrics() []autoscalingv2.MetricSpec { + return nil +} + +// GetHorizontalPodAutoscalerBehavior returns behavior configures the scaling behavior of the target +func (a *RuntimeComponentAutoScaling) GetHorizontalPodAutoscalerBehavior() *autoscalingv2.HorizontalPodAutoscalerBehavior { + return nil +} + // GetSize returns persistent volume size func (s *RuntimeComponentStorage) GetSize() string { return s.Size diff --git a/bundle/manifests/rc.app.stacks_runtimecomponents.yaml b/bundle/manifests/rc.app.stacks_runtimecomponents.yaml index cfb0ed83..d7e8889e 100644 --- a/bundle/manifests/rc.app.stacks_runtimecomponents.yaml +++ b/bundle/manifests/rc.app.stacks_runtimecomponents.yaml @@ -951,6 +951,124 @@ spec: autoscaling: description: Configures the desired resource consumption of pods. properties: + behavior: + description: Scaling behavior of the target. If not set, the default + HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down + to minReplicas pods, with a 300 second stabilization window + (i.e., the highest recommendation for the last 300sec is + used). + properties: + policies: + description: policies is a list of potential scaling polices + which can be used during scaling. At least one policy + must be specified, otherwise the HPAScalingRules will + be discarded as invalid + items: + description: HPAScalingPolicy is a single policy which + must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window + of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less + than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling + policy. + type: string + value: + description: Value contains the amount of change + which is permitted by the policy. It must be greater + than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which policy + should be used. If not set, the default value Max is + used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number + of seconds for which past recommendations should be + considered while scaling up or scaling down. StabilizationWindowSeconds + must be greater than or equal to zero and less than + or equal to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization is + done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + scaleUp: + description: 'scaleUp is scaling policy for scaling Up. If + not set, the default value is the higher of: * increase + no more than 4 pods per 60 seconds * double the number of + pods per 60 seconds No stabilization is used.' + properties: + policies: + description: policies is a list of potential scaling polices + which can be used during scaling. At least one policy + must be specified, otherwise the HPAScalingRules will + be discarded as invalid + items: + description: HPAScalingPolicy is a single policy which + must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window + of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less + than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling + policy. + type: string + value: + description: Value contains the amount of change + which is permitted by the policy. It must be greater + than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which policy + should be used. If not set, the default value Max is + used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number + of seconds for which past recommendations should be + considered while scaling up or scaling down. StabilizationWindowSeconds + must be greater than or equal to zero and less than + or equal to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization is + done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + type: object maxReplicas: description: Required field for autoscaling. Upper limit for the number of pods that can be set by the autoscaler. Parameter @@ -958,6 +1076,492 @@ spec: format: int32 minimum: 1 type: integer + metrics: + description: Specifications used for replica count calculation + items: + description: MetricSpec specifies how to scale based on a single + metric (only `type` and one other matching field should be + set at once). + properties: + containerResource: + description: containerResource refers to a resource metric + (such as those specified in requests and limits) known + to Kubernetes describing a single container in each pod + of the current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have special scaling + options on top of those available to normal per-pod metrics + using the "pods" source. This is an alpha feature and + can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: external refers to a global metric that is + not associated with any Kubernetes object. It allows autoscaling + based on information coming from components running outside + of cluster (for example length of queue in cloud messaging + service, or QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: object refers to a metric describing a single + kubernetes object (for example, hits-per-second on an + Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: pods refers to a metric describing each pod + in the current scale target (for example, transactions-processed-per-second). The + values will be averaged together before being compared + to the target value. + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: resource refers to a resource metric (such + as those specified in requests and limits) known to Kubernetes + describing each pod in the current scale target (e.g. + CPU or memory). Such metrics are built in to Kubernetes, + and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: 'type is the type of metric source. It should + be one of "ContainerResource", "External", "Object", "Pods" + or "Resource", each mapping to a matching field in the + object. Note: "ContainerResource" type is available on + when the feature-gate HPAContainerMetrics is enabled' + type: string + required: + - type + type: object + type: array minReplicas: description: Lower limit for the number of pods that can be set by the autoscaler. @@ -968,6 +1572,11 @@ spec: percentage of requested CPU, over all the pods. format: int32 type: integer + targetMemoryUtilizationPercentage: + description: Target average Memory utilization, represented as + a percentage of requested memory, over all the pods. + format: int32 + type: integer type: object createKnativeService: description: Create Knative resources and use Knative serving. diff --git a/bundle/manifests/runtime-component.clusterserviceversion.yaml b/bundle/manifests/runtime-component.clusterserviceversion.yaml index 6964d499..f69f9a61 100644 --- a/bundle/manifests/runtime-component.clusterserviceversion.yaml +++ b/bundle/manifests/runtime-component.clusterserviceversion.yaml @@ -11,7 +11,7 @@ metadata: "name": "runtimecomponent-sample" }, "spec": { - "applicationImage": "icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a", + "applicationImage": "icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f", "expose": true, "manageTLS": true, "replicas": 1, @@ -44,7 +44,7 @@ metadata: "name": "runtimecomponent-sample" }, "spec": { - "applicationImage": "icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a", + "applicationImage": "icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f", "expose": true, "replicas": 1, "service": { @@ -71,7 +71,7 @@ metadata: categories: Application Runtime certified: "true" containerImage: icr.io/appcafe/runtime-component-operator:daily - createdAt: "2024-02-02T17:39:01Z" + createdAt: "2024-03-27T14:32:40Z" description: Deploys any runtime component with dynamic and auto-tuning configuration olm.skipRange: '>=0.8.0 <1.3.1' operators.openshift.io/infrastructure-features: '["disconnected"]' @@ -187,6 +187,12 @@ spec: if the probe fails. displayName: Liveness Probe path: probes.liveness + - description: Target average Memory utilization, represented as a percentage + of requested memory, over all the pods. + displayName: Target Memory Utilization Percentage + path: autoscaling.targetMemoryUtilizationPercentage + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: Policy for pulling container images. Defaults to IfNotPresent. displayName: Pull Policy path: pullPolicy @@ -1006,7 +1012,7 @@ spec: fieldRef: fieldPath: metadata.annotations['olm.targetNamespaces'] - name: RELATED_IMAGE_LIBERTY_SAMPLE_APP - value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a + value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f - name: RELATED_IMAGE_RUNTIME_COMPONENT_OPERATOR value: icr.io/appcafe/runtime-component-operator:daily image: icr.io/appcafe/runtime-component-operator:daily @@ -1289,7 +1295,7 @@ spec: provider: name: Community relatedImages: - - image: icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a + - image: icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f name: liberty-sample-app - image: icr.io/appcafe/runtime-component-operator:daily name: runtime-component-operator diff --git a/common/types.go b/common/types.go index ffaa55a8..d6059c49 100644 --- a/common/types.go +++ b/common/types.go @@ -4,6 +4,7 @@ import ( routev1 "github.com/openshift/api/route/v1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -103,6 +104,9 @@ type BaseComponentAutoscaling interface { GetMinReplicas() *int32 GetMaxReplicas() int32 GetTargetCPUUtilizationPercentage() *int32 + GetTargetMemoryUtilizationPercentage() *int32 + GetMetrics() []autoscalingv2.MetricSpec + GetHorizontalPodAutoscalerBehavior() *autoscalingv2.HorizontalPodAutoscalerBehavior } // BaseComponentStorage represents basic PVC configuration diff --git a/config/crd/bases/rc.app.stacks_runtimecomponents.yaml b/config/crd/bases/rc.app.stacks_runtimecomponents.yaml index b28faca1..6104d94d 100644 --- a/config/crd/bases/rc.app.stacks_runtimecomponents.yaml +++ b/config/crd/bases/rc.app.stacks_runtimecomponents.yaml @@ -948,6 +948,124 @@ spec: autoscaling: description: Configures the desired resource consumption of pods. properties: + behavior: + description: Scaling behavior of the target. If not set, the default + HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down + to minReplicas pods, with a 300 second stabilization window + (i.e., the highest recommendation for the last 300sec is + used). + properties: + policies: + description: policies is a list of potential scaling polices + which can be used during scaling. At least one policy + must be specified, otherwise the HPAScalingRules will + be discarded as invalid + items: + description: HPAScalingPolicy is a single policy which + must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window + of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less + than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling + policy. + type: string + value: + description: Value contains the amount of change + which is permitted by the policy. It must be greater + than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which policy + should be used. If not set, the default value Max is + used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number + of seconds for which past recommendations should be + considered while scaling up or scaling down. StabilizationWindowSeconds + must be greater than or equal to zero and less than + or equal to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization is + done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + scaleUp: + description: 'scaleUp is scaling policy for scaling Up. If + not set, the default value is the higher of: * increase + no more than 4 pods per 60 seconds * double the number of + pods per 60 seconds No stabilization is used.' + properties: + policies: + description: policies is a list of potential scaling polices + which can be used during scaling. At least one policy + must be specified, otherwise the HPAScalingRules will + be discarded as invalid + items: + description: HPAScalingPolicy is a single policy which + must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window + of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less + than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling + policy. + type: string + value: + description: Value contains the amount of change + which is permitted by the policy. It must be greater + than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which policy + should be used. If not set, the default value Max is + used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number + of seconds for which past recommendations should be + considered while scaling up or scaling down. StabilizationWindowSeconds + must be greater than or equal to zero and less than + or equal to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization is + done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + type: object maxReplicas: description: Required field for autoscaling. Upper limit for the number of pods that can be set by the autoscaler. Parameter @@ -955,6 +1073,492 @@ spec: format: int32 minimum: 1 type: integer + metrics: + description: Specifications used for replica count calculation + items: + description: MetricSpec specifies how to scale based on a single + metric (only `type` and one other matching field should be + set at once). + properties: + containerResource: + description: containerResource refers to a resource metric + (such as those specified in requests and limits) known + to Kubernetes describing a single container in each pod + of the current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have special scaling + options on top of those available to normal per-pod metrics + using the "pods" source. This is an alpha feature and + can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: external refers to a global metric that is + not associated with any Kubernetes object. It allows autoscaling + based on information coming from components running outside + of cluster (for example length of queue in cloud messaging + service, or QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: object refers to a metric describing a single + kubernetes object (for example, hits-per-second on an + Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: pods refers to a metric describing each pod + in the current scale target (for example, transactions-processed-per-second). The + values will be averaged together before being compared + to the target value. + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: resource refers to a resource metric (such + as those specified in requests and limits) known to Kubernetes + describing each pod in the current scale target (e.g. + CPU or memory). Such metrics are built in to Kubernetes, + and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: 'type is the type of metric source. It should + be one of "ContainerResource", "External", "Object", "Pods" + or "Resource", each mapping to a matching field in the + object. Note: "ContainerResource" type is available on + when the feature-gate HPAContainerMetrics is enabled' + type: string + required: + - type + type: object + type: array minReplicas: description: Lower limit for the number of pods that can be set by the autoscaler. @@ -965,6 +1569,11 @@ spec: percentage of requested CPU, over all the pods. format: int32 type: integer + targetMemoryUtilizationPercentage: + description: Target average Memory utilization, represented as + a percentage of requested memory, over all the pods. + format: int32 + type: integer type: object createKnativeService: description: Create Knative resources and use Knative serving. diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index b1238b71..17441c76 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -67,7 +67,7 @@ spec: fieldRef: fieldPath: metadata.annotations['olm.targetNamespaces'] - name: RELATED_IMAGE_LIBERTY_SAMPLE_APP - value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a + value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f - name: RELATED_IMAGE_RUNTIME_COMPONENT_OPERATOR value: OPERATOR_IMAGE securityContext: diff --git a/config/manifests/bases/runtime-component.clusterserviceversion.yaml b/config/manifests/bases/runtime-component.clusterserviceversion.yaml index e6351cca..b4b4a386 100644 --- a/config/manifests/bases/runtime-component.clusterserviceversion.yaml +++ b/config/manifests/bases/runtime-component.clusterserviceversion.yaml @@ -121,6 +121,12 @@ spec: if the probe fails. displayName: Liveness Probe path: probes.liveness + - description: Target average Memory utilization, represented as a percentage + of requested memory, over all the pods. + displayName: Target Memory Utilization Percentage + path: autoscaling.targetMemoryUtilizationPercentage + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: Policy for pulling container images. Defaults to IfNotPresent. displayName: Pull Policy path: pullPolicy diff --git a/config/samples/rc.app.stacks_v1_runtimecomponent.yaml b/config/samples/rc.app.stacks_v1_runtimecomponent.yaml index d9d683c8..8157f0c6 100644 --- a/config/samples/rc.app.stacks_v1_runtimecomponent.yaml +++ b/config/samples/rc.app.stacks_v1_runtimecomponent.yaml @@ -3,7 +3,7 @@ kind: RuntimeComponent metadata: name: runtimecomponent-sample spec: - applicationImage: icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a + applicationImage: icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f expose: true manageTLS: true replicas: 1 diff --git a/config/samples/rc.app.stacks_v1beta2_runtimecomponent.yaml b/config/samples/rc.app.stacks_v1beta2_runtimecomponent.yaml index cb85c07f..f3c69d15 100644 --- a/config/samples/rc.app.stacks_v1beta2_runtimecomponent.yaml +++ b/config/samples/rc.app.stacks_v1beta2_runtimecomponent.yaml @@ -4,7 +4,7 @@ metadata: name: runtimecomponent-sample spec: # Add fields here - applicationImage: icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a + applicationImage: icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f expose: true replicas: 1 service: diff --git a/controllers/runtimecomponent_controller.go b/controllers/runtimecomponent_controller.go index 8c2e691c..21d03cf0 100644 --- a/controllers/runtimecomponent_controller.go +++ b/controllers/runtimecomponent_controller.go @@ -45,6 +45,7 @@ import ( prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -238,9 +239,16 @@ func (r *RuntimeComponentReconciler) Reconcile(ctx context.Context, req ctrl.Req &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: instance.Name + "-headless", Namespace: instance.Namespace}}, &appsv1.Deployment{ObjectMeta: defaultMeta}, &appsv1.StatefulSet{ObjectMeta: defaultMeta}, - &autoscalingv1.HorizontalPodAutoscaler{ObjectMeta: defaultMeta}, + &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: defaultMeta}, &networkingv1.NetworkPolicy{ObjectMeta: defaultMeta}, } + + if r.IsAutoscalingV2Supported() { + resources = append(resources, &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: defaultMeta}) + } else { + resources = append(resources, &autoscalingv1.HorizontalPodAutoscaler{ObjectMeta: defaultMeta}) + } + err = r.DeleteResources(resources) if err != nil { reqLogger.Error(err, "Failed to clean up non-Knative resources") @@ -408,19 +416,32 @@ func (r *RuntimeComponentReconciler) Reconcile(ctx context.Context, req ctrl.Req } if instance.Spec.Autoscaling != nil { - hpa := &autoscalingv1.HorizontalPodAutoscaler{ObjectMeta: defaultMeta} - err = r.CreateOrUpdate(hpa, instance, func() error { - appstacksutils.CustomizeHPA(hpa, instance) - return nil - }) + if r.IsAutoscalingV2Supported() { + hpa := &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: defaultMeta} + err = r.CreateOrUpdate(hpa, instance, func() error { + appstacksutils.CustomizeHPA(hpa, instance) + return nil + }) + } else { + hpa := &autoscalingv1.HorizontalPodAutoscaler{ObjectMeta: defaultMeta} + err = r.CreateOrUpdate(hpa, instance, func() error { + appstacksutils.CustomizeHPAv1(hpa, instance) + return nil + }) + } if err != nil { reqLogger.Error(err, "Failed to reconcile HorizontalPodAutoscaler") return r.ManageError(err, common.StatusConditionTypeReconciled, instance) } } else { - hpa := &autoscalingv1.HorizontalPodAutoscaler{ObjectMeta: defaultMeta} - err = r.DeleteResource(hpa) + if r.IsAutoscalingV2Supported() { + hpa := &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: defaultMeta} + err = r.DeleteResource(hpa) + } else { + hpa := &autoscalingv1.HorizontalPodAutoscaler{ObjectMeta: defaultMeta} + err = r.DeleteResource(hpa) + } if err != nil { reqLogger.Error(err, "Failed to delete HorizontalPodAutoscaler") return r.ManageError(err, common.StatusConditionTypeReconciled, instance) @@ -607,10 +628,16 @@ func (r *RuntimeComponentReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Service{}, builder.WithPredicates(predSubResource)). Owns(&corev1.Secret{}, builder.WithPredicates(predSubResource)). Owns(&appsv1.Deployment{}, builder.WithPredicates(predSubResWithGenCheck)). - Owns(&appsv1.StatefulSet{}, builder.WithPredicates(predSubResWithGenCheck)). - Owns(&autoscalingv1.HorizontalPodAutoscaler{}, builder.WithPredicates(predSubResource)) + Owns(&appsv1.StatefulSet{}, builder.WithPredicates(predSubResWithGenCheck)) + + ok, _ := r.IsGroupVersionSupported(autoscalingv2.SchemeGroupVersion.String(), "HorizontalPodAutoscaler") + if ok { + b = b.Owns(&autoscalingv2.HorizontalPodAutoscaler{}, builder.WithPredicates(predSubResource)) + } else { + b = b.Owns(&autoscalingv1.HorizontalPodAutoscaler{}, builder.WithPredicates(predSubResource)) + } - ok, _ := r.IsGroupVersionSupported(routev1.SchemeGroupVersion.String(), "Route") + ok, _ = r.IsGroupVersionSupported(routev1.SchemeGroupVersion.String(), "Route") if ok { b = b.Owns(&routev1.Route{}, builder.WithPredicates(predSubResource)) } diff --git a/internal/deploy/kubectl/runtime-component-crd.yaml b/internal/deploy/kubectl/runtime-component-crd.yaml index ded240cc..880e5204 100644 --- a/internal/deploy/kubectl/runtime-component-crd.yaml +++ b/internal/deploy/kubectl/runtime-component-crd.yaml @@ -950,6 +950,124 @@ spec: autoscaling: description: Configures the desired resource consumption of pods. properties: + behavior: + description: Scaling behavior of the target. If not set, the default + HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down + to minReplicas pods, with a 300 second stabilization window + (i.e., the highest recommendation for the last 300sec is + used). + properties: + policies: + description: policies is a list of potential scaling polices + which can be used during scaling. At least one policy + must be specified, otherwise the HPAScalingRules will + be discarded as invalid + items: + description: HPAScalingPolicy is a single policy which + must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window + of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less + than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling + policy. + type: string + value: + description: Value contains the amount of change + which is permitted by the policy. It must be greater + than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which policy + should be used. If not set, the default value Max is + used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number + of seconds for which past recommendations should be + considered while scaling up or scaling down. StabilizationWindowSeconds + must be greater than or equal to zero and less than + or equal to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization is + done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + scaleUp: + description: 'scaleUp is scaling policy for scaling Up. If + not set, the default value is the higher of: * increase + no more than 4 pods per 60 seconds * double the number of + pods per 60 seconds No stabilization is used.' + properties: + policies: + description: policies is a list of potential scaling polices + which can be used during scaling. At least one policy + must be specified, otherwise the HPAScalingRules will + be discarded as invalid + items: + description: HPAScalingPolicy is a single policy which + must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window + of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less + than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling + policy. + type: string + value: + description: Value contains the amount of change + which is permitted by the policy. It must be greater + than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which policy + should be used. If not set, the default value Max is + used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number + of seconds for which past recommendations should be + considered while scaling up or scaling down. StabilizationWindowSeconds + must be greater than or equal to zero and less than + or equal to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization is + done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + type: object maxReplicas: description: Required field for autoscaling. Upper limit for the number of pods that can be set by the autoscaler. Parameter @@ -957,6 +1075,492 @@ spec: format: int32 minimum: 1 type: integer + metrics: + description: Specifications used for replica count calculation + items: + description: MetricSpec specifies how to scale based on a single + metric (only `type` and one other matching field should be + set at once). + properties: + containerResource: + description: containerResource refers to a resource metric + (such as those specified in requests and limits) known + to Kubernetes describing a single container in each pod + of the current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have special scaling + options on top of those available to normal per-pod metrics + using the "pods" source. This is an alpha feature and + can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: external refers to a global metric that is + not associated with any Kubernetes object. It allows autoscaling + based on information coming from components running outside + of cluster (for example length of queue in cloud messaging + service, or QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: object refers to a metric describing a single + kubernetes object (for example, hits-per-second on an + Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: pods refers to a metric describing each pod + in the current scale target (for example, transactions-processed-per-second). The + values will be averaged together before being compared + to the target value. + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: resource refers to a resource metric (such + as those specified in requests and limits) known to Kubernetes + describing each pod in the current scale target (e.g. + CPU or memory). Such metrics are built in to Kubernetes, + and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: 'type is the type of metric source. It should + be one of "ContainerResource", "External", "Object", "Pods" + or "Resource", each mapping to a matching field in the + object. Note: "ContainerResource" type is available on + when the feature-gate HPAContainerMetrics is enabled' + type: string + required: + - type + type: object + type: array minReplicas: description: Lower limit for the number of pods that can be set by the autoscaler. @@ -967,6 +1571,11 @@ spec: percentage of requested CPU, over all the pods. format: int32 type: integer + targetMemoryUtilizationPercentage: + description: Target average Memory utilization, represented as + a percentage of requested memory, over all the pods. + format: int32 + type: integer type: object createKnativeService: description: Create Knative resources and use Knative serving. diff --git a/internal/deploy/kubectl/runtime-component-operator.yaml b/internal/deploy/kubectl/runtime-component-operator.yaml index 7871d7e0..6db011da 100644 --- a/internal/deploy/kubectl/runtime-component-operator.yaml +++ b/internal/deploy/kubectl/runtime-component-operator.yaml @@ -303,7 +303,7 @@ spec: - name: WATCH_NAMESPACE value: RUNTIME_COMPONENT_WATCH_NAMESPACE - name: RELATED_IMAGE_LIBERTY_SAMPLE_APP - value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a + value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f - name: RELATED_IMAGE_RUNTIME_COMPONENT_OPERATOR value: icr.io/appcafe/runtime-component-operator:daily image: icr.io/appcafe/runtime-component-operator:daily diff --git a/internal/deploy/kustomize/daily/base/runtime-component-crd.yaml b/internal/deploy/kustomize/daily/base/runtime-component-crd.yaml index ded240cc..880e5204 100644 --- a/internal/deploy/kustomize/daily/base/runtime-component-crd.yaml +++ b/internal/deploy/kustomize/daily/base/runtime-component-crd.yaml @@ -950,6 +950,124 @@ spec: autoscaling: description: Configures the desired resource consumption of pods. properties: + behavior: + description: Scaling behavior of the target. If not set, the default + HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down + to minReplicas pods, with a 300 second stabilization window + (i.e., the highest recommendation for the last 300sec is + used). + properties: + policies: + description: policies is a list of potential scaling polices + which can be used during scaling. At least one policy + must be specified, otherwise the HPAScalingRules will + be discarded as invalid + items: + description: HPAScalingPolicy is a single policy which + must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window + of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less + than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling + policy. + type: string + value: + description: Value contains the amount of change + which is permitted by the policy. It must be greater + than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which policy + should be used. If not set, the default value Max is + used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number + of seconds for which past recommendations should be + considered while scaling up or scaling down. StabilizationWindowSeconds + must be greater than or equal to zero and less than + or equal to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization is + done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + scaleUp: + description: 'scaleUp is scaling policy for scaling Up. If + not set, the default value is the higher of: * increase + no more than 4 pods per 60 seconds * double the number of + pods per 60 seconds No stabilization is used.' + properties: + policies: + description: policies is a list of potential scaling polices + which can be used during scaling. At least one policy + must be specified, otherwise the HPAScalingRules will + be discarded as invalid + items: + description: HPAScalingPolicy is a single policy which + must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window + of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less + than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling + policy. + type: string + value: + description: Value contains the amount of change + which is permitted by the policy. It must be greater + than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: selectPolicy is used to specify which policy + should be used. If not set, the default value Max is + used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number + of seconds for which past recommendations should be + considered while scaling up or scaling down. StabilizationWindowSeconds + must be greater than or equal to zero and less than + or equal to 3600 (one hour). If not set, use the default + values: - For scale up: 0 (i.e. no stabilization is + done). - For scale down: 300 (i.e. the stabilization + window is 300 seconds long).' + format: int32 + type: integer + type: object + type: object maxReplicas: description: Required field for autoscaling. Upper limit for the number of pods that can be set by the autoscaler. Parameter @@ -957,6 +1075,492 @@ spec: format: int32 minimum: 1 type: integer + metrics: + description: Specifications used for replica count calculation + items: + description: MetricSpec specifies how to scale based on a single + metric (only `type` and one other matching field should be + set at once). + properties: + containerResource: + description: containerResource refers to a resource metric + (such as those specified in requests and limits) known + to Kubernetes describing a single container in each pod + of the current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have special scaling + options on top of those available to normal per-pod metrics + using the "pods" source. This is an alpha feature and + can be enabled by the HPAContainerMetrics feature flag. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: external refers to a global metric that is + not associated with any Kubernetes object. It allows autoscaling + based on information coming from components running outside + of cluster (for example length of queue in cloud messaging + service, or QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: object refers to a metric describing a single + kubernetes object (for example, hits-per-second on an + Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: pods refers to a metric describing each pod + in the current scale target (for example, transactions-processed-per-second). The + values will be averaged together before being compared + to the target value. + properties: + metric: + description: metric identifies the target metric by + name and selector + properties: + name: + description: name is the name of the given metric + type: string + selector: + description: selector is the string-encoded form + of a standard kubernetes label selector for the + given metric When set, it is passed as an additional + parameter to the metrics server for more specific + metrics scoping. When unset, just the metricName + will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: resource refers to a resource metric (such + as those specified in requests and limits) known to Kubernetes + describing each pod in the current scale target (e.g. + CPU or memory). Such metrics are built in to Kubernetes, + and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource in question. + type: string + target: + description: target specifies the target value for the + given metric + properties: + averageUtilization: + description: averageUtilization is the target value + of the average of the resource metric across all + relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source + type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: averageValue is the target value of + the average of the metric across all relevant + pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the metric + type is Utilization, Value, or AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of the metric + (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: 'type is the type of metric source. It should + be one of "ContainerResource", "External", "Object", "Pods" + or "Resource", each mapping to a matching field in the + object. Note: "ContainerResource" type is available on + when the feature-gate HPAContainerMetrics is enabled' + type: string + required: + - type + type: object + type: array minReplicas: description: Lower limit for the number of pods that can be set by the autoscaler. @@ -967,6 +1571,11 @@ spec: percentage of requested CPU, over all the pods. format: int32 type: integer + targetMemoryUtilizationPercentage: + description: Target average Memory utilization, represented as + a percentage of requested memory, over all the pods. + format: int32 + type: integer type: object createKnativeService: description: Create Knative resources and use Knative serving. diff --git a/internal/deploy/kustomize/daily/base/runtime-component-operator.yaml b/internal/deploy/kustomize/daily/base/runtime-component-operator.yaml index 319fb45c..d9e282bb 100644 --- a/internal/deploy/kustomize/daily/base/runtime-component-operator.yaml +++ b/internal/deploy/kustomize/daily/base/runtime-component-operator.yaml @@ -50,7 +50,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: RELATED_IMAGE_LIBERTY_SAMPLE_APP - value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:d735c2ceae5945a0f20adcbcb04e55472d2520b6d1abb6d3049c8521234d3b7a + value: icr.io/appcafe/open-liberty/samples/getting-started@sha256:cd4a9cc586dc371c34df44853abd3301bfd9bfde14efa01170625342a070d14f - name: RELATED_IMAGE_RUNTIME_COMPONENT_OPERATOR value: icr.io/appcafe/runtime-component-operator:daily image: icr.io/appcafe/runtime-component-operator:daily diff --git a/utils/utils.go b/utils/utils.go index a2f5f8d4..60f14713 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -30,6 +30,7 @@ import ( routev1 "github.com/openshift/api/route/v1" appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -330,6 +331,15 @@ func customizeProbeDefaults(config *corev1.Probe, defaultProbe *corev1.Probe) *c return probe } +// IsAutoscalingV2Supported returns true if the cluster supports autoscaling/v2 +func (r *ReconcilerBase) IsAutoscalingV2Supported() bool { + isSupported, err := r.IsGroupVersionSupported(autoscalingv2.SchemeGroupVersion.String(), "HorizontalPodAutoscaler") + if err != nil { + return false + } + return isSupported +} + // CustomizeNetworkPolicy configures the network policy. func CustomizeNetworkPolicy(networkPolicy *networkingv1.NetworkPolicy, isOpenShift bool, ba common.BaseComponent) { obj := ba.(metav1.Object) @@ -983,8 +993,55 @@ func CustomizeKnativeService(ksvc *servingv1.Service, ba common.BaseComponent) { } } -// CustomizeHPA ... -func CustomizeHPA(hpa *autoscalingv1.HorizontalPodAutoscaler, ba common.BaseComponent) { +// CustomizeHPA for autoscaling/v2 +func CustomizeHPA(hpa *autoscalingv2.HorizontalPodAutoscaler, ba common.BaseComponent) { + obj := ba.(metav1.Object) + hpa.Labels = ba.GetLabels() + hpa.Annotations = MergeMaps(hpa.Annotations, ba.GetAnnotations()) + + hpa.Spec.MaxReplicas = ba.GetAutoscaling().GetMaxReplicas() + hpa.Spec.MinReplicas = ba.GetAutoscaling().GetMinReplicas() + hpa.Spec.Behavior = ba.GetAutoscaling().GetHorizontalPodAutoscalerBehavior() + + metricsList := ba.GetAutoscaling().GetMetrics() + + cpuPer := ba.GetAutoscaling().GetTargetCPUUtilizationPercentage() + if cpuPer != nil { + metricSpec := autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{Type: autoscalingv2.UtilizationMetricType, AverageUtilization: cpuPer}, + }, + } + metricsList = append(metricsList, metricSpec) + } + + memPer := ba.GetAutoscaling().GetTargetMemoryUtilizationPercentage() + if memPer != nil { + metricSpec := autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{Type: autoscalingv2.UtilizationMetricType, AverageUtilization: memPer}, + }, + } + metricsList = append(metricsList, metricSpec) + } + + hpa.Spec.Metrics = metricsList + hpa.Spec.ScaleTargetRef.Name = obj.GetName() + hpa.Spec.ScaleTargetRef.APIVersion = "apps/v1" + + if ba.GetStatefulSet() != nil { + hpa.Spec.ScaleTargetRef.Kind = "StatefulSet" + } else { + hpa.Spec.ScaleTargetRef.Kind = "Deployment" + } +} + +// CustomizeHPAv1 for autoscaling/v1 +func CustomizeHPAv1(hpa *autoscalingv1.HorizontalPodAutoscaler, ba common.BaseComponent) { obj := ba.(metav1.Object) hpa.Labels = ba.GetLabels() hpa.Annotations = MergeMaps(hpa.Annotations, ba.GetAnnotations()) diff --git a/utils/utils_test.go b/utils/utils_test.go index f62dbe3f..d6ce669d 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,17 +1,19 @@ package utils import ( - appstacksv1 "github.com/application-stacks/runtime-component-operator/api/v1" "os" "reflect" "strconv" "testing" + appstacksv1 "github.com/application-stacks/runtime-component-operator/api/v1" + "github.com/application-stacks/runtime-component-operator/common" routev1 "github.com/openshift/api/route/v1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,20 +26,31 @@ import ( ) var ( - name = "my-app" - namespace = "runtime" - stack = "java-microprofile" - appImage = "my-image" - replicas int32 = 2 - expose = true - createKNS = true - targetCPUPer int32 = 30 - targetPort int32 = 3333 - nodePort int32 = 3011 - autoscaling = &appstacksv1.RuntimeComponentAutoScaling{ - TargetCPUUtilizationPercentage: &targetCPUPer, - MinReplicas: &replicas, - MaxReplicas: 3, + name = "my-app" + namespace = "runtime" + stack = "java-microprofile" + appImage = "my-image" + replicas int32 = 2 + expose = true + createKNS = true + targetPort int32 = 3333 + nodePort int32 = 3011 + targetCPUPer int32 = 30 + targetMemoryPer int32 = 20 + targetValue = &resource.Quantity{Format: "1k"} + metrics = autoscalingv2.MetricSpec{ + Type: autoscalingv2.PodsMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "packets-per-second", + Target: autoscalingv2.MetricTarget{Type: autoscalingv2.AverageValueMetricType, AverageValue: targetValue}, + }, + } + autoscaling = &appstacksv1.RuntimeComponentAutoScaling{ + MinReplicas: &replicas, + MaxReplicas: 3, + TargetCPUUtilizationPercentage: &targetCPUPer, + TargetMemoryUtilizationPercentage: &targetMemoryPer, + Metrics: []autoscalingv2.MetricSpec{metrics}, } envFrom = []corev1.EnvFromSource{{Prefix: namespace}} env = []corev1.EnvVar{{Name: namespace}} @@ -577,24 +590,43 @@ func TestCustomizeHPA(t *testing.T) { logf.SetLogger(logger) spec := appstacksv1.RuntimeComponentSpec{Autoscaling: autoscaling} - hpa, runtime := &autoscalingv1.HorizontalPodAutoscaler{}, createRuntimeComponent(name, namespace, spec) - CustomizeHPA(hpa, runtime) - nilSTRKind := hpa.Spec.ScaleTargetRef.Kind + hpav1, runtime := &autoscalingv1.HorizontalPodAutoscaler{}, createRuntimeComponent(name, namespace, spec) + CustomizeHPAv1(hpav1, runtime) runtimeStatefulSet := &appstacksv1.RuntimeComponentStatefulSet{Storage: &storage} + hpav2 := &autoscalingv2.HorizontalPodAutoscaler{} spec = appstacksv1.RuntimeComponentSpec{Autoscaling: autoscaling, StatefulSet: runtimeStatefulSet} runtime = createRuntimeComponent(name, namespace, spec) - CustomizeHPA(hpa, runtime) - STRKind := hpa.Spec.ScaleTargetRef.Kind + CustomizeHPA(hpav2, runtime) + + cpuMetricSpec := autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{Type: autoscalingv2.UtilizationMetricType, AverageUtilization: &targetCPUPer}, + }, + } + + memoryMetricSpec := autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{Type: autoscalingv2.UtilizationMetricType, AverageUtilization: &targetMemoryPer}, + }, + } + + metricsList := []autoscalingv2.MetricSpec{metrics} + metricsList = append(metricsList, cpuMetricSpec, memoryMetricSpec) testCHPA := []Test{ - {"Max replicas", autoscaling.MaxReplicas, hpa.Spec.MaxReplicas}, - {"Min replicas", *autoscaling.MinReplicas, *hpa.Spec.MinReplicas}, - {"Target CPU utilization", *autoscaling.TargetCPUUtilizationPercentage, *hpa.Spec.TargetCPUUtilizationPercentage}, - {"", name, hpa.Spec.ScaleTargetRef.Name}, - {"", "apps/v1", hpa.Spec.ScaleTargetRef.APIVersion}, - {"Storage not enabled", "Deployment", nilSTRKind}, - {"Storage enabled", "StatefulSet", STRKind}, + {"Target CPU utilization", *autoscaling.TargetCPUUtilizationPercentage, *hpav1.Spec.TargetCPUUtilizationPercentage}, + {"Max replicas", autoscaling.MaxReplicas, hpav2.Spec.MaxReplicas}, + {"Min replicas", *autoscaling.MinReplicas, *hpav2.Spec.MinReplicas}, + {"Metrics", metricsList, hpav2.Spec.Metrics}, + {"Target Reference Name", name, hpav2.Spec.ScaleTargetRef.Name}, + {"Target Reference API Version", "apps/v1", hpav2.Spec.ScaleTargetRef.APIVersion}, + {"Storage not enabled", "Deployment", hpav1.Spec.ScaleTargetRef.Kind}, + {"Storage enabled", "StatefulSet", hpav2.Spec.ScaleTargetRef.Kind}, } verifyTests(testCHPA, t) }