diff --git a/docs/customresourcestate-metrics.md b/docs/customresourcestate-metrics.md index 6442c322ac..501db66add 100644 --- a/docs/customresourcestate-metrics.md +++ b/docs/customresourcestate-metrics.md @@ -41,6 +41,7 @@ spec: metrics: - name: active_count help: "Count of active Foo" + type: Gauge ... - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,foos,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments,verticalpodautoscalers ``` @@ -60,6 +61,7 @@ metadata: foo: bar name: foo spec: + version: v1.2.3 order: - id: 1 value: true @@ -67,6 +69,7 @@ spec: value: false replicas: 1 status: + phase: Pending active: type-a: 1 type-b: 3 @@ -101,7 +104,9 @@ spec: - name: "uptime" help: "Foo uptime" each: - path: [status, uptime] + type: Gauge + gauge: + path: [status, uptime] ``` Produces the metric: @@ -129,17 +134,19 @@ spec: - name: "ready_count" help: "Number Foo Bars ready" each: - # targeting an object or array will produce a metric for each element - # labelsFromPath and value are relative to this path - path: [status, sub] - - # if path targets an object, the object key will be used as label value - labelFromKey: type - # label values can be resolved specific to this path - labelsFromPath: - active: [active] - # The actual field to use as metric value. Should be a number. - value: [ready] + type: Gauge + gauge: + # targeting an object or array will produce a metric for each element + # labelsFromPath and value are relative to this path + path: [status, sub] + + # if path targets an object, the object key will be used as label value + labelFromKey: type + # label values can be resolved specific to this path + labelsFromPath: + active: [active] + # The actual field to use as metric value. Should be a number. + value: [ready] commonLabels: custom_metric: "yes" labelsFromPath: @@ -160,19 +167,117 @@ kube_myteam_io_v1_Foo_active_count{active="1",custom_metric="yes",foo="bar",name kube_myteam_io_v1_Foo_active_count{active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 3 ``` +### Metric types + +The configuration supports three kind of metrics from the [OpenMetrics specification](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md). + +The metric type is specified by the `type` field and its specific configuration at the types specific struct. + +#### Gauge + +> Gauges are current measurements, such as bytes of memory currently used or the number of items in a queue. For gauges the absolute value is what is of interest to a user. [[0]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge) + +Example: + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + metrics: + - name: "uptime" + help: "Foo uptime" + each: + type: Gauge + gauge: + path: [status, uptime] +``` + +Produces the metric: + +```prometheus +kube_myteam_io_v1_Foo_uptime 43.21 +``` + +#### StateSet + +> StateSets represent a series of related boolean values, also called a bitset. If ENUMs need to be encoded this MAY be done via StateSet. [[1]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset) + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + metrics: + - name: "status_phase" + help: "Foo status_phase" + each: + type: StateSet + stateSet: + labelName: phase + path: [status, phase] + list: [Pending, Bar, Baz] +``` + +Metrics of type `SateSet` will generate a metric for each value defined in `list` for each resource. +The value will be 1, if the value matches the one in list. + +Produces the metric: + +```prometheus +kube_myteam_io_v1_Foo_status_phase{phase="Pending"} 1 +kube_myteam_io_v1_Foo_status_phase{phase="Bar"} 0 +kube_myteam_io_v1_Foo_status_phase{phase="Baz"} 0 +``` + +#### Info + +> Info metrics are used to expose textual information which SHOULD NOT change during process lifetime. Common examples are an application's version, revision control commit, and the version of a compiler. [[2]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info) + +Metrics of type `Info` will always have a value of 1. + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: + group: myteam.io + kind: "Foo" + version: "v1" + metrics: + - name: "version" + help: "Foo version" + each: + type: Info + info: + labelsFromPath: + version: [spec, version] +``` + +Produces the metric: + +```prometheus +kube_myteam_io_v1_Foo_version{version="v1.2.3"} 1 +``` + ### Naming The default metric names are prefixed to avoid collisions with other metrics. -By default, a namespace of `kube` and a subsystem based on your custom resource's group+version+kind is used. -You can override these with the namespace and subsystem fields. +By default, a metric prefix of `kube_` concatenated with your custom resource's group+version+kind is used. +You can override this behavior with the `metricNamePrefix` field. ```yaml kind: CustomResourceStateMetrics spec: resources: - groupVersionKind: ... - namespace: myteam - subsystem: foos + metricNamePrefix: myteam_foos metrics: - name: uptime ... @@ -183,7 +288,18 @@ Produces: myteam_foos_uptime 43.21 ``` -To omit namespace and/or subsystem altogether, set them to `_`. +To omit namespace and/or subsystem altogether, set them to the empty string: + +```yaml +kind: CustomResourceStateMetrics +spec: + resources: + - groupVersionKind: ... + metricNamePrefix: "" + metrics: + - name: uptime + ... +``` ### Logging diff --git a/pkg/customresourcestate/config.go b/pkg/customresourcestate/config.go index 05c856ffbd..01dfd8fa8f 100644 --- a/pkg/customresourcestate/config.go +++ b/pkg/customresourcestate/config.go @@ -21,10 +21,70 @@ import ( "strings" "github.com/gobuffalo/flect" - "k8s.io/klog/v2" + + "k8s.io/kube-state-metrics/v2/pkg/customresource" ) +// Metrics is the top level configuration object. +type Metrics struct { + Spec MetricsSpec `yaml:"spec" json:"spec"` +} + +// MetricsSpec is the configuration describing the custom resource state metrics to generate. +type MetricsSpec struct { + // Resources is the list of custom resources to be monitored. A resource with the same GroupVersionKind may appear + // multiple times (e.g., to customize the namespace or subsystem,) but will incur additional overhead. + Resources []Resource `yaml:"resources" json:"resources"` +} + +// Resource configures a custom resource for metric generation. +type Resource struct { + // MetricNamePrefix defines a prefix for all metrics of the resource. + // Falls back to the GroupVersionKind string prefixed with "kube_", with invalid characters replaced by _ if nil. + // If set to "", no prefix will be added. + // Example: If GroupVersionKind is "my-team.io/v1/MyResource", MetricNamePrefix will be "kube_my_team_io_v1_MyResource". + MetricNamePrefix *string `yaml:"metricNamePrefix" json:"metricNamePrefix"` + + // GroupVersionKind of the custom resource to be monitored. + GroupVersionKind GroupVersionKind `yaml:"groupVersionKind" json:"groupVersionKind"` + + // Labels are added to all metrics. If the same key is used in a metric, the value from the metric will overwrite the value here. + Labels `yaml:",inline" json:",inline"` + + // Metrics are the custom resource fields to be collected. + Metrics []Generator `yaml:"metrics" json:"metrics"` + // ErrorLogV defines the verbosity threshold for errors logged for this resource. + ErrorLogV klog.Level `yaml:"errorLogV" json:"errorLogV"` + + // ResourcePlural sets the plural name of the resource. Defaults to the plural version of the Kind according to flect.Pluralize. + ResourcePlural string `yaml:"resourcePlural" json:"resourcePlural"` +} + +// GetMetricNamePrefix returns the prefix to use for metrics. +func (r Resource) GetMetricNamePrefix() string { + if r.MetricNamePrefix == nil { + return strings.NewReplacer( + "/", "_", + ".", "_", + "-", "_", + ).Replace(fmt.Sprintf("kube_%s_%s_%s", r.GroupVersionKind.Group, r.GroupVersionKind.Version, r.GroupVersionKind.Kind)) + } + if *r.MetricNamePrefix == "" { + return "" + } + return *r.MetricNamePrefix +} + +// GetResourceName returns the lowercase, plural form of the resource Kind. This is ResourcePlural if it is set. +func (r Resource) GetResourceName() string { + if r.ResourcePlural != "" { + return r.ResourcePlural + } + // kubebuilder default: + return strings.ToLower(flect.Pluralize(r.GroupVersionKind.Kind)) +} + // GroupVersionKind is the Kubernetes group, version, and kind of a resource. type GroupVersionKind struct { Group string `yaml:"group" json:"group"` @@ -32,18 +92,6 @@ type GroupVersionKind struct { Kind string `yaml:"kind" json:"kind"` } -// MetricPer targets a Path that may be a single value, array, or object. Arrays and objects will generate a metric per element. -type MetricPer struct { - // Path is the path to the value to generate metric(s) for. - Path []string `yaml:"path" json:"path"` - // ValueFrom is the path to a numeric field under Path that will be the metric value. - ValueFrom []string `yaml:"valueFrom" json:"valueFrom"` - // LabelFromKey adds a label with the given name if Path is an object. The label value will be the object key. - LabelFromKey string `yaml:"labelFromKey" json:"labelFromKey"` - // LabelsFromPath adds additional labels where the value of the label is taken from a field under Path. - LabelsFromPath map[string][]string `yaml:"labelsFromPath" json:"labelsFromPath"` -} - // Labels is common configuration of labels to add to metrics. type Labels struct { // CommonLabels are added to all metrics. @@ -82,82 +130,49 @@ type Generator struct { // Help text for the metric. Help string `yaml:"help" json:"help"` // Each targets a value or values from the resource. - Each MetricPer `yaml:"each" json:"each"` + Each Metric `yaml:"each" json:"each"` // Labels are added to all metrics. Labels from Each will overwrite these if using the same key. - Labels `yaml:",inline"` // json will inline because it is already tagged + Labels `yaml:",inline" json:",inline"` // json will inline because it is already tagged // ErrorLogV defines the verbosity threshold for errors logged for this metric. Must be non-zero to override the resource setting. ErrorLogV klog.Level `yaml:"errorLogV" json:"errorLogV"` } -// Resource configures a custom resource for metric generation. -type Resource struct { - // Namespace is an optional prefix for all metrics. Defaults to "kube" if not set. If set to "_", no namespace will be added. - // The combination of Namespace and Subsystem will be prefixed to all metrics generated for this resource. - // e.g., if Namespace is "kube" and Subsystem is "myteam_io_v1_MyResource", all metrics will be prefixed with "kube_myteam_io_v1_MyResource_". - Namespace string `yaml:"namespace" json:"namespace"` - // Subsystem defaults to the GroupVersionKind string, with invalid character replaced with _. If set to "_", no subsystem will be added. - // e.g., if GroupVersionKind is "myteam.io/v1/MyResource", Subsystem will be "myteam_io_v1_MyResource". - Subsystem string `yaml:"subsystem" json:"subsystem"` - - // GroupVersionKind of the custom resource to be monitored. - GroupVersionKind GroupVersionKind `yaml:"groupVersionKind" json:"groupVersionKind"` - - // Labels are added to all metrics. If the same key is used in a metric, the value from the metric will overwrite the value here. - Labels `yaml:",inline"` - - // Metrics are the custom resource fields to be collected. - Metrics []Generator `yaml:"metrics" json:"metrics"` - // ErrorLogV defines the verbosity threshold for errors logged for this resource. - ErrorLogV klog.Level `yaml:"errorLogV" json:"errorLogV"` - - // ResourcePlural sets the plural name of the resource. Defaults to the plural version of the Kind according to flect.Pluralize. - ResourcePlural string `yaml:"resourcePlural" json:"resourcePlural"` +// Metric defines a metric to expose. +// +union +type Metric struct { + // Type defines the type of the metric. + // +unionDiscriminator + Type MetricType `yaml:"type" json:"type"` + + // Gauge defines a gauge metric. + // +optional + Gauge *MetricGauge `yaml:"gauge" json:"gauge"` + // StateSet defines a state set metric. + // +optional + StateSet *MetricStateSet `yaml:"stateSet" json:"stateSet"` + // Info defines an info metric. + // +optional + Info *MetricInfo `yaml:"info" json:"info"` } -// GetNamespace returns the namespace prefix to use for metrics. -func (r Resource) GetNamespace() string { - if r.Namespace == "" { - return "kube" - } - if r.Namespace == "_" { - return "" - } - return r.Namespace +// ConfigDecoder is for use with FromConfig. +type ConfigDecoder interface { + Decode(v interface{}) (err error) } -// GetSubsystem returns the subsystem prefix to use for metrics (will be joined between namespace and the metric name). -func (r Resource) GetSubsystem() string { - if r.Subsystem == "" { - return strings.NewReplacer( - "/", "_", - ".", "_", - "-", "_", - ).Replace(fmt.Sprintf("%s_%s_%s", r.GroupVersionKind.Group, r.GroupVersionKind.Version, r.GroupVersionKind.Kind)) - } - if r.Subsystem == "_" { - return "" +// FromConfig decodes a configuration source into a slice of customresource.RegistryFactory that are ready to use. +func FromConfig(decoder ConfigDecoder) (factories []customresource.RegistryFactory, err error) { + var crconfig Metrics + if err := decoder.Decode(&crconfig); err != nil { + return nil, fmt.Errorf("failed to parse Custom Resource State metrics: %w", err) } - return r.Subsystem -} - -// GetResourceName returns the lowercase, plural form of the resource Kind. This is ResourcePlural if it is set. -func (r Resource) GetResourceName() string { - if r.ResourcePlural != "" { - return r.ResourcePlural + for _, resource := range crconfig.Spec.Resources { + factory, err := NewCustomResourceMetrics(resource) + if err != nil { + return nil, fmt.Errorf("failed to create metrics factory for %s: %w", resource.GroupVersionKind, err) + } + factories = append(factories, factory) } - // kubebuilder default: - return strings.ToLower(flect.Pluralize(r.GroupVersionKind.Kind)) -} - -// Metrics is the top level configuration object. -type Metrics struct { - Spec MetricsSpec `yaml:"spec" json:"spec"` -} - -// MetricsSpec is the configuration describing the custom resource state metrics to generate. -type MetricsSpec struct { - // Resources is the list of custom resources to be monitored. A resource with the same GroupVersionKind may appear - // multiple times (e.g., to customize the namespace or subsystem,) but will incur additional overhead. - Resources []Resource `yaml:"resources" json:"resources"` + return factories, nil } diff --git a/pkg/customresourcestate/config_metrics_types.go b/pkg/customresourcestate/config_metrics_types.go new file mode 100644 index 0000000000..110c4f9934 --- /dev/null +++ b/pkg/customresourcestate/config_metrics_types.go @@ -0,0 +1,67 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +// MetricType is the type of a metric. +type MetricType string + +// Supported metric types. +const ( + MetricTypeGauge MetricType = "Gauge" + MetricTypeStateSet MetricType = "StateSet" + MetricTypeInfo MetricType = "Info" +) + +// MetricMeta are variables which may used for any metric type. +type MetricMeta struct { + // LabelsFromPath adds additional labels where the value of the label is taken from a field under Path. + LabelsFromPath map[string][]string `yaml:"labelsFromPath" json:"labelsFromPath"` + // Path is the path to to generate metric(s) for. + Path []string `yaml:"path" json:"path"` +} + +// MetricGauge targets a Path that may be a single value, array, or object. Arrays and objects will generate a metric per element. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge +type MetricGauge struct { + MetricMeta `yaml:",inline" json:",inline"` + + // ValueFrom is the path to a numeric field under Path that will be the metric value. + ValueFrom []string `yaml:"valueFrom" json:"valueFrom"` + // LabelFromKey adds a label with the given name if Path is an object. The label value will be the object key. + LabelFromKey string `yaml:"labelFromKey" json:"labelFromKey"` + // NilIsZero indicates that if a value is nil it will be treated as zero value. + NilIsZero bool `yaml:"nilIsZero" json:"nilIsZero"` +} + +// MetricInfo is a metric which is used to expose textual information. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info +type MetricInfo struct { + MetricMeta `yaml:",inline" json:",inline"` +} + +// MetricStateSet is a metric which represent a series of related boolean values, also called a bitset. +// Ref: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset +type MetricStateSet struct { + MetricMeta `yaml:",inline" json:",inline"` + + // List is the list of values to expose a value for. + List []string `yaml:"list" json:"list"` + // LabelName is the key of the label which is used for each entry in List to expose the value. + LabelName string `yaml:"labelName" json:"labelName"` + // ValueFrom is the subpath to compare the list to. + ValueFrom []string `yaml:"valueFrom" json:"valueFrom"` +} diff --git a/pkg/customresourcestate/config_test.go b/pkg/customresourcestate/config_test.go index 0065068053..7ae11985f2 100644 --- a/pkg/customresourcestate/config_test.go +++ b/pkg/customresourcestate/config_test.go @@ -35,21 +35,21 @@ func Test_Metrics_deserialization(t *testing.T) { assert.Equal(t, "active_count", m.Spec.Resources[0].Metrics[0].Name) t.Run("can create resource factory", func(t *testing.T) { - rf, err := NewFieldMetrics(m.Spec.Resources[0]) + rf, err := NewCustomResourceMetrics(m.Spec.Resources[0]) assert.NoError(t, err) t.Run("labels are merged", func(t *testing.T) { assert.Equal(t, map[string]string{ "name": mustCompilePath(t, "metadata", "name").String(), - }, toPaths(rf.(*fieldMetrics).Families[1].LabelFromPath)) + }, toPaths(rf.(*customResourceMetrics).Families[1].LabelFromPath)) }) t.Run("errorLogV", func(t *testing.T) { - assert.Equal(t, klog.Level(5), rf.(*fieldMetrics).Families[1].ErrorLogV) + assert.Equal(t, klog.Level(5), rf.(*customResourceMetrics).Families[1].ErrorLogV) }) t.Run("resource name", func(t *testing.T) { - assert.Equal(t, rf.(*fieldMetrics).ResourceName, "foos") + assert.Equal(t, rf.(*customResourceMetrics).ResourceName, "foos") }) }) } diff --git a/pkg/customresourcestate/custom_resource_metrics.go b/pkg/customresourcestate/custom_resource_metrics.go new file mode 100644 index 0000000000..da45a7533a --- /dev/null +++ b/pkg/customresourcestate/custom_resource_metrics.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package customresourcestate + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "k8s.io/kube-state-metrics/v2/pkg/customresource" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" +) + +// customResourceMetrics is an implementation of the customresource.RegistryFactory +// interface which provides metrics for custom resources defined in a configuration file. +type customResourceMetrics struct { + MetricNamePrefix string + GroupVersionKind schema.GroupVersionKind + ResourceName string + Families []compiledFamily +} + +var _ customresource.RegistryFactory = &customResourceMetrics{} + +// NewCustomResourceMetrics creates a customresource.RegistryFactory from a configuration object. +func NewCustomResourceMetrics(resource Resource) (customresource.RegistryFactory, error) { + compiled, err := compile(resource) + if err != nil { + return nil, err + } + gvk := schema.GroupVersionKind(resource.GroupVersionKind) + return &customResourceMetrics{ + MetricNamePrefix: resource.GetMetricNamePrefix(), + GroupVersionKind: gvk, + Families: compiled, + ResourceName: resource.GetResourceName(), + }, nil +} + +func (s customResourceMetrics) Name() string { + return s.ResourceName +} + +func (s customResourceMetrics) CreateClient(cfg *rest.Config) (interface{}, error) { + c, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + return c.Resource(schema.GroupVersionResource{ + Group: s.GroupVersionKind.Group, + Version: s.GroupVersionKind.Version, + Resource: s.ResourceName, + }), nil +} + +func (s customResourceMetrics) MetricFamilyGenerators(_, _ []string) (result []generator.FamilyGenerator) { + klog.InfoS("Custom resource state added metrics", "familyNames", s.names()) + for _, f := range s.Families { + result = append(result, famGen(f)) + } + + return result +} + +func (s customResourceMetrics) ExpectedType() interface{} { + u := unstructured.Unstructured{} + u.SetGroupVersionKind(s.GroupVersionKind) + return &u +} + +func (s customResourceMetrics) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher { + api := customResourceClient.(dynamic.NamespaceableResourceInterface).Namespace(ns) + ctx := context.Background() + return &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = fieldSelector + return api.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = fieldSelector + return api.Watch(ctx, options) + }, + } +} + +func (s customResourceMetrics) names() (names []string) { + for _, family := range s.Families { + names = append(names, family.Name) + } + return names +} diff --git a/pkg/customresourcestate/example_config.yaml b/pkg/customresourcestate/example_config.yaml index d71c141254..39a479b2be 100644 --- a/pkg/customresourcestate/example_config.yaml +++ b/pkg/customresourcestate/example_config.yaml @@ -26,11 +26,13 @@ spec: - name: "active_count" help: "Number Foo Bars active" each: - path: [status, active] - labelFromKey: type - labelsFromPath: - bar: [bar] - value: [count] + type: Gauge + gauge: + path: [status, active] + labelFromKey: type + labelsFromPath: + bar: [bar] + value: [count] commonLabels: custom_metric: "yes" @@ -40,6 +42,23 @@ spec: - name: "other_count" each: - path: [status, other] + type: Gauge + gauge: + path: [status, other] errorLogV: 5 + - name: "info" + each: + type: Info + info: + path: [spec, version] + errorLogV: 5 + + - name: "phase" + each: + type: StateSet + stateSet: + path: [status, phase] + labelName: phase + list: [Active, Running, Terminating] + errorLogV: 5 diff --git a/pkg/customresourcestate/registry_factory.go b/pkg/customresourcestate/registry_factory.go index 0e6f0a6e19..66158782eb 100644 --- a/pkg/customresourcestate/registry_factory.go +++ b/pkg/customresourcestate/registry_factory.go @@ -17,7 +17,7 @@ limitations under the License. package customresourcestate import ( - "context" + "errors" "fmt" "math" "sort" @@ -25,45 +25,13 @@ import ( "strings" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" - "k8s.io/kube-state-metrics/v2/pkg/customresource" "k8s.io/kube-state-metrics/v2/pkg/metric" generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" ) -type fieldMetrics struct { - Namespace string - Subsystem string - GroupVersionKind schema.GroupVersionKind - ResourceName string - Families []compiledFamily -} - -// NewFieldMetrics creates a customresource.RegistryFactory from a configuration object. -func NewFieldMetrics(resource Resource) (customresource.RegistryFactory, error) { - compiled, err := compile(resource) - if err != nil { - return nil, err - } - gvk := schema.GroupVersionKind(resource.GroupVersionKind) - return &fieldMetrics{ - Namespace: resource.GetNamespace(), - Subsystem: resource.GetSubsystem(), - GroupVersionKind: gvk, - Families: compiled, - ResourceName: resource.GetResourceName(), - }, nil -} - func compile(resource Resource) ([]compiledFamily, error) { var families []compiledFamily for _, f := range resource.Metrics { @@ -76,20 +44,29 @@ func compile(resource Resource) ([]compiledFamily, error) { return families, nil } -func compileFamily(f Generator, resource Resource) (*compiledFamily, error) { - labels := resource.Labels.Merge(f.Labels) - eachPath, err := compilePath(f.Each.Path) +func compileCommon(c MetricMeta) (*compiledCommon, error) { + eachPath, err := compilePath(c.Path) if err != nil { - return nil, fmt.Errorf("each.path: %w", err) + return nil, fmt.Errorf("path: %w", err) } - valuePath, err := compilePath(f.Each.ValueFrom) + eachLabelsFromPath, err := compilePaths(c.LabelsFromPath) if err != nil { - return nil, fmt.Errorf("each.valueFrom: %w", err) + return nil, fmt.Errorf("labelsFromPath: %w", err) } - eachLabelsFromPath, err := compilePaths(f.Each.LabelsFromPath) + return &compiledCommon{ + path: eachPath, + labelFromPath: eachLabelsFromPath, + }, nil +} + +func compileFamily(f Generator, resource Resource) (*compiledFamily, error) { + labels := resource.Labels.Merge(f.Labels) + + metric, err := newCompiledMetric(f.Each) if err != nil { - return nil, fmt.Errorf("each.labelsFromPath: %w", err) + return nil, fmt.Errorf("compiling metric: %w", err) } + labelsFromPath, err := compilePaths(labels.LabelsFromPath) if err != nil { return nil, fmt.Errorf("labelsFromPath: %w", err) @@ -100,15 +77,10 @@ func compileFamily(f Generator, resource Resource) (*compiledFamily, error) { errorLogV = resource.ErrorLogV } return &compiledFamily{ - Name: fullName(resource, f), - ErrorLogV: errorLogV, - Help: f.Help, - Each: compiledEach{ - Path: eachPath, - ValueFrom: valuePath, - LabelFromKey: f.Each.LabelFromKey, - LabelFromPath: eachLabelsFromPath, - }, + Name: fullName(resource, f), + ErrorLogV: errorLogV, + Help: f.Help, + Each: metric, Labels: labels.CommonLabels, LabelFromPath: labelsFromPath, }, nil @@ -116,11 +88,8 @@ func compileFamily(f Generator, resource Resource) (*compiledFamily, error) { func fullName(resource Resource, f Generator) string { var parts []string - if resource.GetNamespace() != "" { - parts = append(parts, resource.GetNamespace()) - } - if resource.GetSubsystem() != "" { - parts = append(parts, resource.GetSubsystem()) + if resource.GetMetricNamePrefix() != "" { + parts = append(parts, resource.GetMetricNamePrefix()) } parts = append(parts, f.Name) return strings.Join(parts, "_") @@ -137,98 +106,215 @@ func compilePaths(paths map[string][]string) (result map[string]valuePath, err e return result, nil } -func (s fieldMetrics) Name() string { - return s.ResourceName +type compiledEach compiledMetric + +type compiledCommon struct { + labelFromPath map[string]valuePath + path valuePath } -func (s fieldMetrics) CreateClient(cfg *rest.Config) (interface{}, error) { - c, err := dynamic.NewForConfig(cfg) - if err != nil { - return nil, err - } - return c.Resource(schema.GroupVersionResource{ - Group: s.GroupVersionKind.Group, - Version: s.GroupVersionKind.Version, - Resource: s.ResourceName, - }), nil +func (c compiledCommon) Path() valuePath { + return c.path +} +func (c compiledCommon) LabelFromPath() map[string]valuePath { + return c.labelFromPath } -func (s fieldMetrics) ExpectedType() interface{} { - u := unstructured.Unstructured{} - u.SetGroupVersionKind(s.GroupVersionKind) - return &u +type eachValue struct { + Labels map[string]string + Value float64 } -func (s fieldMetrics) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher { - api := customResourceClient.(dynamic.NamespaceableResourceInterface).Namespace(ns) - ctx := context.Background() - return &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - options.FieldSelector = fieldSelector - return api.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - options.FieldSelector = fieldSelector - return api.Watch(ctx, options) - }, +type compiledMetric interface { + Values(v interface{}) (result []eachValue, err []error) + Path() valuePath + LabelFromPath() map[string]valuePath +} + +// newCompiledMetric returns a compiledMetric depending given the metric type. +func newCompiledMetric(m Metric) (compiledMetric, error) { + switch m.Type { + case MetricTypeGauge: + if m.Gauge == nil { + return nil, errors.New("expected each.gauge to not be nil") + } + cc, err := compileCommon(m.Gauge.MetricMeta) + if err != nil { + return nil, fmt.Errorf("each.gauge: %w", err) + } + valueFromPath, err := compilePath(m.Gauge.ValueFrom) + if err != nil { + return nil, fmt.Errorf("each.gauge.valueFrom: %w", err) + } + return &compiledGauge{ + compiledCommon: *cc, + ValueFrom: valueFromPath, + NilIsZero: m.Gauge.NilIsZero, + }, nil + case MetricTypeInfo: + if m.Info == nil { + return nil, errors.New("expected each.info to not be nil") + } + cc, err := compileCommon(m.Info.MetricMeta) + if err != nil { + return nil, fmt.Errorf("each.info: %w", err) + } + return &compiledInfo{ + compiledCommon: *cc, + }, nil + case MetricTypeStateSet: + if m.StateSet == nil { + return nil, errors.New("expected each.stateSet to not be nil") + } + cc, err := compileCommon(m.StateSet.MetricMeta) + if err != nil { + return nil, fmt.Errorf("each.stateSet: %w", err) + } + valueFromPath, err := compilePath(m.StateSet.ValueFrom) + if err != nil { + return nil, fmt.Errorf("each.gauge.valueFrom: %w", err) + } + return &compiledStateSet{ + compiledCommon: *cc, + List: m.StateSet.List, + LabelName: m.StateSet.LabelName, + ValueFrom: valueFromPath, + }, nil + default: + return nil, fmt.Errorf("unknown metric type %s", m.Type) } } -type compiledEach struct { - Path valuePath - ValueFrom valuePath - LabelFromKey string - LabelFromPath map[string]valuePath +type compiledGauge struct { + compiledCommon + ValueFrom valuePath + LabelFromKey string + NilIsZero bool } -type eachValue struct { - Labels map[string]string - Value float64 +func newCompiledGauge(m *MetricGauge) (*compiledGauge, error) { + cc, err := compileCommon(m.MetricMeta) + if err != nil { + return nil, fmt.Errorf("compile common: %w", err) + } + valueFromPath, err := compilePath(m.ValueFrom) + if err != nil { + return nil, fmt.Errorf("compile path ValueFrom: %w", err) + } + return &compiledGauge{ + compiledCommon: *cc, + ValueFrom: valueFromPath, + }, nil } -func (e compiledEach) Values(obj map[string]interface{}) (result []eachValue, errors []error) { - v := e.Path.Get(obj) +func (c *compiledGauge) Values(v interface{}) (result []eachValue, errs []error) { onError := func(err error) { - errors = append(errors, fmt.Errorf("%s: %v", e.Path, err)) + errs = append(errs, fmt.Errorf("%s: %v", c.Path(), err)) } + switch iter := v.(type) { case map[string]interface{}: for key, it := range iter { - ev, err := e.value(it) + ev, err := c.value(it) if err != nil { onError(fmt.Errorf("[%s]: %w", key, err)) continue } - if key != "" && e.LabelFromKey != "" { - ev.Labels[e.LabelFromKey] = key + if key != "" && c.LabelFromKey != "" { + ev.Labels[c.LabelFromKey] = key } - addPathLabels(it, e.LabelFromPath, ev.Labels) + addPathLabels(it, c.LabelFromPath(), ev.Labels) result = append(result, *ev) } case []interface{}: for i, it := range iter { - value, err := e.value(it) + value, err := c.value(it) if err != nil { onError(fmt.Errorf("[%d]: %w", i, err)) continue } - addPathLabels(it, e.LabelFromPath, value.Labels) + addPathLabels(it, c.LabelFromPath(), value.Labels) result = append(result, *value) } default: - value, err := e.value(v) + value, err := c.value(v) if err != nil { onError(err) break } - addPathLabels(v, e.LabelFromPath, value.Labels) + addPathLabels(v, c.LabelFromPath(), value.Labels) result = append(result, *value) } - // return results in a consistent order (simplifies testing) - sort.Slice(result, func(i, j int) bool { - return less(result[i].Labels, result[j].Labels) - }) - return result, errors + return +} + +type compiledInfo struct { + compiledCommon +} + +func (c *compiledInfo) Values(v interface{}) (result []eachValue, errs []error) { + if vs, isArray := v.([]interface{}); isArray { + for _, obj := range vs { + ev, err := c.values(obj) + if len(err) > 0 { + errs = append(errs, err...) + continue + } + result = append(result, ev...) + } + return + } + + return c.values(v) +} + +func (c *compiledInfo) values(v interface{}) (result []eachValue, err []error) { + value := eachValue{Value: 1, Labels: map[string]string{}} + addPathLabels(v, c.labelFromPath, value.Labels) + result = append(result, value) + return +} + +type compiledStateSet struct { + compiledCommon + ValueFrom valuePath + List []string + LabelName string +} + +func (c *compiledStateSet) Values(v interface{}) (result []eachValue, errs []error) { + if vs, isArray := v.([]interface{}); isArray { + for _, obj := range vs { + ev, err := c.values(obj) + if len(err) > 0 { + errs = append(errs, err...) + continue + } + result = append(result, ev...) + } + return + } + + return c.values(v) +} + +func (c *compiledStateSet) values(v interface{}) (result []eachValue, errs []error) { + comparable := c.ValueFrom.Get(v) + value, ok := comparable.(string) + if !ok { + return []eachValue{}, []error{fmt.Errorf("%s: expected value for path to be string, got %T", c.path, comparable)} + } + + for _, entry := range c.List { + ev := eachValue{Value: 0, Labels: map[string]string{}} + if value == entry { + ev.Value = 1 + } + ev.Labels[c.LabelName] = entry + addPathLabels(v, c.labelFromPath, ev.Labels) + result = append(result, ev) + } + return } // less compares two maps of labels by keys and values @@ -257,11 +343,11 @@ func less(a, b map[string]string) bool { return len(aKeys) < len(bKeys) } -func (e compiledEach) value(it interface{}) (*eachValue, error) { +func (c compiledGauge) value(it interface{}) (*eachValue, error) { labels := make(map[string]string) - value, err := getNum(e.ValueFrom.Get(it)) + value, err := getNum(c.ValueFrom.Get(it), c.NilIsZero) if err != nil { - return nil, fmt.Errorf("%s: %w", e.ValueFrom, err) + return nil, fmt.Errorf("%s: %w", c.ValueFrom, err) } return &eachValue{ Labels: labels, @@ -333,7 +419,12 @@ func addPathLabels(obj interface{}, labels map[string]valuePath, result map[stri if strings.HasPrefix(k, "*") { continue } - result[k] = fmt.Sprintf("%v", v.Get(obj)) + value := v.Get(obj) + // skip label if value is nil + if value == nil { + continue + } + result[k] = fmt.Sprintf("%v", value) } } @@ -377,7 +468,7 @@ func compilePath(path []string) (out valuePath, _ error) { return nil, fmt.Errorf("invalid list lookup: %s", part) } key, val := eq[0], eq[1] - num, notNum := getNum(val) + num, notNum := getNum(val, false) boolVal, notBool := strconv.ParseBool(val) out = append(out, pathOp{ part: part, @@ -395,7 +486,7 @@ func compilePath(path []string) (out valuePath, _ error) { } if notNum == nil { - if i, err := getNum(candidate); err == nil && num == i { + if i, err := getNum(candidate, false); err == nil && num == i { return m } } @@ -439,15 +530,6 @@ func compilePath(path []string) (out valuePath, _ error) { return out, nil } -func (s fieldMetrics) MetricFamilyGenerators(_, _ []string) (result []generator.FamilyGenerator) { - klog.InfoS("Custom resource state added metrics", "familyNames", s.names()) - for _, f := range s.Families { - result = append(result, famGen(f)) - } - - return result -} - func famGen(f compiledFamily) generator.FamilyGenerator { errLog := klog.V(f.ErrorLogV) return generator.FamilyGenerator{ @@ -460,12 +542,13 @@ func famGen(f compiledFamily) generator.FamilyGenerator { } } +// generate generates the metrics for a custom resource. func generate(u *unstructured.Unstructured, f compiledFamily, errLog klog.Verbose) *metric.Family { klog.V(10).InfoS("Checked", "compiledFamilyName", f.Name, "unstructuredName", u.GetName()) var metrics []*metric.Metric baseLabels := f.BaseLabels(u.Object) - values, errors := f.Each.Values(u.Object) + values, errors := scrapeValuesFor(f.Each, u.Object) for _, err := range errors { errLog.ErrorS(err, f.Name) } @@ -481,16 +564,25 @@ func generate(u *unstructured.Unstructured, f compiledFamily, errLog klog.Verbos } } -func (s fieldMetrics) names() (names []string) { - for _, family := range s.Families { - names = append(names, family.Name) - } - return names +func scrapeValuesFor(e compiledEach, obj map[string]interface{}) ([]eachValue, []error) { + v := e.Path().Get(obj) + result, errs := e.Values(v) + + // return results in a consistent order (simplifies testing) + sort.Slice(result, func(i, j int) bool { + return less(result[i].Labels, result[j].Labels) + }) + return result, errs } -func getNum(value interface{}) (float64, error) { +// getNum converts the value to a float64 which is the value type for any metric. +func getNum(value interface{}, nilIsZero bool) (float64, error) { var v float64 + // same as bool==false but for bool pointers if value == nil { + if nilIsZero { + return 0, nil + } return 0, fmt.Errorf("expected number but found nil value") } switch vv := value.(type) { @@ -527,26 +619,3 @@ func getNum(value interface{}) (float64, error) { } return v, nil } - -var _ customresource.RegistryFactory = &fieldMetrics{} - -// ConfigDecoder is for use with FromConfig. -type ConfigDecoder interface { - Decode(v interface{}) (err error) -} - -// FromConfig decodes a configuration source into a slice of customresource.RegistryFactory that are ready to use. -func FromConfig(decoder ConfigDecoder) (factories []customresource.RegistryFactory, err error) { - var crconfig Metrics - if err := decoder.Decode(&crconfig); err != nil { - return nil, fmt.Errorf("failed to parse Custom Resource State metrics: %w", err) - } - for _, resource := range crconfig.Spec.Resources { - factory, err := NewFieldMetrics(resource) - if err != nil { - return nil, fmt.Errorf("failed to create metrics factory for %s: %w", resource.GroupVersionKind, err) - } - factories = append(factories, factory) - } - return factories, nil -} diff --git a/pkg/customresourcestate/registry_factory_test.go b/pkg/customresourcestate/registry_factory_test.go index 166399ac28..09676d8001 100644 --- a/pkg/customresourcestate/registry_factory_test.go +++ b/pkg/customresourcestate/registry_factory_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "k8s.io/utils/pointer" "k8s.io/kube-state-metrics/v2/pkg/metric" ) @@ -34,6 +35,7 @@ func init() { bytes, err := json.Marshal(Obj{ "spec": Obj{ "replicas": 1, + "version": "v0.0.0", "order": Array{ Obj{ "id": 1, @@ -50,6 +52,7 @@ func init() { "type-a": 1, "type-b": 3, }, + "phase": "foo", "sub": Obj{ "type-a": Obj{ "active": 1, @@ -134,100 +137,92 @@ func Test_addPathLabels(t *testing.T) { } } -func Test_compiledEach_Values(t *testing.T) { +func Test_values(t *testing.T) { type tc struct { name string each compiledEach wantResult []eachValue wantErrors []error } - val := func(value float64, labels ...string) eachValue { - t.Helper() - if len(labels)%2 != 0 { - t.Fatalf("labels must be even: %v", labels) - } - m := make(map[string]string) - for i := 0; i < len(labels); i += 2 { - m[labels[i]] = labels[i+1] - } - return eachValue{ - Value: value, - Labels: m, - } - } tests := []tc{ - {name: "single", each: compiledEach{ - Path: mustCompilePath(t, "spec", "replicas"), - }, wantResult: []eachValue{val(1)}}, - {name: "obj", each: compiledEach{ - Path: mustCompilePath(t, "status", "active"), + {name: "single", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "spec", "replicas"), + }, + }, wantResult: []eachValue{newEachValue(t, 1)}}, + {name: "obj", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "active"), + }, LabelFromKey: "type", }, wantResult: []eachValue{ - val(1, "type", "type-a"), - val(3, "type", "type-b"), + newEachValue(t, 1, "type", "type-a"), + newEachValue(t, 3, "type", "type-b"), }}, - {name: "deep obj", each: compiledEach{ - Path: mustCompilePath(t, "status", "sub"), + {name: "deep obj", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "sub"), + labelFromPath: map[string]valuePath{ + "active": mustCompilePath(t, "active"), + }, + }, LabelFromKey: "type", ValueFrom: mustCompilePath(t, "ready"), - LabelFromPath: map[string]valuePath{ - "active": mustCompilePath(t, "active"), - }, }, wantResult: []eachValue{ - val(2, "type", "type-a", "active", "1"), - val(4, "type", "type-b", "active", "3"), + newEachValue(t, 2, "type", "type-a", "active", "1"), + newEachValue(t, 4, "type", "type-b", "active", "3"), }}, - {name: "array", each: compiledEach{ - Path: mustCompilePath(t, "status", "conditions"), + {name: "array", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "conditions"), + labelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "name"), + }, + }, ValueFrom: mustCompilePath(t, "value"), - LabelFromPath: map[string]valuePath{ - "name": mustCompilePath(t, "name"), + }, wantResult: []eachValue{ + newEachValue(t, 45, "name", "a"), + newEachValue(t, 66, "name", "b"), + }}, + {name: "timestamp", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "metadata", "creationTimestamp"), }, }, wantResult: []eachValue{ - val(45, "name", "a"), - val(66, "name", "b"), + newEachValue(t, 1656374400), + }}, + {name: "boolean_string", each: &compiledGauge{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "spec", "paused"), + }, + NilIsZero: true, + }, wantResult: []eachValue{ + newEachValue(t, 0), + }}, + {name: "info", each: &compiledInfo{ + compiledCommon{ + labelFromPath: map[string]valuePath{ + "version": mustCompilePath(t, "spec", "version"), + }, + }, + }, wantResult: []eachValue{ + newEachValue(t, 1, "version", "v0.0.0"), + }}, + {name: "stateset", each: &compiledStateSet{ + compiledCommon: compiledCommon{ + path: mustCompilePath(t, "status", "phase"), + }, + LabelName: "phase", + List: []string{"foo", "bar"}, + }, wantResult: []eachValue{ + newEachValue(t, 0, "phase", "bar"), + newEachValue(t, 1, "phase", "foo"), }}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotResult, gotErrors := tt.each.Values(cr) - assert.Equal(t, tt.wantResult, gotResult) - assert.Equal(t, tt.wantErrors, gotErrors) - }) - } -} - -func Test_compiledEach_Timestamp(t *testing.T) { - type tc struct { - name string - each compiledEach - wantResult []eachValue - wantErrors []error - } - val := func(value float64, labels ...string) eachValue { - t.Helper() - if len(labels)%2 != 0 { - t.Fatalf("labels must be even: %v", labels) - } - m := make(map[string]string) - for i := 0; i < len(labels); i += 2 { - m[labels[i]] = labels[i+1] - } - return eachValue{ - Value: value, - Labels: m, - } - } - - tests := []tc{ - {name: "single_timestamp", each: compiledEach{ - Path: mustCompilePath(t, "metadata", "creationTimestamp"), - }, wantResult: []eachValue{val(1656374400)}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotResult, gotErrors := tt.each.Values(cr) + gotResult, gotErrors := scrapeValuesFor(tt.each, cr) assert.Equal(t, tt.wantResult, gotResult) assert.Equal(t, tt.wantErrors, gotErrors) }) @@ -321,42 +316,26 @@ func Test_fullName(t *testing.T) { { name: "defaults", args: args{ - resource: r("", ""), + resource: r(nil), f: count, }, want: "kube_apps_v1_Deployment_count", }, { - name: "_", + name: "no prefix", args: args{ - resource: r("_", "_"), + resource: r(pointer.String("")), f: count, }, want: "count", }, - { - name: "_namespace", - args: args{ - resource: r("_", ""), - f: count, - }, - want: "apps_v1_Deployment_count", - }, - { - name: "_subsystem", - args: args{ - resource: r("", "_"), - f: count, - }, - want: "kube_count", - }, { name: "custom", args: args{ - resource: r("foo", "bar_baz"), + resource: r(pointer.String("bar_baz")), f: count, }, - want: "foo_bar_baz_count", + want: "bar_baz_count", }, } for _, tt := range tests { @@ -368,8 +347,8 @@ func Test_fullName(t *testing.T) { } } -func r(namespace, subsystem string) Resource { - return Resource{Namespace: namespace, Subsystem: subsystem, GroupVersionKind: gkv("apps", "v1", "Deployment")} +func r(metricNamePrefix *string) Resource { + return Resource{MetricNamePrefix: metricNamePrefix, GroupVersionKind: gkv("apps", "v1", "Deployment")} } func gkv(group, version, kind string) GroupVersionKind { @@ -410,6 +389,21 @@ func Test_valuePath_Get(t *testing.T) { } } +func newEachValue(t *testing.T, value float64, labels ...string) eachValue { + t.Helper() + if len(labels)%2 != 0 { + t.Fatalf("labels must be even: %v", labels) + } + m := make(map[string]string) + for i := 0; i < len(labels); i += 2 { + m[labels[i]] = labels[i+1] + } + return eachValue{ + Value: value, + Labels: m, + } +} + func mustCompilePath(t *testing.T, path ...string) valuePath { t.Helper() out, err := compilePath(path)