Skip to content

Commit

Permalink
exp: contribute cluster-api-state-metrics
Browse files Browse the repository at this point in the history
Signed-off-by: Tobias Giese <tobias.giese@mercedes-benz.com>
Co-authored-by: Christian Schlotter <schlotterc@vmware.com>
  • Loading branch information
tobiasgiese and chrischdi committed May 31, 2022
1 parent c7aebd7 commit 747ef52
Show file tree
Hide file tree
Showing 43 changed files with 4,641 additions and 7 deletions.
27 changes: 25 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
EXP_DIR := exp
BIN_DIR := bin
TEST_DIR := test
STATE_METRICS_DIR := $(EXP_DIR)/state-metrics
TOOLS_DIR := hack/tools
TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR))
E2E_FRAMEWORK_DIR := $(TEST_DIR)/framework
Expand Down Expand Up @@ -156,6 +157,10 @@ KUBEADM_BOOTSTRAP_CONTROLLER_IMG ?= $(REGISTRY)/$(KUBEADM_BOOTSTRAP_IMAGE_NAME)
KUBEADM_CONTROL_PLANE_IMAGE_NAME ?= kubeadm-control-plane-controller
KUBEADM_CONTROL_PLANE_CONTROLLER_IMG ?= $(REGISTRY)/$(KUBEADM_CONTROL_PLANE_IMAGE_NAME)

# state metrics
STATE_METRICS_IMAGE_NAME ?= state-metrics
STATE_METRICS_IMG ?= $(REGISTRY)/$(STATE_METRICS_IMAGE_NAME)

# It is set by Prow GIT_TAG, a git-based tag of the form vYYYYMMDD-hash, e.g., v20210120-v0.3.10-308-gc61521971

TAG ?= dev
Expand Down Expand Up @@ -190,7 +195,7 @@ ALL_GENERATE_MODULES = core kubeadm-bootstrap kubeadm-control-plane

.PHONY: generate
generate: ## Run all generate-manifests-*, generate-go-deepcopy-*, generate-go-conversions-* and generate-go-openapi targets
$(MAKE) generate-modules generate-manifests generate-go-deepcopy generate-go-conversions generate-go-openapi
$(MAKE) generate-modules generate-manifests generate-go-deepcopy generate-go-conversions generate-go-openapi generate-manifests-cluster-api-state-metrics
$(MAKE) -C $(CAPD_DIR) generate

.PHONY: generate-manifests
Expand Down Expand Up @@ -244,6 +249,13 @@ generate-manifests-kubeadm-control-plane: $(CONTROLLER_GEN) ## Generate manifest
output:webhook:dir=./controlplane/kubeadm/config/webhook \
webhook

.PHONY: generate-manifests-cluster-api-state-metrics
generate-manifests-cluster-api-state-metrics: $(CONTROLLER_GEN) ## Generate RBAC cluster-api-state-metrics
cd $(STATE_METRICS_DIR); $(CONTROLLER_GEN) \
paths=./pkg/... \
rbac:roleName=manager-role \
output:rbac:dir=./config/rbac

.PHONY: generate-go-deepcopy
generate-go-deepcopy: ## Run all generate-go-deepcopy-* targets
$(MAKE) $(addprefix generate-go-deepcopy-,$(ALL_GENERATE_MODULES))
Expand Down Expand Up @@ -383,6 +395,7 @@ lint: $(GOLANGCI_LINT) ## Lint the codebase
$(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS)
cd $(TEST_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS)
cd $(TOOLS_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS)
cd $(STATE_METRICS_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS)

.PHONY: lint-fix
lint-fix: $(GOLANGCI_LINT) ## Lint the codebase and run auto-fixers if supported by the linter
Expand All @@ -405,7 +418,7 @@ verify: $(addprefix verify-,$(ALL_VERIFY_CHECKS)) ## Run all verify-* targets

.PHONY: verify-modules
verify-modules: generate-modules ## Verify go modules are up to date
@if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(TEST_DIR)/go.mod $(TEST_DIR)/go.sum); then \
@if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(TEST_DIR)/go.mod $(TEST_DIR)/go.sum $(STATE_METRICS_DIR)/go.mod $(STATE_METRICS_DIR)/go.sum); then \
git diff; \
echo "go module files are out of date"; exit 1; \
fi
Expand Down Expand Up @@ -473,6 +486,10 @@ manager-kubeadm-bootstrap: ## Build the kubeadm bootstrap manager binary into th
manager-kubeadm-control-plane: ## Build the kubeadm control plane manager binary into the ./bin folder
go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/kubeadm-control-plane-manager sigs.k8s.io/cluster-api/controlplane/kubeadm

.PHONY: cluster-api-state-metrics
cluster-api-state-metrics: ## Build the state metrics binary into the ./bin folder
cd $(STATE_METRICS_DIR); go build -trimpath -ldflags "$(LDFLAGS)" -o $(ROOT_DIR)/$(BIN_DIR)/cluster-api-state-metrics sigs.k8s.io/cluster-api/$(STATE_METRICS_DIR)

.PHONY: docker-pull-prerequisites
docker-pull-prerequisites:
docker pull docker.io/docker/dockerfile:1.1-experimental
Expand Down Expand Up @@ -509,6 +526,12 @@ docker-build-kubeadm-control-plane: ## Build the docker image for kubeadm contro
$(MAKE) set-manifest-image MANIFEST_IMG=$(KUBEADM_CONTROL_PLANE_CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./controlplane/kubeadm/config/default/manager_image_patch.yaml"
$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./controlplane/kubeadm/config/default/manager_pull_policy.yaml"

.PHONY: docker-build-cluster-api-state-metrics
docker-build-cluster-api-state-metrics: ## Build the docker image for state metrics
DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./exp/state-metrics --build-arg ldflags="$(LDFLAGS)" . -t $(STATE_METRICS_IMG)-$(ARCH):$(TAG) -f $(STATE_METRICS_DIR)/Dockerfile
$(MAKE) set-manifest-image MANIFEST_IMG=$(STATE_METRICS_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./$(STATE_METRICS_DIR)/config/default/manager_image_patch.yaml"
$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(STATE_METRICS_DIR)/config/default/manager_pull_policy.yaml"

.PHONY: e2e-framework
e2e-framework: ## Builds the CAPI e2e framework
cd $(E2E_FRAMEWORK_DIR); go build ./...
Expand Down
20 changes: 15 additions & 5 deletions OWNERS_ALIASES
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ aliases:
# folks who can review and LGTM any PRs in the repo
cluster-api-reviewers:
- JoelSpeed
- sbueringer
- stmcginnis

# -----------------------------------------------------------
Expand Down Expand Up @@ -77,27 +76,38 @@ aliases:
# OWNER_ALIASES for test
# -----------------------------------------------------------

cluster-api-test-reviewers:
cluster-api-test-maintainers:
cluster-api-test-reviewers:

# -----------------------------------------------------------
# OWNER_ALIASES for test/framework
# -----------------------------------------------------------

cluster-api-test-framework-reviewers:
cluster-api-test-framework-maintainers:
cluster-api-test-framework-reviewers:

# -----------------------------------------------------------
# OWNER_ALIASES for test/infrastructure/docker
# -----------------------------------------------------------

cluster-api-provider-docker-reviewers:
cluster-api-provider-docker-maintainers:
cluster-api-provider-docker-reviewers:

# -----------------------------------------------------------
# OWNER_ALIASES for docs
# -----------------------------------------------------------

cluster-api-docs-maintainers:
cluster-api-docs-reviewers:
- killianmuldoon
cluster-api-docs-maintainers:

# -----------------------------------------------------------
# OWNER_ALIASES for exp/state-metrics
# -----------------------------------------------------------

cluster-api-state-metrics-maintainers:
cluster-api-state-metrics-reviewers:
- chrischdi
- johannesfrey
- seanschneeweiss
- tobiasgiese
63 changes: 63 additions & 0 deletions exp/state-metrics/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# syntax=docker/dockerfile:1.1-experimental

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Build the manager binary
# Run this with docker build --build-arg builder_image=<golang:x.y.z>
ARG builder_image
FROM ${builder_image} as builder
WORKDIR /workspace

# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
ARG goproxy=https://proxy.golang.org
# Run this with docker build --build-arg package=./controlplane/kubeadm or --build-arg package=./bootstrap/kubeadm
ENV GOPROXY=$goproxy

# Copy the Go Modules manifests
COPY go.mod go.mod
COPY go.sum go.sum

# Cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
RUN --mount=type=cache,target=/go/pkg/mod \
go mod download

# Copy the sources
COPY ./ ./

# Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
go build .

# Build
ARG package=.
ARG ARCH
ARG ldflags

# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
RUN --mount=type=cache,target=/root/.cache/go-build \
cd ${package} && \
CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
-o /workspace/manager .

# Production image
FROM gcr.io/distroless/static:nonroot
WORKDIR /
COPY --from=builder /workspace/manager .
# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
USER 65532
ENTRYPOINT ["/manager"]
8 changes: 8 additions & 0 deletions exp/state-metrics/OWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# See the OWNERS docs at https://go.k8s.io/owners

approvers:
- cluster-api-state-metrics-maintainers

reviewers:
- cluster-api-reviewers
- cluster-api-state-metrics-reviewers
63 changes: 63 additions & 0 deletions exp/state-metrics/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# Cluster API State Metrics

## Overview

Cluster API State Metrics (CASM) is a service that listens to the Kubernetes API server and generates metrics about the state of custom resource objects related of [Kubernetes Cluster API].
This project is highly inspired by [kube-state-metrics] and shares some codebase with it and resources which are in scope for kube-state-metrics are not scope of cluster-api-state-metrics.

The metrics are exported on the HTTP endpoint `/metrics` via http (default port `8080`) and are served as plaintext.
The endpoint is designed to get consumed by Prometheus or a scraper which is compatible with a Prometheus client endpoint.
Kubernetes custom resource objects which get deleted are no longer visible to the `/metrics` endpoint.

[Kubernetes Cluster API]: https://cluster-api.sigs.k8s.io/
[kube-state-metrics]: https://github.com/kubernetes/kube-state-metrics

# Usage Documentation

## CLI Arguments

```txt
cluster-api-state-metrics -h
Usage of ./bin/cluster-api-state-metrics:
--add_dir_header If true, adds the file directory to the header of the log messages
--alsologtostderr log to standard error as well as files
--apiserver string The URL of the apiserver to use as a master
--enable-gzip-encoding Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.
-h, --help Print Help text
--host string Host to expose metrics on. (default "::")
--kubeconfig string Absolute path to the kubeconfig file
--log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0)
--log_dir string If non-empty, write log files in this directory
--log_file string If non-empty, use this log file
--log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
--logtostderr log to standard error instead of files (default true)
--metric-allowlist string Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.
--metric-annotations-allowlist string Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional annotations provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').
--metric-denylist string Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.
--metric-labels-allowlist string Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]').
--metric-opt-in-list string Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists
--namespaces string Comma-separated list of namespaces to be enabled. Defaults to ""
--namespaces-denylist string Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.
--one_output If true, only write logs to their native severity level (vs also writing to each lower severity level)
--pod string Name of the pod that contains the kube-state-metrics container. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
--pod-namespace string Name of the namespace of the pod specified by --pod. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
--port int Port to expose metrics on. (default 8080)
--resources string Comma-separated list of Resources to be enabled. Defaults to "clusters,kubeadmcontrolplanes,machinedeployments,machines,machinesets"
--shard int32 The instances shard nominal (zero indexed) within the total number of shards. (default 0)
--skip_headers If true, avoid header prefixes in the log messages
--skip_log_headers If true, avoid headers when opening log files
--stderrthreshold severity logs at or above this threshold go to stderr (default 2)
--telemetry-host string Host to expose kube-state-metrics self metrics on. (default "::")
--telemetry-port int Port to expose kube-state-metrics self metrics on. (default 8081)
--tls-config string Path to the TLS configuration file
--total-shards int The total number of shards. Sharding is disabled when total shards is set to 1. (default 1)
--use-apiserver-cache Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.
-v, --v Level number for the log level verbosity
--version kube-state-metrics build version information
--vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging
```

# Metrics Documentation

Metrics will be made available on port 8080 by default. Alternatively it is possible to pass the command line flag `-addr` to override the port.
An overview of all metrics can be found in [metrics.md](docs/README.md).
28 changes: 28 additions & 0 deletions exp/state-metrics/config/default/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

# Adds namespace to all resources.
namespace: casm-system

# Value of this field is prepended to the
# names of all resources, e.g. a deployment named
# "wordpress" becomes "alices-wordpress".
# Note that it should also match with the prefix (text before '-') of the namespace
# field above.
namePrefix: cluster-api-

# Labels to add to all resources and selectors.
commonLabels:
app: cluster-api-state-metrics

resources:
- metrics_service.yaml
- namespace.yaml

bases:
- ../rbac
- ../manager

patchesStrategicMerge:
- manager_pull_policy.yaml

11 changes: 11 additions & 0 deletions exp/state-metrics/config/default/manager_image_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
spec:
template:
spec:
containers:
- image: gcr.io/k8s-staging-cluster-api/state-metrics:main
name: manager
11 changes: 11 additions & 0 deletions exp/state-metrics/config/default/manager_pull_policy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: state-metrics
namespace: system
spec:
template:
spec:
containers:
- name: manager
imagePullPolicy: Always
31 changes: 31 additions & 0 deletions exp/state-metrics/config/default/metrics_service.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# service for scraping cluster-api-state-metrics
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "8080"
prometheus.io/scheme: "http"
name: state-metrics-service
spec:
ports:
- name: metrics
port: 8080
targetPort: metrics
selector:
---
# service for scraping the telemetry endpoint of cluster-api-state-metrics
# which exposes internal metrics specific to the exporter itself
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "8081"
prometheus.io/scheme: "http"
name: state-metrics-service-telemetry
spec:
ports:
- name: telemetry
port: 8081
targetPort: telemetry
4 changes: 4 additions & 0 deletions exp/state-metrics/config/default/namespace.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: casm-system
11 changes: 11 additions & 0 deletions exp/state-metrics/config/manager/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- manager.yaml

generatorOptions:
disableNameSuffixHash: true
images:
- name: controller
newName: controller
newTag: latest
Loading

0 comments on commit 747ef52

Please sign in to comment.