Skip to content

Commit

Permalink
cherry pick #2750 to release-1.1 (#2782)
Browse files Browse the repository at this point in the history 
Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

Co-authored-by: Song Gao <disxiaofei@163.com>
  • Loading branch information
@ti-srebot @Yisaer
ti-srebot and Yisaer authored Jun 19, 2020
1 parent 70ac92e commit 80d29c6
Show file tree
Hide file tree
Showing 3 changed files with 286 additions and 31 deletions.
117 changes: 87 additions & 30 deletions pkg/monitor/monitor/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,20 @@ package monitor

import (
"fmt"
"path"
"strings"
"time"

"github.com/pingcap/tidb-operator/pkg/util"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/config"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/klog"
"path"
"time"
)

const (
nameLabel = "__meta_kubernetes_pod_label_app_kubernetes_io_name"
instanceLabel = "__meta_kubernetes_pod_label_app_kubernetes_io_instance"
componentLabel = "__meta_kubernetes_pod_label_app_kubernetes_io_component"
scrapeLabel = "__meta_kubernetes_pod_annotation_prometheus_io_scrape"
Expand All @@ -45,6 +48,8 @@ var (
tidbPattern config.Regexp
addressPattern config.Regexp
tiflashPattern config.Regexp
pumpPattern config.Regexp
drainerPattern config.Regexp
dashBoardConfig = `{
"apiVersion": 1,
"providers": [
Expand Down Expand Up @@ -95,6 +100,14 @@ func init() {
if err != nil {
klog.Fatalf("monitor regex template parse error,%v", err)
}
pumpPattern, err = config.NewRegexp("pump")
if err != nil {
klog.Fatalf("monitor regex template parse error,%v", err)
}
drainerPattern, err = config.NewRegexp("drainer")
if err != nil {
klog.Fatalf("monitor regex template parse error,%v", err)
}
}

type MonitorConfigModel struct {
Expand All @@ -105,13 +118,6 @@ type MonitorConfigModel struct {
}

func newPrometheusConfig(cmodel *MonitorConfigModel) *config.Config {
pdReplacement := fmt.Sprintf("$1.$2-%s-peer:$3", "pd")
tikvReplacement := fmt.Sprintf("$1.$2-%s-peer:$3", "tikv")
tidbReplacement := fmt.Sprintf("$1.$2-%s-peer:$3", "tidb")
tiflashReplacement := fmt.Sprintf("$1.$2-%s-peer:$3", "tiflash")
tiflashProxyReplacement := fmt.Sprintf("$1.$2-%s-peer:$3", "tiflash")
tiflashProxyPortLabel := fmt.Sprintf(additionalPortLabelPattern, "tiflash_proxy")

var c = config.Config{
GlobalConfig: config.GlobalConfig{
ScrapeInterval: model.Duration(15 * time.Second),
Expand All @@ -121,41 +127,91 @@ func newPrometheusConfig(cmodel *MonitorConfigModel) *config.Config {
"/prometheus-rules/rules/*.rules.yml",
},
ScrapeConfigs: []*config.ScrapeConfig{
scrapeJob("pd", pdPattern, cmodel, buildAddressRelabelConfig(portLabel, pdReplacement, true)),
scrapeJob("tidb", tidbPattern, cmodel, buildAddressRelabelConfig(portLabel, tidbReplacement, true)),
scrapeJob("tikv", tikvPattern, cmodel, buildAddressRelabelConfig(portLabel, tikvReplacement, true)),
scrapeJob("tiflash", tiflashPattern, cmodel, buildAddressRelabelConfig(portLabel, tiflashReplacement, true)),
scrapeJob("tiflash-proxy", tiflashPattern, cmodel, buildAddressRelabelConfig(tiflashProxyPortLabel, tiflashProxyReplacement, true)),
scrapeJob("pd", pdPattern, cmodel, buildAddressRelabelConfigByComponent("pd")),
scrapeJob("tidb", tidbPattern, cmodel, buildAddressRelabelConfigByComponent("tidb")),
scrapeJob("tikv", tikvPattern, cmodel, buildAddressRelabelConfigByComponent("tikv")),
scrapeJob("tiflash", tiflashPattern, cmodel, buildAddressRelabelConfigByComponent("tiflash")),
scrapeJob("tiflash-proxy", tiflashPattern, cmodel, buildAddressRelabelConfigByComponent("tiflash-proxy")),
scrapeJob("pump", pumpPattern, cmodel, buildAddressRelabelConfigByComponent("pump")),
scrapeJob("drainer", drainerPattern, cmodel, buildAddressRelabelConfigByComponent("drainer")),
},
}
return &c
}

func buildAddressRelabelConfig(portLabelName, replacement string, isTidbClusterComponent bool) *config.RelabelConfig {
addressRelabelConfig := &config.RelabelConfig{
SourceLabels: model.LabelNames{
"__address__",
model.LabelName(portLabelName),
},
Action: config.RelabelReplace,
Regex: portPattern,
Replacement: "$1:$2",
TargetLabel: "__address__",
}
if isTidbClusterComponent {
addressRelabelConfig = &config.RelabelConfig{
func buildAddressRelabelConfigByComponent(kind string) *config.RelabelConfig {
kind = strings.ToLower(kind)
replacement := fmt.Sprintf("$1.$2-%s-peer:$3", kind)
f := func() *config.RelabelConfig {
return &config.RelabelConfig{
Action: config.RelabelReplace,
Regex: addressPattern,
Replacement: replacement,
TargetLabel: "__address__",
SourceLabels: model.LabelNames{
podNameLabel,
instanceLabel,
model.LabelName(portLabelName),
portLabel,
},
}
}
return addressRelabelConfig
switch strings.ToLower(kind) {
case "pd":
return f()
case "tidb":
return f()
case "tikv":
return f()
case "tiflash":
return f()
case "tiflash-proxy":
return &config.RelabelConfig{
Action: config.RelabelReplace,
Regex: addressPattern,
Replacement: "$1.$2-tiflash-peer:$3",
TargetLabel: "__address__",
SourceLabels: model.LabelNames{
podNameLabel,
instanceLabel,
model.LabelName(fmt.Sprintf(additionalPortLabelPattern, "tiflash_proxy")),
},
}
case "pump":
return &config.RelabelConfig{
Action: config.RelabelReplace,
Regex: addressPattern,
Replacement: "$1.$2-pump:$3",
TargetLabel: "__address__",
SourceLabels: model.LabelNames{
podNameLabel,
instanceLabel,
portLabel,
},
}
case "drainer":
return &config.RelabelConfig{
Action: config.RelabelReplace,
Regex: addressPattern,
Replacement: "$1.$2:$3",
TargetLabel: "__address__",
SourceLabels: model.LabelNames{
podNameLabel,
nameLabel,
portLabel,
},
}
default:
return &config.RelabelConfig{
SourceLabels: model.LabelNames{
"__address__",
portLabel,
},
Action: config.RelabelReplace,
Regex: portPattern,
Replacement: "$1:$2",
TargetLabel: "__address__",
}
}
}

func scrapeJob(jobName string, componentPattern config.Regexp, cmodel *MonitorConfigModel, addressRelabelConfig *config.RelabelConfig) *config.ScrapeConfig {
Expand Down Expand Up @@ -273,7 +329,8 @@ func addTlsConfig(pc *config.Config) {

for id, sconfig := range pc.ScrapeConfigs {
// TODO support tiflash tls when it gets ready
if sconfig.JobName == "pd" || sconfig.JobName == "tidb" || sconfig.JobName == "tikv" {
if sconfig.JobName == "pd" || sconfig.JobName == "tidb" || sconfig.JobName == "tikv" ||
sconfig.JobName == "pump" || sconfig.JobName == "drainer" {
sconfig.HTTPClientConfig.TLSConfig = config.TLSConfig{
CAFile: path.Join(util.ClusterClientTLSPath, corev1.ServiceAccountRootCAKey),
CertFile: path.Join(util.ClusterClientTLSPath, corev1.TLSCertKey),
Expand Down
198 changes: 198 additions & 0 deletions pkg/monitor/monitor/template_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,102 @@ scrape_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
- job_name: pump
honor_labels: true
scrape_interval: 15s
scheme: http
kubernetes_sd_configs:
- api_server: null
role: pod
namespaces:
names:
- ns1
- ns2
tls_config:
insecure_skip_verify: true
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
regex: target
action: keep
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
regex: pump
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
regex: "true"
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
regex: (.+)
target_label: __metrics_path__
action: replace
- source_labels: [__meta_kubernetes_pod_name, __meta_kubernetes_pod_label_app_kubernetes_io_instance,
__meta_kubernetes_pod_annotation_prometheus_io_port]
regex: (.+);(.+);(.+)
target_label: __address__
replacement: $1.$2-pump:$3
action: replace
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
action: replace
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
action: replace
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
action: replace
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
- job_name: drainer
honor_labels: true
scrape_interval: 15s
scheme: http
kubernetes_sd_configs:
- api_server: null
role: pod
namespaces:
names:
- ns1
- ns2
tls_config:
insecure_skip_verify: true
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
regex: target
action: keep
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
regex: drainer
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
regex: "true"
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
regex: (.+)
target_label: __metrics_path__
action: replace
- source_labels: [__meta_kubernetes_pod_name, __meta_kubernetes_pod_label_app_kubernetes_io_name,
__meta_kubernetes_pod_annotation_prometheus_io_port]
regex: (.+);(.+);(.+)
target_label: __address__
replacement: $1.$2:$3
action: replace
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
action: replace
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
action: replace
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
action: replace
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
`
model := &MonitorConfigModel{
ReleaseTargetRegex: &target,
Expand Down Expand Up @@ -546,6 +642,108 @@ scrape_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
- job_name: pump
honor_labels: true
scrape_interval: 15s
scheme: https
kubernetes_sd_configs:
- api_server: null
role: pod
namespaces:
names:
- ns1
- ns2
tls_config:
ca_file: /var/lib/cluster-client-tls/ca.crt
cert_file: /var/lib/cluster-client-tls/tls.crt
key_file: /var/lib/cluster-client-tls/tls.key
insecure_skip_verify: false
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
regex: target
action: keep
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
regex: pump
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
regex: "true"
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
regex: (.+)
target_label: __metrics_path__
action: replace
- source_labels: [__meta_kubernetes_pod_name, __meta_kubernetes_pod_label_app_kubernetes_io_instance,
__meta_kubernetes_pod_annotation_prometheus_io_port]
regex: (.+);(.+);(.+)
target_label: __address__
replacement: $1.$2-pump:$3
action: replace
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
action: replace
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
action: replace
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
action: replace
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
- job_name: drainer
honor_labels: true
scrape_interval: 15s
scheme: https
kubernetes_sd_configs:
- api_server: null
role: pod
namespaces:
names:
- ns1
- ns2
tls_config:
ca_file: /var/lib/cluster-client-tls/ca.crt
cert_file: /var/lib/cluster-client-tls/tls.crt
key_file: /var/lib/cluster-client-tls/tls.key
insecure_skip_verify: false
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
regex: target
action: keep
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
regex: drainer
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
regex: "true"
action: keep
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
regex: (.+)
target_label: __metrics_path__
action: replace
- source_labels: [__meta_kubernetes_pod_name, __meta_kubernetes_pod_label_app_kubernetes_io_name,
__meta_kubernetes_pod_annotation_prometheus_io_port]
regex: (.+);(.+);(.+)
target_label: __address__
replacement: $1.$2:$3
action: replace
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
action: replace
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
action: replace
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
action: replace
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
target_label: cluster
action: replace
`
model := &MonitorConfigModel{
ReleaseTargetRegex: &target,
Expand Down
Loading

0 comments on commit 80d29c6

Please sign in to comment.