Skip to content

Commit

Permalink
cluster: add ng-monitoring support (#1601)
Browse files Browse the repository at this point in the history
  • Loading branch information
nexustar authored Nov 11, 2021
1 parent ab0e748 commit 01843be
Show file tree
Hide file tree
Showing 9 changed files with 219 additions and 9 deletions.
26 changes: 26 additions & 0 deletions embed/templates/config/ngmonitoring.toml.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# NG Monitoring Server Configuration.

# Server address.
address = "0.0.0.0:{{.Port}}"
advertise-address = "{{.IP}}:{{.Port}}"

[log]
# Log path
path = "{{.LogDir}}"

# Log level: INFO, WARN, ERROR
level = "INFO"

[pd]
# Addresses of PD instances within the TiDB cluster. Multiple addresses are separated by commas, e.g. "10.0.0.1:2379","10.0.0.2:2379"
endpoints = [{{.PDAddrs}}]

{{- if .TLSEnabled}}
[security]
ca-path = "{{.DeployDir}}/tls/ca.crt"
cert-path = "{{.DeployDir}}/tls/prometheus.crt"
key-path = "{{.DeployDir}}/tls/prometheus.pem"
{{- end}}

[storage]
path = "{{.DataDir}}"
23 changes: 23 additions & 0 deletions embed/templates/scripts/run_prometheus.sh.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,29 @@ cd "${DEPLOY_DIR}" || exit 1
# WARNING: This file was auto-generated. Do not edit!
# All your edit might be overwritten!

if [ -e "bin/ng-monitoring-server" ]; then
echo "#!/bin/bash
# WARNING: This file was auto-generated to restart ng-monitoring when fail.
# Do not edit! All your edit might be overwritten!
while true
do
{{- if .NumaNode}}
numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/ng-monitoring-server \
{{- else}}
bin/ng-monitoring-server \
{{- end}}
--config {{.DeployDir}}/conf/ngmonitoring.toml \
>/dev/null 2>&1
sleep 15s
done" > scripts/ng-wrapper.sh
fi

{{- if .EnableNG}}
/bin/bash scripts/ng-wrapper.sh &
{{- end}}

exec > >(tee -i -a "{{.LogDir}}/prometheus.log")
exec 2>&1

Expand Down
33 changes: 29 additions & 4 deletions pkg/cluster/spec/prometheus.go → pkg/cluster/spec/monitoring.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type PrometheusSpec struct {
Patched bool `yaml:"patched,omitempty"`
IgnoreExporter bool `yaml:"ignore_exporter,omitempty"`
Port int `yaml:"port" default:"9090"`
NgPort int `yaml:"ng_port,omitempty" validate:"ng_port:editable"`
DeployDir string `yaml:"deploy_dir,omitempty"`
DataDir string `yaml:"data_dir,omitempty"`
LogDir string `yaml:"log_dir,omitempty"`
Expand Down Expand Up @@ -108,7 +109,7 @@ func (c *MonitorComponent) Instances() []Instance {
ins := make([]Instance, 0, len(servers))

for _, s := range servers {
ins = append(ins, &MonitorInstance{BaseInstance{
mi := &MonitorInstance{BaseInstance{
InstanceSpec: s,
Name: c.Name(),
Host: s.Host,
Expand All @@ -128,7 +129,11 @@ func (c *MonitorComponent) Instances() []Instance {
UptimeFn: func(tlsCfg *tls.Config) time.Duration {
return UptimeByHost(s.Host, s.Port, tlsCfg)
},
}, c.Topology})
}, c.Topology}
if s.NgPort > 0 {
mi.BaseInstance.Ports = append(mi.BaseInstance.Ports, s.NgPort)
}
ins = append(ins, mi)
}
return ins
}
Expand Down Expand Up @@ -163,7 +168,9 @@ func (i *MonitorInstance) InitConfig(
paths.Log,
).WithPort(spec.Port).
WithNumaNode(spec.NumaNode).
WithRetention(spec.Retention)
WithRetention(spec.Retention).
WithNG(spec.NgPort)

fp := filepath.Join(paths.Cache, fmt.Sprintf("run_prometheus_%s_%d.sh", i.GetHost(), i.GetPort()))
if err := cfg.ConfigToFile(fp); err != nil {
return err
Expand All @@ -184,18 +191,20 @@ func (i *MonitorInstance) InitConfig(
monitoredOptions := i.topo.GetMonitoredOptions()

// transfer config
fp = filepath.Join(paths.Cache, fmt.Sprintf("prometheus_%s_%d.yml", i.GetHost(), i.GetPort()))
cfig := config.NewPrometheusConfig(clusterName, clusterVersion, enableTLS)
if monitoredOptions != nil {
cfig.AddBlackbox(i.GetHost(), uint64(monitoredOptions.BlackboxExporterPort))
}
uniqueHosts := set.NewStringSet()

ngcfg := config.NewNgMonitoringConfig(clusterName, clusterVersion, enableTLS)

if servers, found := topoHasField("PDServers"); found {
for i := 0; i < servers.Len(); i++ {
pd := servers.Index(i).Interface().(*PDSpec)
uniqueHosts.Insert(pd.Host)
cfig.AddPD(pd.Host, uint64(pd.ClientPort))
ngcfg.AddPD(pd.Host, uint64(pd.ClientPort))
}
}
if servers, found := topoHasField("TiKVServers"); found {
Expand Down Expand Up @@ -308,6 +317,22 @@ func (i *MonitorInstance) InitConfig(
return err
}

ngcfg.AddIP(i.GetHost())
ngcfg.AddPort(spec.NgPort)
ngcfg.AddDeployDir(paths.Deploy)
ngcfg.AddDataDir(paths.Data[0])
ngcfg.AddLog(paths.Log)

fp = filepath.Join(paths.Cache, fmt.Sprintf("ngmonitoring_%s_%d.toml", i.GetHost(), i.GetPort()))
if err := ngcfg.ConfigToFile(fp); err != nil {
return err
}
dst = filepath.Join(paths.Deploy, "conf", "ngmonitoring.toml")
if err := e.Transfer(ctx, fp, dst, false, 0); err != nil {
return err
}

fp = filepath.Join(paths.Cache, fmt.Sprintf("prometheus_%s_%d.yml", i.GetHost(), i.GetPort()))
if err := cfig.ConfigToFile(fp); err != nil {
return err
}
Expand Down
File renamed without changes.
10 changes: 8 additions & 2 deletions pkg/cluster/spec/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -559,9 +559,14 @@ func (s *Specification) portInvalidDetect() error {
for i := 0; i < compSpec.NumField(); i++ {
if strings.HasSuffix(compSpec.Type().Field(i).Name, "Port") {
port := int(compSpec.Field(i).Int())
if port <= 0 || port >= 65535 {
portTags := strings.Split(compSpec.Type().Field(i).Tag.Get("yaml"), ",")
// when use not specify ng_port, its default value is 0
if port == 0 && len(portTags) > 1 && portTags[1] == "omitempty" {
continue
}
if port < 1 || port > 65535 {
portField := strings.Split(compSpec.Type().Field(i).Tag.Get("yaml"), ",")[0]
return errors.Errorf("`%s` of %s=%d is invalid, port should be in the range [0, 65535]", cfg, portField, port)
return errors.Errorf("`%s` of %s=%d is invalid, port should be in the range [1, 65535]", cfg, portField, port)
}
}
}
Expand Down Expand Up @@ -612,6 +617,7 @@ func (s *Specification) portConflictsDetect() error {
"TCPPort",
"HTTPPort",
"ClusterPort",
"NgPort",
}

portStats := map[usedPort]conflict{}
Expand Down
6 changes: 3 additions & 3 deletions pkg/cluster/spec/validate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -985,7 +985,7 @@ global:
ssh_port: 65536
`), &topo)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "`global` of ssh_port=65536 is invalid, port should be in the range [0, 65535]")
c.Assert(err.Error(), Equals, "`global` of ssh_port=65536 is invalid, port should be in the range [1, 65535]")

err = yaml.Unmarshal([]byte(`
global:
Expand All @@ -995,14 +995,14 @@ tidb_servers:
port: -1
`), &topo)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "`tidb_servers` of port=-1 is invalid, port should be in the range [0, 65535]")
c.Assert(err.Error(), Equals, "`tidb_servers` of port=-1 is invalid, port should be in the range [1, 65535]")

err = yaml.Unmarshal([]byte(`
monitored:
node_exporter_port: 102400
`), &topo)
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "`monitored` of node_exporter_port=102400 is invalid, port should be in the range [0, 65535]")
c.Assert(err.Error(), Equals, "`monitored` of node_exporter_port=102400 is invalid, port should be in the range [1, 65535]")
}

func (s *metaSuiteTopo) TestInvalidUserGroup(c *C) {
Expand Down
119 changes: 119 additions & 0 deletions pkg/cluster/template/config/ngmonitoring.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
"bytes"
"fmt"
"os"
"path"
"text/template"

"github.com/pingcap/tiup/embed"
)

// NgMonitoringConfig represent the data to generate NgMonitoring config
type NgMonitoringConfig struct {
ClusterName string
TLSEnabled bool
IP string
Port int
PDAddrs string
DeployDir string
DataDir string
LogDir string
}

// NewNgMonitoringConfig returns a PrometheusConfig
func NewNgMonitoringConfig(clusterName, clusterVersion string, enableTLS bool) *NgMonitoringConfig {
cfg := &NgMonitoringConfig{
ClusterName: clusterName,
TLSEnabled: enableTLS,
}
return cfg
}

// AddPD add a PD address
func (c *NgMonitoringConfig) AddPD(ip string, port uint64) *NgMonitoringConfig {
if c.PDAddrs == "" {
c.PDAddrs = fmt.Sprintf("\"%s:%d\"", ip, port)
} else {
c.PDAddrs += fmt.Sprintf(",\"%s:%d\"", ip, port)
}
return c
}

// AddIP add ip to ng-monitoring conf
func (c *NgMonitoringConfig) AddIP(ip string) *NgMonitoringConfig {
c.IP = ip
return c
}

// AddLog add logdir to ng-monitoring conf
func (c *NgMonitoringConfig) AddLog(dir string) *NgMonitoringConfig {
c.LogDir = dir
return c
}

// AddDeployDir add logdir to ng-monitoring conf
func (c *NgMonitoringConfig) AddDeployDir(dir string) *NgMonitoringConfig {
c.DeployDir = dir
return c
}

// AddDataDir add logdir to ng-monitoring conf
func (c *NgMonitoringConfig) AddDataDir(dir string) *NgMonitoringConfig {
c.DataDir = dir
return c
}

// AddPort add port to ng-monitoring conf
func (c *NgMonitoringConfig) AddPort(port int) *NgMonitoringConfig {
c.Port = port
return c
}

// ConfigWithTemplate generate the Prometheus config content by tpl
func (c *NgMonitoringConfig) ConfigWithTemplate(tpl string) ([]byte, error) {
tmpl, err := template.New("NgMonitoring").Parse(tpl)
if err != nil {
return nil, err
}

content := bytes.NewBufferString("")
if err := tmpl.Execute(content, c); err != nil {
return nil, err
}

return content.Bytes(), nil
}

// Config generate the config file data.
func (c *NgMonitoringConfig) Config() ([]byte, error) {
fp := path.Join("templates", "config", "ngmonitoring.toml.tpl")
tpl, err := embed.ReadTemplate(fp)
if err != nil {
return nil, err
}
return c.ConfigWithTemplate(string(tpl))
}

// ConfigToFile write config content to specific path
func (c *NgMonitoringConfig) ConfigToFile(file string) error {
config, err := c.Config()
if err != nil {
return err
}
return os.WriteFile(file, config, 0755)
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type PrometheusScript struct {
NumaNode string
Retention string
tplFile string
EnableNG bool
}

// NewPrometheusScript returns a PrometheusScript with given arguments
Expand Down Expand Up @@ -75,6 +76,16 @@ func (c *PrometheusScript) WithTPLFile(fname string) *PrometheusScript {
return c
}

// WithNG set if enable ng-monitoring.
func (c *PrometheusScript) WithNG(ngPort int) *PrometheusScript {
if ngPort > 0 {
c.EnableNG = true
} else {
c.EnableNG = false
}
return c
}

// Config generate the config file data.
func (c *PrometheusScript) Config() ([]byte, error) {
fp := c.tplFile
Expand Down

0 comments on commit 01843be

Please sign in to comment.