cluster: add ng-monitoring support

embed/templates/config/ngmonitoring.toml.tpl

@@ -0,0 +1,26 @@
+# NG Monitoring Server Configuration.
+
+# Server address.
+address = "0.0.0.0:{{.Port}}"
+advertise-address = "{{.IP}}:{{.Port}}"
+
+[log]
+# Log path
+path = "{{.LogDir}}"
+
+# Log level: INFO, WARN, ERROR
+level = "INFO"
+
+[pd]
+# Addresses of PD instances within the TiDB cluster. Multiple addresses are separated by commas, e.g. "10.0.0.1:2379","10.0.0.2:2379"
+endpoints = [{{.PDAddrs}}]
+
+{{- if .TLSEnabled}}
+[security]
+ca-path = "{{.DeployDir}}/tls/ca.crt"
+cert-path = "{{.DeployDir}}/tls/prometheus.crt"
+key-path = "{{.DeployDir}}/tls/prometheus.pem"
+{{- end}}
+
+[storage]
+path = "{{.DataDir}}"

embed/templates/scripts/run_prometheus.sh.tpl

@@ -7,6 +7,29 @@ cd "${DEPLOY_DIR}" || exit 1
 # WARNING: This file was auto-generated. Do not edit!
 #          All your edit might be overwritten!
 
+if [ -e "bin/ng-monitoring-server" ]; then
+echo "#!/bin/bash
+
+# WARNING: This file was auto-generated to restart ng-monitoring when fail. 
+#          Do not edit! All your edit might be overwritten!
+
+while true
+do
+{{- if .NumaNode}}
+    numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/ng-monitoring-server \
+{{- else}}
+    bin/ng-monitoring-server \
+{{- end}}
+        --config {{.DeployDir}}/conf/ngmonitoring.toml \
+        >/dev/null 2>&1
+    sleep 15s
+done" > scripts/ng-wrapper.sh
+fi
+
+{{- if .EnableNG}}
+/bin/bash scripts/ng-wrapper.sh &
+{{- end}}
+
 exec > >(tee -i -a "{{.LogDir}}/prometheus.log")
 exec 2>&1
 

pkg/cluster/spec/prometheus.go → pkg/cluster/spec/monitoring.go

@@ -39,6 +39,7 @@ type PrometheusSpec struct {
 	Patched               bool                   `yaml:"patched,omitempty"`
 	IgnoreExporter        bool                   `yaml:"ignore_exporter,omitempty"`
 	Port                  int                    `yaml:"port" default:"9090"`
+	NgPort                int                    `yaml:"ng_port,omitempty" validate:"ng_port:editable"`
 	DeployDir             string                 `yaml:"deploy_dir,omitempty"`
 	DataDir               string                 `yaml:"data_dir,omitempty"`
 	LogDir                string                 `yaml:"log_dir,omitempty"`
@@ -108,7 +109,7 @@ func (c *MonitorComponent) Instances() []Instance {
 	ins := make([]Instance, 0, len(servers))
 
 	for _, s := range servers {
-		ins = append(ins, &MonitorInstance{BaseInstance{
+		mi := &MonitorInstance{BaseInstance{
 			InstanceSpec: s,
 			Name:         c.Name(),
 			Host:         s.Host,
@@ -128,7 +129,11 @@ func (c *MonitorComponent) Instances() []Instance {
 			UptimeFn: func(tlsCfg *tls.Config) time.Duration {
 				return UptimeByHost(s.Host, s.Port, tlsCfg)
 			},
-		}, c.Topology})
+		}, c.Topology}
+		if s.NgPort > 0 {
+			mi.BaseInstance.Ports = append(mi.BaseInstance.Ports, s.NgPort)
+		}
+		ins = append(ins, mi)
 	}
 	return ins
 }
@@ -163,7 +168,9 @@ func (i *MonitorInstance) InitConfig(
 		paths.Log,
 	).WithPort(spec.Port).
 		WithNumaNode(spec.NumaNode).
-		WithRetention(spec.Retention)
+		WithRetention(spec.Retention).
+		WithNG(spec.NgPort)
+
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_prometheus_%s_%d.sh", i.GetHost(), i.GetPort()))
 	if err := cfg.ConfigToFile(fp); err != nil {
 		return err
@@ -184,18 +191,20 @@ func (i *MonitorInstance) InitConfig(
 	monitoredOptions := i.topo.GetMonitoredOptions()
 
 	// transfer config
-	fp = filepath.Join(paths.Cache, fmt.Sprintf("prometheus_%s_%d.yml", i.GetHost(), i.GetPort()))
 	cfig := config.NewPrometheusConfig(clusterName, clusterVersion, enableTLS)
 	if monitoredOptions != nil {
 		cfig.AddBlackbox(i.GetHost(), uint64(monitoredOptions.BlackboxExporterPort))
 	}
 	uniqueHosts := set.NewStringSet()
 
+	ngcfg := config.NewNgMonitoringConfig(clusterName, clusterVersion, enableTLS)
+
 	if servers, found := topoHasField("PDServers"); found {
 		for i := 0; i < servers.Len(); i++ {
 			pd := servers.Index(i).Interface().(*PDSpec)
 			uniqueHosts.Insert(pd.Host)
 			cfig.AddPD(pd.Host, uint64(pd.ClientPort))
+			ngcfg.AddPD(pd.Host, uint64(pd.ClientPort))
 		}
 	}
 	if servers, found := topoHasField("TiKVServers"); found {
@@ -308,6 +317,22 @@ func (i *MonitorInstance) InitConfig(
 		return err
 	}
 
+	ngcfg.AddIP(i.GetHost())
+	ngcfg.AddPort(spec.NgPort)
+	ngcfg.AddDeployDir(paths.Deploy)
+	ngcfg.AddDataDir(paths.Data[0])
+	ngcfg.AddLog(paths.Log)
+
+	fp = filepath.Join(paths.Cache, fmt.Sprintf("ngmonitoring_%s_%d.toml", i.GetHost(), i.GetPort()))
+	if err := ngcfg.ConfigToFile(fp); err != nil {
+		return err
+	}
+	dst = filepath.Join(paths.Deploy, "conf", "ngmonitoring.toml")
+	if err := e.Transfer(ctx, fp, dst, false, 0); err != nil {
+		return err
+	}
+
+	fp = filepath.Join(paths.Cache, fmt.Sprintf("prometheus_%s_%d.yml", i.GetHost(), i.GetPort()))
 	if err := cfig.ConfigToFile(fp); err != nil {
 		return err
 	}

pkg/cluster/spec/validate.go

@@ -559,9 +559,14 @@ func (s *Specification) portInvalidDetect() error {
 		for i := 0; i < compSpec.NumField(); i++ {
 			if strings.HasSuffix(compSpec.Type().Field(i).Name, "Port") {
 				port := int(compSpec.Field(i).Int())
-				if port <= 0 || port >= 65535 {
+				portTags := strings.Split(compSpec.Type().Field(i).Tag.Get("yaml"), ",")
+				// when use not specify ng_port, its default value is 0
+				if port == 0 && len(portTags) > 1 && portTags[1] == "omitempty" {
+					continue
+				}
+				if port < 1 || port > 65535 {
 					portField := strings.Split(compSpec.Type().Field(i).Tag.Get("yaml"), ",")[0]
-					return errors.Errorf("`%s` of %s=%d is invalid, port should be in the range [0, 65535]", cfg, portField, port)
+					return errors.Errorf("`%s` of %s=%d is invalid, port should be in the range [1, 65535]", cfg, portField, port)
 				}
 			}
 		}
@@ -612,6 +617,7 @@ func (s *Specification) portConflictsDetect() error {
 		"TCPPort",
 		"HTTPPort",
 		"ClusterPort",
+		"NgPort",
 	}
 
 	portStats := map[usedPort]conflict{}

pkg/cluster/spec/validate_test.go

@@ -985,7 +985,7 @@ global:
   ssh_port: 65536
 `), &topo)
 	c.Assert(err, NotNil)
-	c.Assert(err.Error(), Equals, "`global` of ssh_port=65536 is invalid, port should be in the range [0, 65535]")
+	c.Assert(err.Error(), Equals, "`global` of ssh_port=65536 is invalid, port should be in the range [1, 65535]")
 
 	err = yaml.Unmarshal([]byte(`
 global:
@@ -995,14 +995,14 @@ tidb_servers:
     port: -1
 `), &topo)
 	c.Assert(err, NotNil)
-	c.Assert(err.Error(), Equals, "`tidb_servers` of port=-1 is invalid, port should be in the range [0, 65535]")
+	c.Assert(err.Error(), Equals, "`tidb_servers` of port=-1 is invalid, port should be in the range [1, 65535]")
 
 	err = yaml.Unmarshal([]byte(`
 monitored:
     node_exporter_port: 102400
 `), &topo)
 	c.Assert(err, NotNil)
-	c.Assert(err.Error(), Equals, "`monitored` of node_exporter_port=102400 is invalid, port should be in the range [0, 65535]")
+	c.Assert(err.Error(), Equals, "`monitored` of node_exporter_port=102400 is invalid, port should be in the range [1, 65535]")
 }
 
 func (s *metaSuiteTopo) TestInvalidUserGroup(c *C) {

pkg/cluster/template/config/ngmonitoring.go

@@ -0,0 +1,119 @@
+// Copyright 2020 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path"
+	"text/template"
+
+	"github.com/pingcap/tiup/embed"
+)
+
+// NgMonitoringConfig represent the data to generate NgMonitoring config
+type NgMonitoringConfig struct {
+	ClusterName string
+	TLSEnabled  bool
+	IP          string
+	Port        int
+	PDAddrs     string
+	DeployDir   string
+	DataDir     string
+	LogDir      string
+}
+
+// NewNgMonitoringConfig returns a PrometheusConfig
+func NewNgMonitoringConfig(clusterName, clusterVersion string, enableTLS bool) *NgMonitoringConfig {
+	cfg := &NgMonitoringConfig{
+		ClusterName: clusterName,
+		TLSEnabled:  enableTLS,
+	}
+	return cfg
+}
+
+// AddPD add a PD address
+func (c *NgMonitoringConfig) AddPD(ip string, port uint64) *NgMonitoringConfig {
+	if c.PDAddrs == "" {
+		c.PDAddrs = fmt.Sprintf("\"%s:%d\"", ip, port)
+	} else {
+		c.PDAddrs += fmt.Sprintf(",\"%s:%d\"", ip, port)
+	}
+	return c
+}
+
+// AddIP add ip to ng-monitoring conf
+func (c *NgMonitoringConfig) AddIP(ip string) *NgMonitoringConfig {
+	c.IP = ip
+	return c
+}
+
+// AddLog add logdir to ng-monitoring conf
+func (c *NgMonitoringConfig) AddLog(dir string) *NgMonitoringConfig {
+	c.LogDir = dir
+	return c
+}
+
+// AddDeployDir add logdir to ng-monitoring conf
+func (c *NgMonitoringConfig) AddDeployDir(dir string) *NgMonitoringConfig {
+	c.DeployDir = dir
+	return c
+}
+
+// AddDataDir add logdir to ng-monitoring conf
+func (c *NgMonitoringConfig) AddDataDir(dir string) *NgMonitoringConfig {
+	c.DataDir = dir
+	return c
+}
+
+// AddPort add port to ng-monitoring conf
+func (c *NgMonitoringConfig) AddPort(port int) *NgMonitoringConfig {
+	c.Port = port
+	return c
+}
+
+// ConfigWithTemplate generate the Prometheus config content by tpl
+func (c *NgMonitoringConfig) ConfigWithTemplate(tpl string) ([]byte, error) {
+	tmpl, err := template.New("NgMonitoring").Parse(tpl)
+	if err != nil {
+		return nil, err
+	}
+
+	content := bytes.NewBufferString("")
+	if err := tmpl.Execute(content, c); err != nil {
+		return nil, err
+	}
+
+	return content.Bytes(), nil
+}
+
+// Config generate the config file data.
+func (c *NgMonitoringConfig) Config() ([]byte, error) {
+	fp := path.Join("templates", "config", "ngmonitoring.toml.tpl")
+	tpl, err := embed.ReadTemplate(fp)
+	if err != nil {
+		return nil, err
+	}
+	return c.ConfigWithTemplate(string(tpl))
+}
+
+// ConfigToFile write config content to specific path
+func (c *NgMonitoringConfig) ConfigToFile(file string) error {
+	config, err := c.Config()
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(file, config, 0755)
+}

pkg/cluster/template/scripts/prometheus.go → pkg/cluster/template/scripts/monitoring.go

@@ -33,6 +33,7 @@ type PrometheusScript struct {
 	NumaNode  string
 	Retention string
 	tplFile   string
+	EnableNG  bool
 }
 
 // NewPrometheusScript returns a PrometheusScript with given arguments
@@ -75,6 +76,16 @@ func (c *PrometheusScript) WithTPLFile(fname string) *PrometheusScript {
 	return c
 }
 
+// WithNG set if enable ng-monitoring.
+func (c *PrometheusScript) WithNG(ngPort int) *PrometheusScript {
+	if ngPort > 0 {
+		c.EnableNG = true
+	} else {
+		c.EnableNG = false
+	}
+	return c
+}
+
 // Config generate the config file data.
 func (c *PrometheusScript) Config() ([]byte, error) {
 	fp := c.tplFile