cluster: add ng-monitoring support

embed/templates/config/ngmonitoring.toml.tpl

@@ -0,0 +1,26 @@
+# NG Monitoring Server Configuration.
+
+# Server address.
+address = "0.0.0.0:{{.Port}}"
+advertise-address = "{{.IP}}:{{.Port}}"
+
+[log]
+# Log path
+path = "{{.LogDir}}"
+
+# Log level: INFO, WARN, ERROR
+level = "INFO"
+
+[pd]
+# Addresses of PD instances within the TiDB cluster. Multiple addresses are separated by commas, e.g. "10.0.0.1:2379","10.0.0.2:2379"
+endpoints = [{{.PDAddrs}}]
+
+{{- if .TLSEnabled}}
+[security]
+ca-path = "{{.DeployDir}}/tls/ca.crt"
+cert-path = "{{.DeployDir}}/tls/prometheus.crt"
+key-path = "{{.DeployDir}}/tls/prometheus.pem"
+{{- end}}
+
+[storage]
+path = "{{.DataDir}}"

embed/templates/scripts/run_prometheus.sh.tpl

@@ -7,6 +7,26 @@ cd "${DEPLOY_DIR}" || exit 1
 # WARNING: This file was auto-generated. Do not edit!
 #          All your edit might be overwritten!
 
+if [ -e "bin/ng-monitoring-server" ]; then
+echo "#!/bin/bash
+
+# WARNING: This file was auto-generated to restart ng-monitoring when fail. 
+#          Do not edit! All your edit might be overwritten!
+
+while true
+do
+{{- if .NumaNode}}
+    numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/ng-monitoring-server \
+{{- else}}
+    bin/ng-monitoring-server \
+{{- end}}
+        --config {{.DeployDir}}/conf/ngmonitoring.toml \
+        >/dev/null 2>&1
+done" > bin/ng-wrapper.sh
+fi
+
+/bin/bash bin/ng-wrapper.sh &
+
 exec > >(tee -i -a "{{.LogDir}}/prometheus.log")
 exec 2>&1
 

pkg/cluster/spec/prometheus.go → pkg/cluster/spec/monitoring.go

@@ -39,6 +39,7 @@ type PrometheusSpec struct {
 	Patched               bool                   `yaml:"patched,omitempty"`
 	IgnoreExporter        bool                   `yaml:"ignore_exporter,omitempty"`
 	Port                  int                    `yaml:"port" default:"9090"`
+	NgPort                int                    `yaml:"ng_port" default:"8428"`
 	DeployDir             string                 `yaml:"deploy_dir,omitempty"`
 	DataDir               string                 `yaml:"data_dir,omitempty"`
 	LogDir                string                 `yaml:"log_dir,omitempty"`
@@ -117,6 +118,7 @@ func (c *MonitorComponent) Instances() []Instance {
 
 			Ports: []int{
 				s.Port,
+				s.NgPort,
 			},
 			Dirs: []string{
 				s.DeployDir,
@@ -164,6 +166,7 @@ func (i *MonitorInstance) InitConfig(
 	).WithPort(spec.Port).
 		WithNumaNode(spec.NumaNode).
 		WithRetention(spec.Retention)
+
 	fp := filepath.Join(paths.Cache, fmt.Sprintf("run_prometheus_%s_%d.sh", i.GetHost(), i.GetPort()))
 	if err := cfg.ConfigToFile(fp); err != nil {
 		return err
@@ -184,18 +187,20 @@ func (i *MonitorInstance) InitConfig(
 	monitoredOptions := i.topo.GetMonitoredOptions()
 
 	// transfer config
-	fp = filepath.Join(paths.Cache, fmt.Sprintf("prometheus_%s_%d.yml", i.GetHost(), i.GetPort()))
 	cfig := config.NewPrometheusConfig(clusterName, clusterVersion, enableTLS)
 	if monitoredOptions != nil {
 		cfig.AddBlackbox(i.GetHost(), uint64(monitoredOptions.BlackboxExporterPort))
 	}
 	uniqueHosts := set.NewStringSet()
 
+	ngcfg := config.NewNgMonitoringConfig(clusterName, clusterVersion, enableTLS)
+
 	if servers, found := topoHasField("PDServers"); found {
 		for i := 0; i < servers.Len(); i++ {
 			pd := servers.Index(i).Interface().(*PDSpec)
 			uniqueHosts.Insert(pd.Host)
 			cfig.AddPD(pd.Host, uint64(pd.ClientPort))
+			ngcfg.AddPD(pd.Host, uint64(pd.ClientPort))
 		}
 	}
 	if servers, found := topoHasField("TiKVServers"); found {
@@ -308,6 +313,21 @@ func (i *MonitorInstance) InitConfig(
 		return err
 	}
 
+	ngcfg.AddIP(i.GetHost())
+	ngcfg.AddPort(spec.NgPort)
+	ngcfg.AddDeployDir(paths.Deploy)
+	ngcfg.AddDataDir(paths.Data[0])
+	ngcfg.AddLog(paths.Log)
+	fp = filepath.Join(paths.Cache, fmt.Sprintf("ngmonitoring_%s_%d.toml", i.GetHost(), i.GetPort()))
+	if err := ngcfg.ConfigToFile(fp); err != nil {
+		return err
+	}
+	dst = filepath.Join(paths.Deploy, "conf", "ngmonitoring.toml")
+	if err := e.Transfer(ctx, fp, dst, false, 0); err != nil {
+		return err
+	}
+
+	fp = filepath.Join(paths.Cache, fmt.Sprintf("prometheus_%s_%d.yml", i.GetHost(), i.GetPort()))
 	if err := cfig.ConfigToFile(fp); err != nil {
 		return err
 	}

pkg/cluster/template/config/ngmonitoring.go

@@ -0,0 +1,119 @@
+// Copyright 2020 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path"
+	"text/template"
+
+	"github.com/pingcap/tiup/embed"
+)
+
+// NgMonitoringConfig represent the data to generate NgMonitoring config
+type NgMonitoringConfig struct {
+	ClusterName string
+	TLSEnabled  bool
+	IP          string
+	Port        int
+	PDAddrs     string
+	DeployDir   string
+	DataDir     string
+	LogDir      string
+}
+
+// NewNgMonitoringConfig returns a PrometheusConfig
+func NewNgMonitoringConfig(clusterName, clusterVersion string, enableTLS bool) *NgMonitoringConfig {
+	cfg := &NgMonitoringConfig{
+		ClusterName: clusterName,
+		TLSEnabled:  enableTLS,
+	}
+	return cfg
+}
+
+// AddPD add a PD address
+func (c *NgMonitoringConfig) AddPD(ip string, port uint64) *NgMonitoringConfig {
+	if c.PDAddrs == "" {
+		c.PDAddrs = fmt.Sprintf("\"%s:%d\"", ip, port)
+	} else {
+		c.PDAddrs += fmt.Sprintf(",\"%s:%d\"", ip, port)
+	}
+	return c
+}
+
+// AddIP add ip to ng-monitoring conf
+func (c *NgMonitoringConfig) AddIP(ip string) *NgMonitoringConfig {
+	c.IP = ip
+	return c
+}
+
+// AddLog add logdir to ng-monitoring conf
+func (c *NgMonitoringConfig) AddLog(dir string) *NgMonitoringConfig {
+	c.LogDir = dir
+	return c
+}
+
+// AddDeployDir add logdir to ng-monitoring conf
+func (c *NgMonitoringConfig) AddDeployDir(dir string) *NgMonitoringConfig {
+	c.DeployDir = dir
+	return c
+}
+
+// AddDataDir add logdir to ng-monitoring conf
+func (c *NgMonitoringConfig) AddDataDir(dir string) *NgMonitoringConfig {
+	c.DataDir = dir
+	return c
+}
+
+// AddPort add port to ng-monitoring conf
+func (c *NgMonitoringConfig) AddPort(port int) *NgMonitoringConfig {
+	c.Port = port
+	return c
+}
+
+// ConfigWithTemplate generate the Prometheus config content by tpl
+func (c *NgMonitoringConfig) ConfigWithTemplate(tpl string) ([]byte, error) {
+	tmpl, err := template.New("NgMonitoring").Parse(tpl)
+	if err != nil {
+		return nil, err
+	}
+
+	content := bytes.NewBufferString("")
+	if err := tmpl.Execute(content, c); err != nil {
+		return nil, err
+	}
+
+	return content.Bytes(), nil
+}
+
+// Config generate the config file data.
+func (c *NgMonitoringConfig) Config() ([]byte, error) {
+	fp := path.Join("templates", "config", "ngmonitoring.toml.tpl")
+	tpl, err := embed.ReadTemplate(fp)
+	if err != nil {
+		return nil, err
+	}
+	return c.ConfigWithTemplate(string(tpl))
+}
+
+// ConfigToFile write config content to specific path
+func (c *NgMonitoringConfig) ConfigToFile(file string) error {
+	config, err := c.Config()
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(file, config, 0755)
+}