Merge 80ee2e0 into 37487dc

components/cluster/command/display.go

@@ -80,6 +80,7 @@ func newDisplayCmd() *cobra.Command {
 
 	cmd.Flags().StringSliceVarP(&gOpt.Roles, "role", "R", nil, "Only display specified roles")
 	cmd.Flags().StringSliceVarP(&gOpt.Nodes, "node", "N", nil, "Only display specified nodes")
+	cmd.Flags().BoolVar(&gOpt.ShowUptime, "uptime", false, "Display with uptime")
 	cmd.Flags().BoolVar(&showDashboardOnly, "dashboard", false, "Only display TiDB Dashboard information")
 	cmd.Flags().BoolVar(&showVersionOnly, "version", false, "Only display TiDB cluster version")
 

components/dm/command/display.go

@@ -54,6 +54,7 @@ func newDisplayCmd() *cobra.Command {
 	cmd.Flags().StringSliceVarP(&gOpt.Roles, "role", "R", nil, "Only display specified roles")
 	cmd.Flags().StringSliceVarP(&gOpt.Nodes, "node", "N", nil, "Only display specified nodes")
 	cmd.Flags().BoolVar(&showVersionOnly, "version", false, "Only display DM cluster version")
+	cmd.Flags().BoolVar(&gOpt.ShowUptime, "uptime", false, "Display DM with uptime")
 
 	return cmd
 }

components/dm/spec/logic.go

@@ -15,9 +15,11 @@ package spec
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/pingcap/tiup/pkg/logger/log"
 	"github.com/pingcap/tiup/pkg/meta"
@@ -88,6 +90,9 @@ func (c *DMMasterComponent) Instances() []Instance {
 					s.DataDir,
 				},
 				StatusFn: s.Status,
+				UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+					return spec.UptimeByHost(s.Host, s.Port, tlsCfg)
+				},
 			},
 			topo: c.Topology,
 		})
@@ -216,6 +221,9 @@ func (c *DMWorkerComponent) Instances() []Instance {
 					s.DataDir,
 				},
 				StatusFn: s.Status,
+				UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+					return spec.UptimeByHost(s.Host, s.Port, tlsCfg)
+				},
 			},
 			topo: c.Topology,
 		})

go.mod

@@ -46,6 +46,7 @@ require (
 	github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8 // indirect
 	github.com/pingcap/tidb-insight v0.3.2
 	github.com/prometheus/client_model v0.2.0
+	github.com/prometheus/common v0.15.0
 	github.com/prometheus/prom2json v1.3.0
 	github.com/r3labs/diff/v2 v2.12.0
 	github.com/relex/aini v1.2.1

pkg/cluster/manager/display.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"sort"
 	"strings"
 	"time"
@@ -44,6 +45,7 @@ type InstInfo struct {
 	Ports     string `json:"ports"`
 	OsArch    string `json:"os_arch"`
 	Status    string `json:"status"`
+	Since     string `json:"since"`
 	DataDir   string `json:"data_dir"`
 	DeployDir string `json:"deploy_dir"`
 
@@ -87,22 +89,28 @@ func (m *Manager) Display(name string, opt operator.Options) error {
 	}
 
 	// display topology
-	clusterTable := [][]string{
-		// Header
-		{"ID", "Role", "Host", "Ports", "OS/Arch", "Status", "Data Dir", "Deploy Dir"},
+	var clusterTable [][]string
+	if opt.ShowUptime {
+		clusterTable = append(clusterTable, []string{"ID", "Role", "Host", "Ports", "OS/Arch", "Status", "Since", "Data Dir", "Deploy Dir"})
+	} else {
+		clusterTable = append(clusterTable, []string{"ID", "Role", "Host", "Ports", "OS/Arch", "Status", "Data Dir", "Deploy Dir"})
 	}
+
 	masterActive := make([]string, 0)
 	for _, v := range clusterInstInfos {
-		clusterTable = append(clusterTable, []string{
+		row := []string{
 			color.CyanString(v.ID),
 			v.Role,
 			v.Host,
 			v.Ports,
 			v.OsArch,
 			formatInstanceStatus(v.Status),
-			v.DataDir,
-			v.DeployDir,
-		})
+		}
+		if opt.ShowUptime {
+			row = append(row, v.Since)
+		}
+		row = append(row, v.DataDir, v.DeployDir)
+		clusterTable = append(clusterTable, row)
 
 		if v.ComponentName != spec.ComponentPD && v.ComponentName != spec.ComponentDMMaster {
 			continue
@@ -237,18 +245,28 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI
 			status = ins.Status(tlsCfg, masterActive...)
 		}
 
-		// Query the service status
-		if status == "-" {
+		since := "-"
+		if opt.ShowUptime {
+			since = formatInstanceSince(ins.Uptime(tlsCfg))
+		}
+
+		// Query the service status and uptime
+		if status == "-" || (opt.ShowUptime && since == "-") {
 			e, found := ctxt.GetInner(ctx).GetExecutor(ins.GetHost())
 			if found {
 				active, _ := operator.GetServiceStatus(ctx, e, ins.ServiceName())
-				if parts := strings.Split(strings.TrimSpace(active), " "); len(parts) > 2 {
-					if parts[1] == "active" {
-						status = "Up"
-					} else {
-						status = parts[1]
+				if status == "-" {
+					if parts := strings.Split(strings.TrimSpace(active), " "); len(parts) > 2 {
+						if parts[1] == "active" {
+							status = "Up"
+						} else {
+							status = parts[1]
+						}
 					}
 				}
+				if opt.ShowUptime && since == "-" {
+					since = formatInstanceSince(parseSystemctlSince(active))
+				}
 			}
 		}
 
@@ -268,6 +286,7 @@ func (m *Manager) GetClusterTopology(name string, opt operator.Options) ([]InstI
 			DeployDir:     deployDir,
 			ComponentName: ins.ComponentName(),
 			Port:          ins.GetPort(),
+			Since:         since,
 		})
 	})
 
@@ -303,7 +322,7 @@ func formatInstanceStatus(status string) string {
 		return color.HiGreenString(status)
 	case startsWith("up", "healthy", "free"):
 		return color.GreenString(status)
-	case startsWith("down", "err"): // down, down|ui
+	case startsWith("down", "err", "inactive"): // down, down|ui
 		return color.RedString(status)
 	case startsWith("tombstone", "disconnected", "n/a"), strings.Contains(status, "offline"):
 		return color.YellowString(status)
@@ -312,6 +331,71 @@ func formatInstanceStatus(status string) string {
 	}
 }
 
+func formatInstanceSince(uptime time.Duration) string {
+	if uptime == 0 {
+		return "-"
+	}
+
+	d := int64(uptime.Hours() / 24)
+	h := int64(math.Mod(uptime.Hours(), 24))
+	m := int64(math.Mod(uptime.Minutes(), 60))
+	s := int64(math.Mod(uptime.Seconds(), 60))
+
+	chunks := []struct {
+		unit  string
+		value int64
+	}{
+		{"d", d},
+		{"h", h},
+		{"m", m},
+		{"s", s},
+	}
+
+	parts := []string{}
+
+	for _, chunk := range chunks {
+		switch chunk.value {
+		case 0:
+			continue
+		default:
+			parts = append(parts, fmt.Sprintf("%d%s", chunk.value, chunk.unit))
+		}
+	}
+
+	return strings.Join(parts, "")
+}
+
+// `systemctl status xxx.service` returns as below
+// Active: active (running) since Sat 2021-03-27 10:51:11 CST; 41min ago
+func parseSystemctlSince(str string) (dur time.Duration) {
+	// if service is not found or other error, don't need to parse it
+	if str == "" {
+		return 0
+	}
+	defer func() {
+		if dur == 0 {
+			log.Warnf("failed to parse systemctl since '%s'", str)
+		}
+	}()
+	parts := strings.Split(str, ";")
+	if len(parts) != 2 {
+		return
+	}
+	parts = strings.Split(parts[0], " ")
+	if len(parts) < 3 {
+		return
+	}
+
+	dateStr := strings.Join(parts[len(parts)-3:], " ")
+
+	tm, err := time.Parse("2006-01-02 15:04:05 MST", dateStr)
+	if err != nil {
+		return
+	}
+
+	return time.Since(tm)
+}
+
 // SetSSHKeySet set ssh key set.
 func SetSSHKeySet(ctx context.Context, privateKeyPath string, publicKeyPath string) error {
 	ctxt.GetInner(ctx).PrivateKeyPath = privateKeyPath

pkg/cluster/operation/operation.go

@@ -40,6 +40,9 @@ type Options struct {
 	// Some data will be retained when destroying instances
 	RetainDataRoles []string
 	RetainDataNodes []string
+
+	// Show uptime or not
+	ShowUptime bool
 }
 
 // Operation represents the type of cluster operation

pkg/cluster/spec/alertmanager.go

@@ -18,6 +18,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"path/filepath"
+	"time"
 
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
 	"github.com/pingcap/tiup/pkg/cluster/template/config"
@@ -100,7 +101,10 @@ func (c *AlertManagerComponent) Instances() []Instance {
 					s.DataDir,
 				},
 				StatusFn: func(_ *tls.Config, _ ...string) string {
-					return "-"
+					return statusByHost(s.Host, s.WebPort, "/-/ready", nil)
+				},
+				UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+					return UptimeByHost(s.Host, s.WebPort, tlsCfg)
 				},
 			},
 			topo: c.Topology,

pkg/cluster/spec/cdc.go

@@ -18,6 +18,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"path/filepath"
+	"time"
 
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
@@ -95,12 +96,10 @@ func (c *CDCComponent) Instances() []Instance {
 				s.DeployDir,
 			},
 			StatusFn: func(tlsCfg *tls.Config, _ ...string) string {
-				scheme := "http"
-				if tlsCfg != nil {
-					scheme = "https"
-				}
-				url := fmt.Sprintf("%s://%s:%d/status", scheme, s.Host, s.Port)
-				return statusByURL(url, tlsCfg)
+				return statusByHost(s.Host, s.Port, "/status", tlsCfg)
+			},
+			UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+				return UptimeByHost(s.Host, s.Port, tlsCfg)
 			},
 		}, c.Topology})
 	}

pkg/cluster/spec/drainer.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"time"
 
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
 	"github.com/pingcap/tiup/pkg/cluster/template/scripts"
@@ -98,12 +99,10 @@ func (c *DrainerComponent) Instances() []Instance {
 				s.DataDir,
 			},
 			StatusFn: func(tlsCfg *tls.Config, _ ...string) string {
-				scheme := "http"
-				if tlsCfg != nil {
-					scheme = "https"
-				}
-				url := fmt.Sprintf("%s://%s:%d/status", scheme, s.Host, s.Port)
-				return statusByURL(url, tlsCfg)
+				return statusByHost(s.Host, s.Port, "/status", tlsCfg)
+			},
+			UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+				return UptimeByHost(s.Host, s.Port, tlsCfg)
 			},
 		}, c.Topology})
 	}

pkg/cluster/spec/grafana.go

@@ -20,6 +20,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"strings"
+	"time"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
@@ -101,7 +102,10 @@ func (c *GrafanaComponent) Instances() []Instance {
 					s.DeployDir,
 				},
 				StatusFn: func(_ *tls.Config, _ ...string) string {
-					return "-"
+					return statusByHost(s.Host, s.Port, "", nil)
+				},
+				UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+					return UptimeByHost(s.Host, s.Port, tlsCfg)
 				},
 			},
 			topo: c.Topology,

pkg/cluster/spec/instance.go

@@ -95,6 +95,7 @@ type Instance interface {
 	UsedPorts() []int
 	UsedDirs() []string
 	Status(tlsCfg *tls.Config, pdList ...string) string
+	Uptime(tlsCfg *tls.Config) time.Duration
 	DataDir() string
 	LogDir() string
 	OS() string // only linux supported now
@@ -138,6 +139,7 @@ type BaseInstance struct {
 	Ports    []int
 	Dirs     []string
 	StatusFn func(tlsCfg *tls.Config, pdHosts ...string) string
+	UptimeFn func(tlsCfg *tls.Config) time.Duration
 }
 
 // Ready implements Instance interface
@@ -448,3 +450,8 @@ func (i *BaseInstance) UsedDirs() []string {
 func (i *BaseInstance) Status(tlsCfg *tls.Config, pdList ...string) string {
 	return i.StatusFn(tlsCfg, pdList...)
 }
+
+// Uptime implements Instance interface
+func (i *BaseInstance) Uptime(tlsCfg *tls.Config) time.Duration {
+	return i.UptimeFn(tlsCfg)
+}

pkg/cluster/spec/pd.go

@@ -131,6 +131,9 @@ func (c *PDComponent) Instances() []Instance {
 					s.DataDir,
 				},
 				StatusFn: s.Status,
+				UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+					return UptimeByHost(s.Host, s.ClientPort, tlsCfg)
+				},
 			},
 			topo: c.Topology,
 		})

pkg/cluster/spec/prometheus.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"strings"
+	"time"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
@@ -116,7 +117,10 @@ func (c *MonitorComponent) Instances() []Instance {
 				s.DataDir,
 			},
 			StatusFn: func(_ *tls.Config, _ ...string) string {
-				return "-"
+				return statusByHost(s.Host, s.Port, "/-/ready", nil)
+			},
+			UptimeFn: func(tlsCfg *tls.Config) time.Duration {
+				return UptimeByHost(s.Host, s.Port, tlsCfg)
 			},
 		}, c.Topology})
 	}