Skip to content

Commit

Permalink
Merge branch 'master' into fix_tag
Browse files Browse the repository at this point in the history
  • Loading branch information
nexustar authored Nov 1, 2021
2 parents f986c94 + 9200644 commit f2e2b82
Show file tree
Hide file tree
Showing 11 changed files with 598 additions and 69 deletions.
66 changes: 62 additions & 4 deletions components/cluster/command/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@
package command

import (
"bytes"
"fmt"
"path"
"text/template"

"github.com/pingcap/errors"
"github.com/pingcap/tiup/embed"
Expand All @@ -26,7 +28,24 @@ import (
type TemplateOptions struct {
Full bool // print full template
MultiDC bool // print template for deploying to multiple data center
Local bool // print local template
Local bool // print and render local template
}

// LocalTemplate contains the variables for print local template.
type LocalTemplate struct {
GlobalUser string // global.user in yaml template
GlobalGroup string // global.group in yaml template
GlobalSSHPort int // global.ssh_port in yaml template
GlobalDeployDir string // global.deploy_dir in yaml template
GlobalDataDir string // global.data_dir in yaml template
GlobalArch string // global.arch in yaml template
PDServers []string // pd_servers in yaml template
TiDBServers []string // tidb_servers in yaml template
TiKVServers []string // tikv_servers in yaml template
TiFlashServers []string // tiflash_servers in yaml template
MonitoringServers []string // monitoring_servers in yaml template
GrafanaServers []string // grafana_servers in yaml template
AlertManagerServers []string // alertmanager_servers in yaml template
}

// This is used to identify how many bool type options are set, so that an
Expand All @@ -43,6 +62,7 @@ func sumBool(b ...bool) int {

func newTemplateCmd() *cobra.Command {
opt := TemplateOptions{}
localOpt := LocalTemplate{}

cmd := &cobra.Command{
Use: "template",
Expand All @@ -58,7 +78,7 @@ func newTemplateCmd() *cobra.Command {
case opt.MultiDC:
name = "multi-dc.yaml"
case opt.Local:
name = "local.yaml"
name = "local.tpl"
}

fp := path.Join("examples", "cluster", name)
Expand All @@ -67,14 +87,52 @@ func newTemplateCmd() *cobra.Command {
return err
}

fmt.Println(string(tpl))
if !opt.Local {
// print example yaml and return
fmt.Fprintln(cmd.OutOrStdout(), string(tpl))
return nil
}

// redner template

// validate arch
if localOpt.GlobalArch != "amd64" && localOpt.GlobalArch != "arm64" {
return fmt.Errorf(`supported values are "amd64" or "arm64" in global.arch`)
}

tmpl, err := template.New(name).Parse(string(tpl))
if err != nil {
return err
}

content := bytes.NewBufferString("")
if err := tmpl.Execute(content, &localOpt); err != nil {
return err
}

fmt.Fprintln(cmd.OutOrStdout(), content.String())
return nil
},
}

cmd.Flags().BoolVar(&opt.Full, "full", false, "Print the full topology template for TiDB cluster.")
cmd.Flags().BoolVar(&opt.MultiDC, "multi-dc", false, "Print template for deploying to multiple data center.")
cmd.Flags().BoolVar(&opt.Local, "local", false, "Print template for deploying a simple cluster locally.")
cmd.Flags().BoolVar(&opt.Local, "local", false, "Print and render template for deploying a simple cluster locally.")

// template values for rendering
cmd.Flags().StringVar(&localOpt.GlobalUser, "user", "tidb", "The user who runs the tidb cluster.")
cmd.Flags().StringVar(&localOpt.GlobalGroup, "group", "", "group is used to specify the group name the user belong to if it's not the same as user.")
cmd.Flags().IntVar(&localOpt.GlobalSSHPort, "ssh-port", 22, "SSH port of servers in the managed cluster.")
cmd.Flags().StringVar(&localOpt.GlobalDeployDir, "deploy-dir", "/tidb-deploy", "Storage directory for cluster deployment files, startup scripts, and configuration files.")
cmd.Flags().StringVar(&localOpt.GlobalDataDir, "data-dir", "/tidb-data", "TiDB Cluster data storage directory.")
cmd.Flags().StringVar(&localOpt.GlobalArch, "arch", "amd64", "Supported values: \"amd64\", \"arm64\".")
cmd.Flags().StringSliceVar(&localOpt.PDServers, "pd-servers", []string{"127.0.0.1"}, "List of PD servers")
cmd.Flags().StringSliceVar(&localOpt.TiDBServers, "tidb-servers", []string{"127.0.0.1"}, "List of TiDB servers")
cmd.Flags().StringSliceVar(&localOpt.TiKVServers, "tikv-servers", []string{"127.0.0.1"}, "List of TiKV servers")
cmd.Flags().StringSliceVar(&localOpt.TiFlashServers, "tiflash-servers", nil, "List of TiFlash servers")
cmd.Flags().StringSliceVar(&localOpt.MonitoringServers, "monitoring-servers", []string{"127.0.0.1"}, "List of monitor servers")
cmd.Flags().StringSliceVar(&localOpt.GrafanaServers, "grafana-servers", []string{"127.0.0.1"}, "List of grafana servers")
cmd.Flags().StringSliceVar(&localOpt.AlertManagerServers, "alertmanager-servers", nil, "List of alermanager servers")

return cmd
}
128 changes: 128 additions & 0 deletions components/cluster/command/template_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
package command

import (
"bytes"
"io"
"strings"
"testing"
)

func Test_TemplateLocalCommandSingle(t *testing.T) {
tests := []struct {
optKey string
optVal string
expected string
}{
{"user", "ubuntu", "user: \"ubuntu\""},
{"group", "ubuntu", "group: \"ubuntu\""},
{"ssh-port", "2222", "ssh_port: 2222"},
{"deploy-dir", "/path/to/deploy", "deploy_dir: \"/path/to/deploy\""},
{"data-dir", "/path/to/data", "data_dir: \"/path/to/data\""},
{"arch", "arm64", "arch: \"arm64\""},
{"pd-servers", "a,b,c", "pd_servers:\n - host: a\n - host: b\n - host: c"},
{"tidb-servers", "a,b,c", "tidb_servers:\n - host: a\n - host: b\n - host: c"},
{"tikv-servers", "a,b,c", "tikv_servers:\n - host: a\n - host: b\n - host: c"},
{"tiflash-servers", "a,b,c", "tiflash_servers:\n - host: a\n - host: b\n - host: c"},
{"monitoring-servers", "a,b,c", "monitoring_servers:\n - host: a\n - host: b\n - host: c"},
{"grafana-servers", "a,b,c", "grafana_servers:\n - host: a\n - host: b\n - host: c"},
{"alertmanager-servers", "a,b,c", "alertmanager_servers:\n - host: a\n - host: b\n - host: c"},
}

for _, test := range tests {
cmd := newTemplateCmd()
b := bytes.NewBufferString("")
cmd.SetOut(b)
_ = cmd.Flags().Set("local", "true") // add --local
_ = cmd.Flags().Set(test.optKey, test.optVal)

if err := cmd.Execute(); err != nil {
t.Fatal(err)
}

out, err := io.ReadAll(b)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), test.expected) {
t.Fatalf("expected \"%s\", got \"%s\"", test.expected, string(out))
}
}
}

func Test_TemplateLocalCommandMulti(t *testing.T) {
cmd := newTemplateCmd()
b := bytes.NewBufferString("")
cmd.SetOut(b)
_ = cmd.Flags().Set("local", "true") // add --local
_ = cmd.Flags().Set("user", "ubuntu") // add --user=ubuntu
_ = cmd.Flags().Set("group", "ubuntu") // add --group=ubuntu
_ = cmd.Flags().Set("tidb-servers", "a,b,c") // add --tidb-servers=a,b,c
_ = cmd.Flags().Set("alertmanager-servers", "a,b,c") // add --alertmanager-servers=a,b,c

if err := cmd.Execute(); err != nil {
t.Fatal(err)
}

out, err := io.ReadAll(b)
if err != nil {
t.Fatal(err)
}

for _, b := range []bool{
strings.Contains(string(out), "user: \"ubuntu\""),
strings.Contains(string(out), "group: \"ubuntu\""),
strings.Contains(string(out), "tidb_servers:\n - host: a\n - host: b\n - host: c"),
strings.Contains(string(out), "alertmanager_servers:\n - host: a\n - host: b\n - host: c"),
} {
if !b {
t.Fatalf("unexpected output. got \"%s\"", string(out))
}
}
}

func Test_TemplateLocalCommandNoopt(t *testing.T) {
cmd := newTemplateCmd()
b := bytes.NewBufferString("")
cmd.SetOut(b)
_ = cmd.Flags().Set("local", "true") // add --local

if err := cmd.Execute(); err != nil {
t.Fatal(err)
}

out, err := io.ReadAll(b)
if err != nil {
t.Fatal(err)
}

// check default output
for _, b := range []bool{
strings.Contains(string(out), "user: \"tidb\""),
strings.Contains(string(out), "ssh_port: 22"),
strings.Contains(string(out), "deploy_dir: \"/tidb-deploy\""),
strings.Contains(string(out), "data_dir: \"/tidb-data\""),
strings.Contains(string(out), "arch: \"amd64\""),
strings.Contains(string(out), "pd_servers:\n - host: 127.0.0.1"),
strings.Contains(string(out), "tidb_servers:\n - host: 127.0.0.1"),
strings.Contains(string(out), "tikv_servers:\n - host: 127.0.0.1"),
strings.Contains(string(out), "monitoring_servers:\n - host: 127.0.0.1"),
strings.Contains(string(out), "grafana_servers:\n - host: 127.0.0.1"),
} {
if !b {
t.Fatalf("unexpected output. got \"%s\"", string(out))
}
}
}

func Test_TemplateLocalCommandValidate(t *testing.T) {
cmd := newTemplateCmd()
b := bytes.NewBufferString("")
cmd.SetOut(b)
_ = cmd.Flags().Set("local", "true") // add --local
_ = cmd.Flags().Set("arch", "i386") // add --arch=i386 (invalid)

// should returns err
if err := cmd.Execute(); err == nil {
t.Fatal(err)
}
}
78 changes: 75 additions & 3 deletions components/dm/command/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,28 +14,50 @@
package command

import (
"bytes"
"fmt"
"path"
"text/template"

"github.com/pingcap/tiup/embed"
"github.com/spf13/cobra"
)

// TemplateOptions contains the options for print topology template.
type TemplateOptions struct {
Full bool // print full template
Full bool // print full template
Local bool // print and render local template
}

// LocalTemplate contains the variables for print local template.
type LocalTemplate struct {
GlobalUser string // global.user in yaml template
GlobalGroup string // global.group in yaml template
GlobalSSHPort int // global.ssh_port in yaml template
GlobalDeployDir string // global.deploy_dir in yaml template
GlobalDataDir string // global.data_dir in yaml template
GlobalArch string // global.arch in yaml template
MasterServers []string // master_servers in yaml template
WorkerServers []string // worker_servers in yaml template
MonitoringServers []string // monitoring_servers in yaml template
GrafanaServers []string // grafana_servers in yaml template
AlertManagerServers []string // alertmanager_servers in yaml template
}

func newTemplateCmd() *cobra.Command {
opt := TemplateOptions{}
localOpt := LocalTemplate{}

cmd := &cobra.Command{
Use: "template",
Short: "Print topology template",
RunE: func(cmd *cobra.Command, args []string) error {
name := "minimal.yaml"
if opt.Full {
switch {
case opt.Full:
name = "topology.example.yaml"
case opt.Local:
name = "local.tpl"
}

fp := path.Join("examples", "dm", name)
Expand All @@ -44,12 +66,62 @@ func newTemplateCmd() *cobra.Command {
return err
}

fmt.Println(string(tpl))
if !opt.Local {
// print example yaml and return
fmt.Fprintln(cmd.OutOrStdout(), string(tpl))
return nil
}

// redner template

// validate arch
if localOpt.GlobalArch != "amd64" && localOpt.GlobalArch != "arm64" {
return fmt.Errorf(`supported values are "amd64" or "arm64" in global.arch`)
}

// validate number of masters and workers
if len(localOpt.MasterServers) < 3 {
return fmt.Errorf(
"at least 3 masters must be defined (given %d servers)",
len(localOpt.MasterServers),
)
}
if len(localOpt.WorkerServers) < 3 {
return fmt.Errorf(
"at least 3 workers must be defined (given %d servers)",
len(localOpt.WorkerServers),
)
}

tmpl, err := template.New(name).Parse(string(tpl))
if err != nil {
return err
}
content := bytes.NewBufferString("")
if err := tmpl.Execute(content, &localOpt); err != nil {
return err
}

fmt.Fprintln(cmd.OutOrStdout(), content.String())
return nil
},
}

cmd.Flags().BoolVar(&opt.Full, "full", false, "Print the full topology template for DM cluster.")
cmd.Flags().BoolVar(&opt.Local, "local", false, "Print and render template for deploying a simple DM cluster locally.")

// template values for rendering
cmd.Flags().StringVar(&localOpt.GlobalUser, "user", "tidb", "The user who runs the tidb cluster.")
cmd.Flags().StringVar(&localOpt.GlobalGroup, "group", "", "group is used to specify the group name the user belong to if it's not the same as user.")
cmd.Flags().IntVar(&localOpt.GlobalSSHPort, "ssh-port", 22, "SSH port of servers in the managed cluster.")
cmd.Flags().StringVar(&localOpt.GlobalDeployDir, "deploy-dir", "/tidb-deploy", "Storage directory for cluster deployment files, startup scripts, and configuration files.")
cmd.Flags().StringVar(&localOpt.GlobalDataDir, "data-dir", "/tidb-data", "TiDB Cluster data storage directory.")
cmd.Flags().StringVar(&localOpt.GlobalArch, "arch", "amd64", "Supported values: \"amd64\", \"arm64\".")
cmd.Flags().StringSliceVar(&localOpt.MasterServers, "master-servers", []string{"172.19.0.101", "172.19.0.102", "172.19.0.103"}, "List of Master servers")
cmd.Flags().StringSliceVar(&localOpt.WorkerServers, "worker-servers", []string{"172.19.0.101", "172.19.0.102", "172.19.0.103"}, "List of Worker servers")
cmd.Flags().StringSliceVar(&localOpt.MonitoringServers, "monitoring-servers", []string{"172.19.0.101"}, "List of monitor servers")
cmd.Flags().StringSliceVar(&localOpt.GrafanaServers, "grafana-servers", []string{"172.19.0.101"}, "List of grafana servers")
cmd.Flags().StringSliceVar(&localOpt.AlertManagerServers, "alertmanager-servers", []string{"172.19.0.101"}, "List of alermanager servers")

return cmd
}
Loading

0 comments on commit f2e2b82

Please sign in to comment.