[tools] CLI tool to simplify some tasks, and eliminate the curl commands from the docs

Makefile

@@ -93,6 +93,7 @@ TOOLS :=               \
 	verify_index_files   \
 	carbon_load          \
 	docs_test            \
+	m3db_tool            \
 
 .PHONY: setup
 setup:

docs/operational_guide/placement_configuration.md

@@ -118,7 +118,7 @@ curl -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{
             "endpoint": "<NODE_3_HOST_NAME>:<NODE_3_PORT>",
             "hostname": "<NODE_3_HOST_NAME>",
             "port": <NODE_3_PORT>
-        },
+        }
     ]
 }'
 ```

src/cmd/tools/m3db_tool/README.md

@@ -0,0 +1,62 @@
+M3DB Tool
+========
+
+This is a CLI tool to do some things that may be desirable for cluster introspection, or for operational purposes.
+
+Where configuration data is required its provided via YAML.
+
+You can:
+
+* create a database per the simplified database create API
+* list namespaces
+* delete namespaces
+* list placements
+* delete placements
+* add nodes
+* remove nodes
+
+NOTE: This tool can delete namespaces and placements.  It can be quite hazardous if used without adequate understanding of your m3db cluster's topology, or without a decent understanding of how m3db works.
+
+Examples
+-------
+
+    # show help
+    m3db-tool -h
+    # create a database
+    m3db-tool db -create ./database/examples/devel.yaml
+    # list namespaces
+    m3db-tool ns
+    # delete a namespace
+    m3db-tool ns -delete default
+    # list placements
+    m3db-tool pl
+    # point to some remote and list namespaces
+    m3db-tool -endpoint http://localhost:7201 ns
+    # list the ids of the placements
+    m3db-tool -endpoint http://localhost:7201 pl | jq .placement.instances[].id
+
+Some example yaml files are provided in the examples directories. Here's one for database creation:
+
+    ---
+    type: cluster
+    namespace_name: default
+    retention_time: 168h
+    num_shards: 64
+    replication_factor: 1
+    hosts:
+    - id: m3db_seed
+      isolation_group: rack-a
+      zone: embedded
+      weight: 1024
+      endpoint: m3db_seed:9000
+      hostname: m3db_seed
+      port: 9000
+
+
+See the examples directories below.
+
+References
+==========
+
+[https://m3db.github.io/m3/operational_guide](operational guide)
+[api docs](https://www.m3db.io/openapi/)

src/cmd/tools/m3db_tool/main/common/http.go

@@ -0,0 +1,115 @@
+package common
+
+import (
+	"flag"
+	"fmt"
+	"go.uber.org/zap"
+	"io"
+	"io/ioutil"
+	"net/http"
+)
+
+var (
+	EndPoint *string
+)
+
+// This is used across all the commands
+func init() {
+	EndPoint = flag.String("endpoint", "http://localhost:7201", "The url for target m3db backend.")
+}
+
+func DoGet(url string, logger *zap.SugaredLogger, getter func(reader io.Reader, logger *zap.SugaredLogger)) {
+
+	logger.Debugf("DoGet:url:%s:\n", url)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		logger.Fatal(err)
+	}
+
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}()
+
+	checkForAndHandleError(url, resp, logger)
+
+	getter(resp.Body, logger)
+
+}
+
+func DoPost(url string, data io.Reader, logger *zap.SugaredLogger, getter func(reader io.Reader, logger *zap.SugaredLogger)) {
+
+	logger.Debugf("DoPost:url:%s:\n", url)
+
+	client := &http.Client{}
+
+	req, err := http.NewRequest(http.MethodPost, url, data)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		logger.Fatal(err)
+
+	}
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+	}()
+
+	checkForAndHandleError(url, resp, logger)
+
+	getter(resp.Body, logger)
+
+}
+
+func DoDelete(url string, logger *zap.SugaredLogger, getter func(reader io.Reader, logger *zap.SugaredLogger)) {
+
+	logger.Debugf("DoDelete:url:%s:\n", url)
+
+	client := &http.Client{}
+
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		logger.Fatal(err)
+
+	}
+	defer func() {
+		ioutil.ReadAll(resp.Body)
+
+		resp.Body.Close()
+	}()
+
+	checkForAndHandleError(url, resp, logger)
+
+	getter(resp.Body, logger)
+
+}
+
+func DoDump(in io.Reader, log *zap.SugaredLogger) {
+
+	dat, err := ioutil.ReadAll(in)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(string(dat))
+}
+
+func checkForAndHandleError(url string, resp *http.Response, log *zap.SugaredLogger) {
+
+	log.Debugf("resp.StatusCode:%d:\n", resp.StatusCode)
+
+	if resp.StatusCode > 299 {
+		dat, _ := ioutil.ReadAll(resp.Body)
+
+		if dat != nil {
+			fmt.Println(string(dat))
+		}
+
+		log.Fatalf("error from m3db:%s:url:%s:", resp.Status, url)
+	}
+}

src/cmd/tools/m3db_tool/main/common/yaml.go

@@ -0,0 +1,35 @@
+package common
+
+import (
+	"bytes"
+	"github.com/ghodss/yaml"
+	"github.com/gogo/protobuf/jsonpb"
+	"github.com/gogo/protobuf/proto"
+	"go.uber.org/zap"
+	"io"
+	"io/ioutil"
+)
+
+func LoadYaml(path string, target proto.Message, log *zap.SugaredLogger) io.Reader {
+
+	log.Debugf("path:%s:\n", path)
+
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if err = yaml.Unmarshal(content, target); err != nil {
+		log.Fatalf("cannot unmarshal data:%v:from yaml file:%s:", err, path)
+	}
+
+	var data *bytes.Buffer
+	data = bytes.NewBuffer(nil)
+
+	marshaller := &jsonpb.Marshaler{}
+	if err = marshaller.Marshal(data, target); err != nil {
+		log.Fatal(err)
+	}
+
+	return data
+}

src/cmd/tools/m3db_tool/main/database/cmd.go

@@ -0,0 +1,65 @@
+package database
+
+import (
+	"flag"
+	"fmt"
+	"go.uber.org/zap"
+	"os"
+
+	"github.com/m3db/m3/src/cmd/tools/m3db_tool/main/common"
+	"github.com/m3db/m3/src/cmd/tools/m3db_tool/main/flagvar"
+	"github.com/m3db/m3/src/query/generated/proto/admin"
+)
+
+var (
+	CmdFlags   *flag.FlagSet
+	createYaml = flagvar.File{}
+
+	defaultPath = "/api/v1/database"
+	debugQS     = "debug=true"
+)
+
+func init() {
+	CmdFlags = flag.NewFlagSet("db", flag.ExitOnError)
+	CmdFlags.Var(&createYaml, "create", "Path to yaml for simplified db creation with sane defaults.")
+	CmdFlags.Usage = func() {
+		fmt.Fprintf(CmdFlags.Output(), `
+This is the "%s" subcommand for database scoped operations.
+
+Description:
+
+This subcommand allows the creation of a new database from a yaml specification.
+
+Usage of %s:
+
+`, CmdFlags.Name(), CmdFlags.Name())
+		CmdFlags.PrintDefaults()
+	}
+}
+
+func Cmd(log *zap.SugaredLogger) {
+
+	if err := CmdFlags.Parse(flag.Args()[1:]); err != nil {
+		CmdFlags.Usage()
+		os.Exit(1)
+	}
+
+	if CmdFlags.NFlag() > 1 {
+		fmt.Fprintf(os.Stderr, "Please specify only one action.  There were too many cli arguments provided.\n")
+		CmdFlags.Usage()
+		os.Exit(1)
+	}
+
+	if len(createYaml.Value) < 1 {
+		CmdFlags.Usage()
+		os.Exit(1)
+	}
+
+	data := common.LoadYaml(createYaml.Value, &admin.DatabaseCreateRequest{}, log)
+
+	url := fmt.Sprintf("%s%s/create", *common.EndPoint, defaultPath)
+
+	common.DoPost(url, data, log, common.DoDump)
+
+	return
+}

src/cmd/tools/m3db_tool/main/database/examples/create.yaml

@@ -0,0 +1,26 @@
+---
+type: cluster
+namespace_name: 1week_namespace
+retention_time: 168h
+num_shards: 1024
+replication_factor: 3
+hosts:
+- id: m3db001
+  isolationGroup: us-east1-a
+  zone: embedded
+  weight: 100
+  address: 10.142.0.1
+  port: 9000
+- id: m3db002
+  isolationGroup: us-east1-b
+  zone: embedded
+  weight: 100
+  address: 10.142.0.2
+  port: 9000
+- id: m3db003
+  isolationGroup: us-east1-c
+  zone: embedded
+  weight: 100
+  address: 10.142.0.3
+  port: 9000
+

src/cmd/tools/m3db_tool/main/database/examples/devel.yaml

@@ -0,0 +1,15 @@
+---
+type: cluster
+namespace_name: default
+retention_time: 168h
+num_shards: 64
+replication_factor: 1
+hosts:
+- id: m3db_seed
+  isolation_group: rack-a
+  zone: embedded
+  weight: 1024
+  endpoint: m3db_seed:9000
+  hostname: m3db_seed
+  port: 9000
+

src/cmd/tools/m3db_tool/main/flagvar/file.go

@@ -0,0 +1,52 @@
+package flagvar
+
+import (
+	"os"
+	"strings"
+)
+
+// File is a `flag.Value` for file path arguments.
+// By default, any errors from os.Stat are returned.
+// Alternatively, the value of the `Validate` field is used as a validator when specified.
+type File struct {
+	Validate func(os.FileInfo, error) error
+
+	Value string
+}
+
+// Set is flag.Value.Set
+func (fv *File) Set(v string) error {
+	info, err := os.Stat(v)
+	fv.Value = v
+	if fv.Validate != nil {
+		return fv.Validate(info, err)
+	}
+	return err
+}
+
+func (fv *File) String() string {
+	return fv.Value
+}
+
+// Files is a `flag.Value` for file path arguments.
+// By default, any errors from os.Stat are returned.
+// Alternatively, the value of the `Validate` field is used as a validator when specified.
+type Files struct {
+	Validate func(os.FileInfo, error) error
+
+	Values []string
+}
+
+// Set is flag.Value.Set
+func (fv *Files) Set(v string) error {
+	info, err := os.Stat(v)
+	fv.Values = append(fv.Values, v)
+	if fv.Validate != nil {
+		return fv.Validate(info, err)
+	}
+	return err
+}
+
+func (fv *Files) String() string {
+	return strings.Join(fv.Values, ",")
+}