diff --git a/.fossa.yml b/.fossa.yml index f495e83c0a..45a9f148c6 100755 --- a/.fossa.yml +++ b/.fossa.yml @@ -42,10 +42,10 @@ analyze: path: src/cmd/services/m3coordinator/main options: allow-unresolved: true - - name: github.com/m3db/m3/src/cmd/services/m3ctl/main + - name: github.com/m3db/m3/src/cmd/services/r2ctl/main type: go - target: github.com/m3db/m3/src/cmd/services/m3ctl/main - path: src/cmd/services/m3ctl/main + target: github.com/m3db/m3/src/cmd/services/r2ctl/main + path: src/cmd/services/r2ctl/main options: allow-unresolved: true - name: github.com/m3db/m3/src/cmd/services/m3dbnode/main diff --git a/Makefile b/Makefile index a6a5df5992..947093fb42 100644 --- a/Makefile +++ b/Makefile @@ -61,11 +61,11 @@ SERVICES := \ m3aggregator \ m3query \ m3collector \ - m3ctl \ m3em_agent \ m3nsch_server \ m3nsch_client \ m3comparator \ + r2ctl \ SUBDIRS := \ x \ @@ -95,6 +95,7 @@ TOOLS := \ verify_index_files \ carbon_load \ docs_test \ + m3ctl \ .PHONY: setup setup: diff --git a/docs/operational_guide/placement_configuration.md b/docs/operational_guide/placement_configuration.md index 4bb980a7e2..389c46dda7 100644 --- a/docs/operational_guide/placement_configuration.md +++ b/docs/operational_guide/placement_configuration.md @@ -118,7 +118,7 @@ curl -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{ "endpoint": ":", "hostname": "", "port": - }, + } ] }' ``` diff --git a/src/cmd/services/m3ctl/config/config.go b/src/cmd/services/r2ctl/config/config.go similarity index 100% rename from src/cmd/services/m3ctl/config/config.go rename to src/cmd/services/r2ctl/config/config.go diff --git a/src/cmd/services/m3ctl/config/server.go b/src/cmd/services/r2ctl/config/server.go similarity index 100% rename from src/cmd/services/m3ctl/config/server.go rename to src/cmd/services/r2ctl/config/server.go diff --git a/src/cmd/services/m3ctl/main/main.go b/src/cmd/services/r2ctl/main/main.go similarity index 99% rename from src/cmd/services/m3ctl/main/main.go rename to src/cmd/services/r2ctl/main/main.go index 40fe8e1332..7ad9973fc8 100644 --- a/src/cmd/services/m3ctl/main/main.go +++ b/src/cmd/services/r2ctl/main/main.go @@ -29,7 +29,7 @@ import ( "syscall" "time" - "github.com/m3db/m3/src/cmd/services/m3ctl/config" + "github.com/m3db/m3/src/cmd/services/r2ctl/config" "github.com/m3db/m3/src/ctl/auth" "github.com/m3db/m3/src/ctl/server/http" "github.com/m3db/m3/src/ctl/service/health" diff --git a/src/cmd/tools/m3ctl/README.md b/src/cmd/tools/m3ctl/README.md new file mode 100644 index 0000000000..36754834f1 --- /dev/null +++ b/src/cmd/tools/m3ctl/README.md @@ -0,0 +1,82 @@ +M3DB CLI Tool +============= + +This is a CLI tool to do some things that may be desirable for +cluster introspection, or for operational purposes. + +Where configuration data is required its provided via YAML. + +You can: + +* create a database per the simplified database create API +* list namespaces +* delete namespaces +* list placements +* delete placements +* add nodes +* remove nodes + +NOTE: This tool can delete namespaces and placements. It can be +quite hazardous if used without adequate understanding of your m3db +cluster's topology, or without a decent understanding of how m3db +works. + +Examples +------- + + # show help + m3ctl -h + # create a database + m3ctl apply -f ./database/examples/dbcreate.yaml + # list namespaces + m3ctl get ns + # delete a namespace + m3ctl delete ns -id default + # list placements + m3ctl get pl + # point to some remote and list namespaces + m3ctl -endpoint http://localhost:7201 get ns + # check the namespaces in a kubernetes cluster + # first setup a tunnel via kubectl port-forward ... 7201 + m3ctl -endpoint http://localhost:7201 get ns + # list the ids of the placements + m3ctl -endpoint http://localhost:7201 get pl | jq .placement.instances[].id + +Some example yaml files for the "apply" subcommand are provided in the yaml/examples directory. +Here's one to initialize a topology: + + --- + operation: init + num_shards: 64 + replication_factor: 1 + instances: + - id: nodeid1 + isolation_group: isogroup1 + zone: etcd1 + weight: 100 + endpoint: node1:9000 + hostname: node1 + port: 9000 + - id: nodeid2 + isolation_group: isogroup2 + zone: etcd1 + weight: 100 + endpoint: node2:9000 + hostname: node2 + port: 9000 + - id: nodeid3 + isolation_group: isogroup3 + zone: etcd1 + weight: 100 + endpoint: node3:9000 + hostname: node3 + port: 9000 + + +See the examples directories below. + +References +========== + +[https://m3db.github.io/m3/operational_guide](operational guide) +[api docs](https://www.m3db.io/openapi/) diff --git a/src/cmd/tools/m3ctl/apply/apply.go b/src/cmd/tools/m3ctl/apply/apply.go new file mode 100644 index 0000000000..235020b423 --- /dev/null +++ b/src/cmd/tools/m3ctl/apply/apply.go @@ -0,0 +1,47 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package apply + +import ( + "fmt" + + "go.uber.org/zap" + + "github.com/m3db/m3/src/cmd/tools/m3ctl/client" + "github.com/m3db/m3/src/cmd/tools/m3ctl/yaml" +) + +// DoApply does the API call to handle the kubectl apply command +// It looks at the yaml, figures out what kind of API call to make +// Sends it all off to the API +func DoApply( + endpoint string, + headers map[string]string, + filepath string, + logger *zap.Logger, +) ([]byte, error) { + path, data, err := yaml.Load(filepath, logger) + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s%s", endpoint, path) + return client.DoPost(url, headers, data, logger) +} diff --git a/src/cmd/tools/m3ctl/client/checker.go b/src/cmd/tools/m3ctl/client/checker.go new file mode 100644 index 0000000000..c9bbabf79d --- /dev/null +++ b/src/cmd/tools/m3ctl/client/checker.go @@ -0,0 +1,48 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package client + +import ( + "fmt" + "io/ioutil" + "net/http" + + "go.uber.org/zap" +) + +func checkForAndHandleError(url string, resp *http.Response, zl *zap.Logger) error { + if resp.StatusCode/100 != 2 { + body, err := ioutil.ReadAll(resp.Body) + if err == nil { + zl.Error("error response", + zap.Error(fmt.Errorf("status %d", resp.StatusCode)), + zap.String("url", url), + zap.ByteString("response", body)) + } else { + zl.Error("error response", + zap.Error(fmt.Errorf("status %d", resp.StatusCode)), + zap.String("url", url), + zap.Error(fmt.Errorf("response not available: %v", err))) + } + return fmt.Errorf("error response: status=%s, url=%s", resp.Status, url) + } + return nil +} diff --git a/src/cmd/tools/m3ctl/client/http.go b/src/cmd/tools/m3ctl/client/http.go new file mode 100644 index 0000000000..fa617a591d --- /dev/null +++ b/src/cmd/tools/m3ctl/client/http.go @@ -0,0 +1,130 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package client + +import ( + "io" + "io/ioutil" + "net/http" + "time" + + "go.uber.org/zap" +) + +const timeout = time.Duration(5 * time.Second) + +// DoGet is the low level call to the backend api for gets. +func DoGet( + url string, + headers map[string]string, + l *zap.Logger, +) ([]byte, error) { + l.Info("request", zap.String("method", "get"), zap.String("url", url)) + client := http.Client{ + Timeout: timeout, + } + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + setHeadersWithDefaults(req, headers) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { + ioutil.ReadAll(resp.Body) + resp.Body.Close() + }() + if err := checkForAndHandleError(url, resp, l); err != nil { + return nil, err + } + return ioutil.ReadAll(resp.Body) +} + +// DoPost is the low level call to the backend api for posts. +func DoPost( + url string, + headers map[string]string, + data io.Reader, + l *zap.Logger, +) ([]byte, error) { + l.Info("request", zap.String("method", "post"), zap.String("url", url)) + client := &http.Client{ + Timeout: timeout, + } + req, err := http.NewRequest(http.MethodPost, url, data) + if err != nil { + return nil, err + } + + setHeadersWithDefaults(req, headers) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { + ioutil.ReadAll(resp.Body) + resp.Body.Close() + }() + if err := checkForAndHandleError(url, resp, l); err != nil { + return nil, err + } + return ioutil.ReadAll(resp.Body) +} + +// DoDelete is the low level call to the backend api for deletes. +func DoDelete( + url string, + headers map[string]string, + l *zap.Logger, +) ([]byte, error) { + l.Info("request", zap.String("method", "delete"), zap.String("url", url)) + client := &http.Client{ + Timeout: timeout, + } + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return nil, err + } + + setHeadersWithDefaults(req, headers) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { + ioutil.ReadAll(resp.Body) + resp.Body.Close() + }() + if err := checkForAndHandleError(url, resp, l); err != nil { + return nil, err + } + return ioutil.ReadAll(resp.Body) +} + +func setHeadersWithDefaults(req *http.Request, headers map[string]string) { + req.Header.Set("Content-Type", "application/json") + for k, v := range headers { + req.Header.Set(k, v) + } +} diff --git a/src/cmd/tools/m3ctl/main/main.go b/src/cmd/tools/m3ctl/main/main.go new file mode 100644 index 0000000000..875ffb0e1d --- /dev/null +++ b/src/cmd/tools/m3ctl/main/main.go @@ -0,0 +1,249 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/m3db/m3/src/cmd/tools/m3ctl/apply" + "github.com/m3db/m3/src/cmd/tools/m3ctl/namespaces" + "github.com/m3db/m3/src/cmd/tools/m3ctl/placements" + "github.com/m3db/m3/src/query/generated/proto/admin" + + "github.com/gogo/protobuf/jsonpb" + "github.com/spf13/cobra" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const ( + defaultEndpoint = "http://localhost:7201" +) + +// Defaults are (so output is easily consumable by JSON tools like "jq"). +// - Error log level so usually not printing anything unless error encountered +// so the output can be completely JSON. +// - Do not print log stack traces so errors aren't overwhelming output. +var defaultLoggerOptions = loggerOptions{ + level: zapcore.ErrorLevel, + enableStacktrace: false, +} + +type loggerOptions struct { + level zapcore.Level + enableStacktrace bool +} + +func mustNewLogger(opts loggerOptions) *zap.Logger { + loggerCfg := zap.NewDevelopmentConfig() + loggerCfg.Level = zap.NewAtomicLevelAt(opts.level) + loggerCfg.DisableStacktrace = !opts.enableStacktrace + logger, err := loggerCfg.Build() + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + return logger +} + +func main() { + var ( + debug bool + endPoint string + headers = make(map[string]string) + yamlPath string + showAll bool + deleteAll bool + nodeName string + ) + + logger := mustNewLogger(defaultLoggerOptions) + defer func() { + logger.Sync() + fmt.Printf("\n") // End line since most commands finish without an endpoint. + }() + + rootCmd := &cobra.Command{ + Use: "cobra", + } + + getCmd := &cobra.Command{ + Use: "get", + Short: "Get specified resources from the remote", + } + + deleteCmd := &cobra.Command{ + Use: "delete", + Short: "Delete specified resources from the remote", + } + + applyCmd := &cobra.Command{ + Use: "apply", + Short: "Apply various yamls to remote endpoint", + Long: `This will take specific yamls and send them over to the remote +endpoint. See the yaml/examples directory for examples. Operations such as +database creation, database init, adding a node, and replacing a node, are supported. +`, + Run: func(cmd *cobra.Command, args []string) { + fileArg := cmd.LocalFlags().Lookup("file").Value.String() + logger.Debug("running command", zap.String("name", cmd.Name()), zap.String("args", fileArg)) + + if len(fileArg) == 0 { + logger.Fatal("need to specify a path to YAML file") + } + + resp, err := apply.DoApply(endPoint, headers, yamlPath, logger) + if err != nil { + logger.Fatal("apply failed", zap.Error(err)) + } + + os.Stdout.Write(resp) + }, + } + + getNamespaceCmd := &cobra.Command{ + Use: "namespace []", + Short: "Get the namespaces from the remote endpoint", + Aliases: []string{"ns"}, + Run: func(cmd *cobra.Command, args []string) { + logger.Debug("running command", zap.String("command", cmd.Name())) + + resp, err := namespaces.DoGet(endPoint, headers, logger) + if err != nil { + logger.Fatal("get namespace failed", zap.Error(err)) + } + + if !showAll { + var registry admin.NamespaceGetResponse + unmarshaller := &jsonpb.Unmarshaler{AllowUnknownFields: true} + reader := bytes.NewReader(resp) + if err := unmarshaller.Unmarshal(reader, ®istry); err != nil { + logger.Fatal("could not unmarshal response", zap.Error(err)) + } + var namespaces []string + for k := range registry.Registry.Namespaces { + namespaces = append(namespaces, k) + } + // Keep output consistent and output JSON. + if err := json.NewEncoder(os.Stdout).Encode(namespaces); err != nil { + logger.Fatal("could not encode output", zap.Error(err)) + } + return + } + + os.Stdout.Write(resp) + }, + } + + getPlacementCmd := &cobra.Command{ + Use: "placement", + Short: "Get the placement from the remote endpoint", + Aliases: []string{"pl"}, + Run: func(cmd *cobra.Command, args []string) { + logger.Debug("running command", zap.String("command", cmd.Name())) + + resp, err := placements.DoGet(endPoint, headers, logger) + if err != nil { + logger.Fatal("get placement failed", zap.Error(err)) + } + + os.Stdout.Write(resp) + }, + } + + deletePlacementCmd := &cobra.Command{ + Use: "placement", + Short: "Delete the placement from the remote endpoint", + Aliases: []string{"pl"}, + Run: func(cmd *cobra.Command, args []string) { + logger.Debug("running command", zap.String("command", cmd.Name())) + + resp, err := placements.DoDelete(endPoint, headers, nodeName, deleteAll, logger) + if err != nil { + logger.Fatal("delete placement failed", zap.Error(err)) + } + + os.Stdout.Write(resp) + }, + } + + deleteNamespaceCmd := &cobra.Command{ + Use: "namespace", + Short: "Delete the namespace from the remote endpoint", + Aliases: []string{"ns"}, + Run: func(cmd *cobra.Command, args []string) { + logger.Debug("running command", zap.String("command", cmd.Name())) + + resp, err := namespaces.DoDelete(endPoint, headers, nodeName, logger) + if err != nil { + logger.Fatal("delete namespace failed", zap.Error(err)) + } + + os.Stdout.Write(resp) + }, + } + + rootCmd.AddCommand(getCmd, applyCmd, deleteCmd) + getCmd.AddCommand(getNamespaceCmd) + getCmd.AddCommand(getPlacementCmd) + deleteCmd.AddCommand(deletePlacementCmd) + deleteCmd.AddCommand(deleteNamespaceCmd) + + var headersSlice []string + rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "debug log output level (cannot use JSON output)") + rootCmd.PersistentFlags().StringVar(&endPoint, "endpoint", defaultEndpoint, "m3coordinator endpoint URL") + rootCmd.PersistentFlags().StringSliceVarP(&headersSlice, "header", "H", []string{}, "headers to append to requests") + applyCmd.Flags().StringVarP(&yamlPath, "file", "f", "", "times to echo the input") + getNamespaceCmd.Flags().BoolVarP(&showAll, "show-all", "a", false, "times to echo the input") + deletePlacementCmd.Flags().BoolVarP(&deleteAll, "delete-all", "a", false, "delete the entire placement") + deleteCmd.PersistentFlags().StringVarP(&nodeName, "name", "n", "", "which namespace or node to delete") + + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + // Override logger if debug flag set. + if debug { + logger = mustNewLogger(loggerOptions{ + level: zapcore.DebugLevel, + enableStacktrace: true, + }) + } + + // Parse headers slice. + for _, h := range headersSlice { + parts := strings.Split(h, ":") + if len(parts) != 2 { + return fmt.Errorf( + "header must be of format 'name: value': actual='%s'", h) + } + + name, value := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + headers[name] = value + } + + return nil + } + + rootCmd.Execute() +} diff --git a/src/cmd/tools/m3ctl/namespaces/delete.go b/src/cmd/tools/m3ctl/namespaces/delete.go new file mode 100644 index 0000000000..4feafe49eb --- /dev/null +++ b/src/cmd/tools/m3ctl/namespaces/delete.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package namespaces + +import ( + "fmt" + + "github.com/m3db/m3/src/cmd/tools/m3ctl/client" + "go.uber.org/zap" +) + +// DoDelete calls the delete namespaces api on the backend +func DoDelete( + endpoint string, + headers map[string]string, + nsName string, + logger *zap.Logger, +) ([]byte, error) { + url := fmt.Sprintf("%s%s/%s", endpoint, "/api/v1/services/m3db/namespace", nsName) + return client.DoDelete(url, headers, logger) +} diff --git a/src/cmd/tools/m3ctl/namespaces/get.go b/src/cmd/tools/m3ctl/namespaces/get.go new file mode 100644 index 0000000000..00efd478e5 --- /dev/null +++ b/src/cmd/tools/m3ctl/namespaces/get.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package namespaces + +import ( + "fmt" + + "go.uber.org/zap" + + "github.com/m3db/m3/src/cmd/tools/m3ctl/client" +) + +// DoGet calls the backend get namespace api +func DoGet( + endpoint string, + headers map[string]string, + logger *zap.Logger, +) ([]byte, error) { + url := fmt.Sprintf("%s%s?%s", endpoint, DefaultPath, DebugQS) + return client.DoGet(url, headers, logger) +} diff --git a/src/cmd/tools/m3ctl/namespaces/types.go b/src/cmd/tools/m3ctl/namespaces/types.go new file mode 100644 index 0000000000..8046270391 --- /dev/null +++ b/src/cmd/tools/m3ctl/namespaces/types.go @@ -0,0 +1,28 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package namespaces + +const ( + // DefaultPath is the default url path for the namespace api calls + DefaultPath = "/api/v1/namespace" + // DebugQS this is the query string to activate debug output in api responses + DebugQS = "debug=true" +) diff --git a/src/cmd/tools/m3ctl/placements/delete.go b/src/cmd/tools/m3ctl/placements/delete.go new file mode 100644 index 0000000000..1bf29a1aac --- /dev/null +++ b/src/cmd/tools/m3ctl/placements/delete.go @@ -0,0 +1,45 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package placements + +import ( + "fmt" + + "github.com/m3db/m3/src/cmd/tools/m3ctl/client" + + "go.uber.org/zap" +) + +// DoDelete does the delete api calls for placements +func DoDelete( + endpoint string, + headers map[string]string, + nodeName string, + deleteEntire bool, + logger *zap.Logger, +) ([]byte, error) { + if deleteEntire { + url := fmt.Sprintf("%s%s", endpoint, DefaultPath) + return client.DoDelete(url, headers, logger) + } + url := fmt.Sprintf("%s%s/%s", endpoint, DefaultPath, nodeName) + return client.DoDelete(url, headers, logger) +} diff --git a/src/cmd/tools/m3ctl/placements/get.go b/src/cmd/tools/m3ctl/placements/get.go new file mode 100644 index 0000000000..4845a0065f --- /dev/null +++ b/src/cmd/tools/m3ctl/placements/get.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package placements + +import ( + "fmt" + + "github.com/m3db/m3/src/cmd/tools/m3ctl/client" + + "go.uber.org/zap" +) + +// DoGet calls the backend api for get placements +func DoGet( + endpoint string, + headers map[string]string, + logger *zap.Logger, +) ([]byte, error) { + url := fmt.Sprintf("%s%s", endpoint, DefaultPath) + return client.DoGet(url, headers, logger) +} diff --git a/src/cmd/tools/m3ctl/placements/types.go b/src/cmd/tools/m3ctl/placements/types.go new file mode 100644 index 0000000000..00917c3a34 --- /dev/null +++ b/src/cmd/tools/m3ctl/placements/types.go @@ -0,0 +1,26 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package placements + +const ( + // DefaultPath is the url path for api calls for placements + DefaultPath = "/api/v1/services/m3db/placement" +) diff --git a/src/cmd/tools/m3ctl/yaml/db_create.proto b/src/cmd/tools/m3ctl/yaml/db_create.proto new file mode 100644 index 0000000000..5cca79a367 --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/db_create.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +package yaml; + +import "github.com/m3db/m3/src/query/generated/proto/admin/database.proto"; +message DatabaseCreateRequestYaml { + string operation = 1; + admin.DatabaseCreateRequest request = 2; +} diff --git a/src/cmd/tools/m3ctl/yaml/examples/create.yaml b/src/cmd/tools/m3ctl/yaml/examples/create.yaml new file mode 100644 index 0000000000..0d53cbfe82 --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/examples/create.yaml @@ -0,0 +1,28 @@ +--- +operation: create +request: + type: cluster + namespace_name: 1week_namespace + retention_time: 168h + num_shards: 1024 + replication_factor: 3 + hosts: + - id: m3db001 + isolationGroup: us-east1-a + zone: embedded + weight: 100 + address: 10.142.0.1 + port: 9000 + - id: m3db002 + isolationGroup: us-east1-b + zone: embedded + weight: 100 + address: 10.142.0.2 + port: 9000 + - id: m3db003 + isolationGroup: us-east1-c + zone: embedded + weight: 100 + address: 10.142.0.3 + port: 9000 + diff --git a/src/cmd/tools/m3ctl/yaml/examples/develdb.yaml b/src/cmd/tools/m3ctl/yaml/examples/develdb.yaml new file mode 100644 index 0000000000..6b7d774d9e --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/examples/develdb.yaml @@ -0,0 +1,17 @@ +--- +operation: create +request: + type: cluster + namespace_name: default + retention_time: 168h + num_shards: 64 + replication_factor: 1 + hosts: + - id: m3db_seed + isolation_group: rack-a + zone: embedded + weight: 1024 + endpoint: m3db_seed:9000 + hostname: m3db_seed + port: 9000 + diff --git a/src/cmd/tools/m3ctl/yaml/examples/init.yaml b/src/cmd/tools/m3ctl/yaml/examples/init.yaml new file mode 100644 index 0000000000..f86723c266 --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/examples/init.yaml @@ -0,0 +1,27 @@ +--- +operation: init +request: + num_shards: 64 + replication_factor: 1 + instances: + - id: nodeid1 + isolation_group: isogroup1 + zone: etcd1 + weight: 100 + endpoint: node1:9000 + hostname: node1 + port: 9000 + - id: nodeid2 + isolation_group: isogroup2 + zone: etcd1 + weight: 100 + endpoint: node2:9000 + hostname: node2 + port: 9000 + - id: nodeid3 + isolation_group: isogroup3 + zone: etcd1 + weight: 100 + endpoint: node3:9000 + hostname: node3 + port: 9000 \ No newline at end of file diff --git a/src/cmd/tools/m3ctl/yaml/examples/new_node.yaml b/src/cmd/tools/m3ctl/yaml/examples/new_node.yaml new file mode 100644 index 0000000000..abdccf73bd --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/examples/new_node.yaml @@ -0,0 +1,11 @@ +--- +operation: newNode +request: + instances: + - id: node1 + isolationGroup: isoGroup1 + zone: embedded + weight: 100 + endpoint: targetHostname1:9000 + hostname: newNodeHostname1 + port: 9000 \ No newline at end of file diff --git a/src/cmd/tools/m3ctl/yaml/examples/replace_node.yaml b/src/cmd/tools/m3ctl/yaml/examples/replace_node.yaml new file mode 100644 index 0000000000..37ee51120a --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/examples/replace_node.yaml @@ -0,0 +1,14 @@ +--- +operation: replaceNode +request: + leavingInstanceIDs: + - oldnodeid1 + candidates: + - id: newnodeid1 + isolationGroup: newnodeisogroup1 + zone: etcdzone1 + weight: 100 + endpoint: node11:9000 + hostname: node11 + port: 9000 + diff --git a/src/cmd/tools/m3ctl/yaml/generated/db_create.pb.go b/src/cmd/tools/m3ctl/yaml/generated/db_create.pb.go new file mode 100644 index 0000000000..425a5ab05a --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/generated/db_create.pb.go @@ -0,0 +1,112 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: db_create.proto + +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package yaml + +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" + admin "github.com/m3db/m3/src/query/generated/proto/admin" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type DatabaseCreateRequestYaml struct { + Operation string `protobuf:"bytes,1,opt,name=operation,proto3" json:"operation,omitempty"` + Request *admin.DatabaseCreateRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatabaseCreateRequestYaml) Reset() { *m = DatabaseCreateRequestYaml{} } +func (m *DatabaseCreateRequestYaml) String() string { return proto.CompactTextString(m) } +func (*DatabaseCreateRequestYaml) ProtoMessage() {} +func (*DatabaseCreateRequestYaml) Descriptor() ([]byte, []int) { + return fileDescriptor_57e276f15713f139, []int{0} +} + +func (m *DatabaseCreateRequestYaml) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatabaseCreateRequestYaml.Unmarshal(m, b) +} +func (m *DatabaseCreateRequestYaml) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatabaseCreateRequestYaml.Marshal(b, m, deterministic) +} +func (m *DatabaseCreateRequestYaml) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatabaseCreateRequestYaml.Merge(m, src) +} +func (m *DatabaseCreateRequestYaml) XXX_Size() int { + return xxx_messageInfo_DatabaseCreateRequestYaml.Size(m) +} +func (m *DatabaseCreateRequestYaml) XXX_DiscardUnknown() { + xxx_messageInfo_DatabaseCreateRequestYaml.DiscardUnknown(m) +} + +var xxx_messageInfo_DatabaseCreateRequestYaml proto.InternalMessageInfo + +func (m *DatabaseCreateRequestYaml) GetOperation() string { + if m != nil { + return m.Operation + } + return "" +} + +func (m *DatabaseCreateRequestYaml) GetRequest() *admin.DatabaseCreateRequest { + if m != nil { + return m.Request + } + return nil +} + +func init() { + proto.RegisterType((*DatabaseCreateRequestYaml)(nil), "yaml.DatabaseCreateRequestYaml") +} + +func init() { proto.RegisterFile("db_create.proto", fileDescriptor_57e276f15713f139) } + +var fileDescriptor_57e276f15713f139 = []byte{ + // 177 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x8d, 0xb1, 0xca, 0xc2, 0x30, + 0x14, 0x46, 0xe9, 0xcf, 0x8f, 0xd2, 0x38, 0x08, 0x9d, 0xaa, 0x74, 0x28, 0x4e, 0x9d, 0x72, 0xc1, + 0x82, 0xbb, 0xe8, 0x13, 0x74, 0x73, 0x92, 0x9b, 0xe6, 0x52, 0x0b, 0x4d, 0xd3, 0xa6, 0x37, 0x43, + 0xdf, 0x5e, 0x8c, 0x15, 0x17, 0xd7, 0x8f, 0xf3, 0x9d, 0x23, 0xb6, 0x5a, 0xdd, 0x6b, 0x47, 0xc8, + 0x24, 0x07, 0x67, 0xd9, 0x26, 0xff, 0x33, 0x9a, 0x6e, 0x7f, 0x6e, 0x5a, 0x7e, 0x78, 0x25, 0x6b, + 0x6b, 0xc0, 0x94, 0x5a, 0x81, 0x29, 0x61, 0x72, 0x35, 0x8c, 0x9e, 0xdc, 0x0c, 0x0d, 0xf5, 0xe4, + 0x90, 0x49, 0x43, 0xf8, 0x00, 0x6a, 0xd3, 0xf6, 0xa0, 0x91, 0x51, 0xe1, 0xb4, 0x88, 0x0e, 0xa3, + 0xd8, 0x5d, 0x97, 0xe5, 0x12, 0x02, 0x15, 0x8d, 0x9e, 0x26, 0xbe, 0xa1, 0xe9, 0x92, 0x4c, 0xc4, + 0x76, 0x78, 0x39, 0x5a, 0xdb, 0xa7, 0x51, 0x1e, 0x15, 0x71, 0xf5, 0x1d, 0x92, 0x93, 0x58, 0xbb, + 0x37, 0x9c, 0xfe, 0xe5, 0x51, 0xb1, 0x39, 0x66, 0x32, 0x24, 0xe4, 0x4f, 0x61, 0xf5, 0x81, 0xd5, + 0x2a, 0x94, 0xcb, 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbe, 0x30, 0x42, 0x7b, 0xd5, 0x00, 0x00, + 0x00, +} diff --git a/src/cmd/tools/m3ctl/yaml/generated/placement.pb.go b/src/cmd/tools/m3ctl/yaml/generated/placement.pb.go new file mode 100644 index 0000000000..a6144371b8 --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/generated/placement.pb.go @@ -0,0 +1,161 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: placement.proto + +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package yaml + +import ( + fmt "fmt" + math "math" + + proto "github.com/golang/protobuf/proto" + admin "github.com/m3db/m3/src/query/generated/proto/admin" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type PlacementInitRequestYaml struct { + Operation string `protobuf:"bytes,1,opt,name=operation,proto3" json:"operation,omitempty"` + Request *admin.PlacementInitRequest `protobuf:"bytes,2,opt,name=request,proto3" json:"request,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PlacementInitRequestYaml) Reset() { *m = PlacementInitRequestYaml{} } +func (m *PlacementInitRequestYaml) String() string { return proto.CompactTextString(m) } +func (*PlacementInitRequestYaml) ProtoMessage() {} +func (*PlacementInitRequestYaml) Descriptor() ([]byte, []int) { + return fileDescriptor_ae0216eeb0d08e49, []int{0} +} + +func (m *PlacementInitRequestYaml) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PlacementInitRequestYaml.Unmarshal(m, b) +} +func (m *PlacementInitRequestYaml) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PlacementInitRequestYaml.Marshal(b, m, deterministic) +} +func (m *PlacementInitRequestYaml) XXX_Merge(src proto.Message) { + xxx_messageInfo_PlacementInitRequestYaml.Merge(m, src) +} +func (m *PlacementInitRequestYaml) XXX_Size() int { + return xxx_messageInfo_PlacementInitRequestYaml.Size(m) +} +func (m *PlacementInitRequestYaml) XXX_DiscardUnknown() { + xxx_messageInfo_PlacementInitRequestYaml.DiscardUnknown(m) +} + +var xxx_messageInfo_PlacementInitRequestYaml proto.InternalMessageInfo + +func (m *PlacementInitRequestYaml) GetOperation() string { + if m != nil { + return m.Operation + } + return "" +} + +func (m *PlacementInitRequestYaml) GetRequest() *admin.PlacementInitRequest { + if m != nil { + return m.Request + } + return nil +} + +type PlacementReplaceRequestYaml struct { + Operation string `protobuf:"bytes,3,opt,name=operation,proto3" json:"operation,omitempty"` + Request *admin.PlacementReplaceRequest `protobuf:"bytes,4,opt,name=request,proto3" json:"request,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PlacementReplaceRequestYaml) Reset() { *m = PlacementReplaceRequestYaml{} } +func (m *PlacementReplaceRequestYaml) String() string { return proto.CompactTextString(m) } +func (*PlacementReplaceRequestYaml) ProtoMessage() {} +func (*PlacementReplaceRequestYaml) Descriptor() ([]byte, []int) { + return fileDescriptor_ae0216eeb0d08e49, []int{1} +} + +func (m *PlacementReplaceRequestYaml) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PlacementReplaceRequestYaml.Unmarshal(m, b) +} +func (m *PlacementReplaceRequestYaml) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PlacementReplaceRequestYaml.Marshal(b, m, deterministic) +} +func (m *PlacementReplaceRequestYaml) XXX_Merge(src proto.Message) { + xxx_messageInfo_PlacementReplaceRequestYaml.Merge(m, src) +} +func (m *PlacementReplaceRequestYaml) XXX_Size() int { + return xxx_messageInfo_PlacementReplaceRequestYaml.Size(m) +} +func (m *PlacementReplaceRequestYaml) XXX_DiscardUnknown() { + xxx_messageInfo_PlacementReplaceRequestYaml.DiscardUnknown(m) +} + +var xxx_messageInfo_PlacementReplaceRequestYaml proto.InternalMessageInfo + +func (m *PlacementReplaceRequestYaml) GetOperation() string { + if m != nil { + return m.Operation + } + return "" +} + +func (m *PlacementReplaceRequestYaml) GetRequest() *admin.PlacementReplaceRequest { + if m != nil { + return m.Request + } + return nil +} + +func init() { + proto.RegisterType((*PlacementInitRequestYaml)(nil), "yaml.PlacementInitRequestYaml") + proto.RegisterType((*PlacementReplaceRequestYaml)(nil), "yaml.PlacementReplaceRequestYaml") +} + +func init() { proto.RegisterFile("placement.proto", fileDescriptor_ae0216eeb0d08e49) } + +var fileDescriptor_ae0216eeb0d08e49 = []byte{ + // 201 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2f, 0xc8, 0x49, 0x4c, + 0x4e, 0xcd, 0x4d, 0xcd, 0x2b, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xa9, 0x4c, 0xcc, + 0xcd, 0x91, 0x72, 0x4a, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0xcf, 0x35, + 0x4e, 0x49, 0xd2, 0xcf, 0x35, 0xd6, 0x2f, 0x2e, 0x4a, 0xd6, 0x2f, 0x2c, 0x4d, 0x2d, 0xaa, 0xd4, + 0x4f, 0x4f, 0xcd, 0x4b, 0x2d, 0x4a, 0x2c, 0x49, 0x4d, 0xd1, 0x07, 0xeb, 0xd1, 0x4f, 0x4c, 0xc9, + 0xcd, 0xcc, 0xd3, 0x47, 0x33, 0x49, 0x29, 0x9f, 0x4b, 0x22, 0x00, 0x26, 0xe4, 0x99, 0x97, 0x59, + 0x12, 0x94, 0x5a, 0x58, 0x9a, 0x5a, 0x5c, 0x12, 0x99, 0x98, 0x9b, 0x23, 0x24, 0xc3, 0xc5, 0x99, + 0x5f, 0x00, 0x32, 0x23, 0x33, 0x3f, 0x4f, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, 0x21, 0x20, + 0x64, 0xca, 0xc5, 0x5e, 0x04, 0x51, 0x2c, 0xc1, 0xa4, 0xc0, 0xa8, 0xc1, 0x6d, 0x24, 0xad, 0x07, + 0xb6, 0x42, 0x0f, 0x9b, 0x79, 0x41, 0x30, 0xb5, 0x4a, 0xa5, 0x5c, 0xd2, 0x70, 0x05, 0x41, 0xa9, + 0x60, 0xe7, 0xe0, 0xb4, 0x93, 0x19, 0xdd, 0x4e, 0x0b, 0x84, 0x9d, 0x2c, 0x60, 0x3b, 0xe5, 0xd0, + 0xed, 0x44, 0x35, 0x12, 0x6e, 0x6d, 0x12, 0x1b, 0xd8, 0xbb, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x5e, 0x24, 0x27, 0x1e, 0x4b, 0x01, 0x00, 0x00, +} diff --git a/src/cmd/tools/m3ctl/yaml/load.go b/src/cmd/tools/m3ctl/yaml/load.go new file mode 100644 index 0000000000..34c93a05a9 --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/load.go @@ -0,0 +1,63 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package yaml + +import ( + "bytes" + "go.uber.org/zap" + "io" + "io/ioutil" + + "github.com/gogo/protobuf/jsonpb" + "github.com/gogo/protobuf/proto" +) + +// Load reads a yaml representation of an m3 structure +// and produces an io.Reader of it protocol buffer-encoded +// +// we don't know anything about what the user it trying to do +// so peek at it to see what's the intended action then load it +// +// See the examples directories. +// +func Load(path string, zl *zap.Logger) (string, io.Reader, error) { + content, err := ioutil.ReadFile(path) + if err != nil { + return "", nil, err + } + url, pbmessage, err := peeker(content) + if err != nil { + return "", nil, err + } + rv, err := load(pbmessage) + return url, rv, nil +} + +func load(target proto.Message) (io.Reader, error) { + // marshal it into protocol buffers + var data *bytes.Buffer + data = bytes.NewBuffer(nil) + marshaller := &jsonpb.Marshaler{} + if err := marshaller.Marshal(data, target); err != nil { + return nil, err + } + return data, nil +} diff --git a/src/cmd/tools/m3ctl/yaml/peeker.go b/src/cmd/tools/m3ctl/yaml/peeker.go new file mode 100644 index 0000000000..94699e523c --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/peeker.go @@ -0,0 +1,80 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package yaml + +import ( + "fmt" + + "github.com/ghodss/yaml" + "github.com/gogo/protobuf/proto" + + "github.com/m3db/m3/src/cmd/tools/m3ctl/placements" + "github.com/m3db/m3/src/query/generated/proto/admin" +) + +// peek into the yaml to see what it is expected to be +// +// returns the url path, proto.Message, and error +func peeker(data []byte) (string, proto.Message, error) { + type peeker struct { + Operation string + } + peek := &peeker{} + + // this really does nothing more than unpack into the above + // private type to take a peek at Operation + // its just a peek + if err := yaml.Unmarshal(data, &peek); err != nil { + return "", nil, err + } + + // now the payload is of known type + // unmarshal it and return the proto.Message + switch peek.Operation { + case opCreate: + payload := struct{ Request admin.DatabaseCreateRequest }{} + if err := yaml.Unmarshal(data, &payload); err != nil { + return "", nil, err + } + return dbcreatePath, &payload.Request, nil + case opInit: + payload := struct{ Request admin.PlacementInitRequest }{} + if err := yaml.Unmarshal(data, &payload); err != nil { + return "", nil, err + } + return fmt.Sprintf("%s/init", placements.DefaultPath), &payload.Request, nil + case opReplace: + payload := struct{ Request admin.PlacementReplaceRequest }{} + if err := yaml.Unmarshal(data, &payload); err != nil { + return "", nil, err + } + return fmt.Sprintf("%s/replace", placements.DefaultPath), &payload.Request, nil + case opNewNode: + payload := struct{ Request admin.PlacementInitRequest }{} + if err := yaml.Unmarshal(data, &payload); err != nil { + return "", nil, err + } + return fmt.Sprintf("%s", placements.DefaultPath), &payload.Request, nil + default: + return "", nil, fmt.Errorf("Unknown operation specified in the yaml\n") + } + +} diff --git a/src/cmd/tools/m3ctl/yaml/peeker_test.go b/src/cmd/tools/m3ctl/yaml/peeker_test.go new file mode 100644 index 0000000000..f822b4cb4e --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/peeker_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package yaml + +import ( + "io/ioutil" + "testing" + + "github.com/gogo/protobuf/jsonpb" + + "github.com/m3db/m3/src/query/generated/proto/admin" +) + +func TestPeekerPositive(t *testing.T) { + content, err := ioutil.ReadFile("./testdata/basic_create.yaml") + if err != nil { + t.Fatalf("failed to read yaml test data:%v:\n", err) + } + urlpath, pbmessage, err := peeker(content) + if err != nil { + t.Fatalf("operation selector failed to encode the unknown operation yaml test data:%v:\n", err) + } + if urlpath != dbcreatePath { + t.Errorf("urlpath is wrong:expected:%s:got:%s:\n", dbcreatePath, urlpath) + } + data, err := load(pbmessage) + if err != nil { + t.Fatalf("failed to encode to protocol:%v:\n", err) + } + var dest admin.DatabaseCreateRequest + unmarshaller := &jsonpb.Unmarshaler{AllowUnknownFields: true} + if err := unmarshaller.Unmarshal(data, &dest); err != nil { + t.Fatalf("operation selector failed to unmarshal unknown operation data:%v:\n", err) + } + t.Logf("dest:%v:\n", dest) + if dest.NamespaceName != "default" { + t.Errorf("dest NamespaceName does not have the correct value via operation:expected:%v:got:%v:", opCreate, dest.NamespaceName) + } + if dest.Type != "cluster" { + t.Errorf("dest type does not have the correct value via operation:expected:%v:got:%v:", opCreate, dest.Type) + } + if dest.ReplicationFactor != 327 { + t.Errorf("in and out ReplicationFactor did not match:expected:%d:got:%d:\n", 327, dest.ReplicationFactor) + } + if len(dest.Hosts) != 1 { + t.Errorf("number of hosts is wrong:expected:%d:got:%d:\n", 1, len(dest.Hosts)) + } + if dest.Hosts[0].Id != "m3db_seed" { + t.Errorf("hostname is wrong:expected:%s:got:%s:\n", "m3db_seed", dest.Hosts[0]) + } +} +func TestPeekerNegative(t *testing.T) { + content, err := ioutil.ReadFile("./testdata/unknown_operation.yaml") + if err != nil { + t.Fatalf("failed to read yaml test data:%v:\n", err) + } + if _, _, err := peeker(content); err == nil { + t.Fatalf("operation selector should have returned an error\n") + } +} diff --git a/src/cmd/tools/m3ctl/yaml/placement.proto b/src/cmd/tools/m3ctl/yaml/placement.proto new file mode 100644 index 0000000000..bdf71f77eb --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/placement.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package yaml; + +import "github.com/m3db/m3/src/query/generated/proto/admin/placement.proto"; +message PlacementInitRequestYaml { + string operation = 1; + admin.PlacementInitRequest request = 2; +} +message PlacementReplaceRequestYaml { + string operation = 3; + admin.PlacementReplaceRequest request = 4; +} \ No newline at end of file diff --git a/src/cmd/tools/m3ctl/yaml/testdata/basic_create.yaml b/src/cmd/tools/m3ctl/yaml/testdata/basic_create.yaml new file mode 100644 index 0000000000..a5a513a481 --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/testdata/basic_create.yaml @@ -0,0 +1,17 @@ +--- +operation: create +request: + type: cluster + namespace_name: default + retention_time: 168h + num_shards: 64 + replication_factor: 327 + hosts: + - id: m3db_seed + isolation_group: rack-a + zone: embedded + weight: 1024 + endpoint: m3db_seed:9000 + hostname: m3db_seed + port: 9000 + diff --git a/src/cmd/tools/m3ctl/yaml/testdata/unknown_operation.yaml b/src/cmd/tools/m3ctl/yaml/testdata/unknown_operation.yaml new file mode 100644 index 0000000000..c721f32050 --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/testdata/unknown_operation.yaml @@ -0,0 +1,17 @@ +--- +operation: unknown +request: + type: cluster + namespace_name: default + retention_time: 168h + num_shards: 64 + replication_factor: 327 + hosts: + - id: m3db_seed + isolation_group: rack-a + zone: embedded + weight: 1024 + endpoint: m3db_seed:9000 + hostname: m3db_seed + port: 9000 + diff --git a/src/cmd/tools/m3ctl/yaml/types.go b/src/cmd/tools/m3ctl/yaml/types.go new file mode 100644 index 0000000000..9a70a88e7a --- /dev/null +++ b/src/cmd/tools/m3ctl/yaml/types.go @@ -0,0 +1,29 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package yaml + +const ( + opCreate = "create" + opInit = "init" + opReplace = "replaceNode" + opNewNode = "newNode" + dbcreatePath = "/api/v1/database/create" +) diff --git a/src/ctl/public/r2/v1/swagger/swagger.json b/src/ctl/public/r2/v1/swagger/swagger.json index 6a7523be34..90d569d0a3 100644 --- a/src/ctl/public/r2/v1/swagger/swagger.json +++ b/src/ctl/public/r2/v1/swagger/swagger.json @@ -6,7 +6,7 @@ "title": "R2 API", "license": { "name": "MIT", - "url": "https://github.com/m3db/m3ctl/blob/master/LICENSE.md" + "url": "https://github.com/m3db/m3/blob/master/LICENSE.md" } }, "host": "localhost:9000",