diff --git a/.gitignore b/.gitignore index 9ae1bd8ae1a..9795d60161c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ __pycache__/ # Project specific ignore files *.swp bin -/katib-cli # Go ignore files ## Binaries for programs and plugins diff --git a/cmd/cli/Dockerfile b/cmd/cli/Dockerfile deleted file mode 100644 index cec27d10aa6..00000000000 --- a/cmd/cli/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM golang:alpine AS build-env -# The GOPATH in the image is /go. -ADD . /go/src/github.com/kubeflow/katib -WORKDIR /go/src/github.com/kubeflow/katib/cmd/cli -RUN go build -o katib-cli - -FROM alpine:3.7 -WORKDIR /app -COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/cli/katib-cli /app/ diff --git a/cmd/cli/command/create-study.go b/cmd/cli/command/create-study.go deleted file mode 100644 index bcbbc34cd75..00000000000 --- a/cmd/cli/command/create-study.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "context" - "fmt" - "io/ioutil" - "log" - - "github.com/spf13/cobra" - "google.golang.org/grpc" - yaml "gopkg.in/yaml.v2" - - "github.com/kubeflow/katib/pkg/api" -) - -type createStudyOpt struct { - conf string - args []string -} - -//NewCommandCreateStudy generate create study cmd -func NewCommandCreateStudy() *cobra.Command { - var opt createStudyOpt - cmd := &cobra.Command{ - Use: "study", - Args: cobra.NoArgs, - Short: "Create a study from a file", - Long: "YAML formats are accepted.", - Aliases: []string{"st"}, - Run: func(cmd *cobra.Command, args []string) { - opt.args = args - createStudy(cmd, &opt) - }, - } - cmd.Flags().StringVarP(&opt.conf, "config", "f", "", "File path of study config(required)") - cmd.MarkFlagRequired("config") - return cmd -} - -func createStudy(cmd *cobra.Command, opt *createStudyOpt) { - //check and get persistent flag volume - var pf *PersistentFlags - pf, err := CheckPersistentFlags() - if err != nil { - log.Fatalf("Fail to Check Flags: %v", err) - return - } - - var sc api.StudyConfig - buf, _ := ioutil.ReadFile(opt.conf) - err = yaml.Unmarshal(buf, &sc) - - conn, err := grpc.Dial(pf.server, grpc.WithInsecure()) - if err != nil { - log.Fatalf("could not connect: %v", err) - return - } - defer conn.Close() - req := &api.CreateStudyRequest{StudyConfig: &sc} - c := api.NewManagerClient(conn) - r, err := c.CreateStudy(context.Background(), req) - if err != nil { - log.Fatalf("CreateStudy failed: %v", err) - } - fmt.Printf("Study %v is created.", r.StudyId) -} diff --git a/cmd/cli/command/create.go b/cmd/cli/command/create.go deleted file mode 100644 index 696d28a158e..00000000000 --- a/cmd/cli/command/create.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "github.com/spf13/cobra" -) - -//NewCommandCreate generate create cmd -func NewCommandCreate() *cobra.Command { - cmd := &cobra.Command{ - Use: "create", - Short: "Create a resource from a file", - Long: `YAML formats are accepted.`, - } - - cmd.AddCommand(NewCommandCreateStudy()) - - return cmd -} diff --git a/cmd/cli/command/get-model.go b/cmd/cli/command/get-model.go deleted file mode 100644 index b8c2627d9ff..00000000000 --- a/cmd/cli/command/get-model.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "context" - "fmt" - "log" - "os" - "strings" - "text/tabwriter" - "unicode/utf8" - - "github.com/spf13/cobra" - "google.golang.org/grpc" - - "github.com/kubeflow/katib/pkg/api" -) - -type getModelOpt struct { - detail bool - args []string -} - -//NewCommandGetModel generate get model cmd -func NewCommandGetModel() *cobra.Command { - var opt getModelOpt - cmd := &cobra.Command{ - Use: "model", - Args: cobra.MaximumNArgs(2), - Short: "Display Model Info", - Long: `Display Information of saved model`, - Aliases: []string{"md"}, - Run: func(cmd *cobra.Command, args []string) { - opt.args = args - getModel(cmd, &opt) - }, - } - cmd.Flags().BoolVarP(&opt.detail, "detail", "d", false, "Display detail information of Model") - return cmd -} - -func getModel(cmd *cobra.Command, opt *getModelOpt) { - //check and get persistent flag volume - var pf *PersistentFlags - pf, err := CheckPersistentFlags() - if err != nil { - log.Fatalf("Fail to Check Flags: %v", err) - return - } - - conn, err := grpc.Dial(pf.server, grpc.WithInsecure()) - if err != nil { - log.Fatalf("could not connect: %v", err) - return - } - defer conn.Close() - c := api.NewManagerClient(conn) - // Search study if Study ID or name is set - req := &api.GetStudyListRequest{} - r, err := c.GetStudyList(context.Background(), req) - if err != nil { - log.Fatalf("GetModels failed: %v", err) - } - if len(r.StudyOverviews) == 0 { - log.Println("No Study found") - return - } - for _, si := range r.StudyOverviews { - if len(opt.args) > 0 { - if utf8.RuneCountInString(opt.args[0]) >= 7 { - if !strings.HasPrefix(si.Id, opt.args[0]) { - break - } - } - if si.Name != opt.args[0] { - break - } - } - // Search Models from ModelDB - mreq := &api.GetSavedModelsRequest{StudyName: si.Name} - mr, err := c.GetSavedModels(context.Background(), mreq) - if err != nil { - log.Fatalf("GetModels failed: %v", err) - return - } - w := new(tabwriter.Writer) - w.Init(os.Stdout, 0, 8, 0, '\t', tabwriter.TabIndent) - fmt.Printf("Study %v Owner %v Saved Model Num %v:\n", si.Name, si.Owner, len(mr.Models)) - if opt.detail { - for _, m := range mr.Models { - if len(opt.args) > 1 { - if !strings.HasPrefix(m.WorkerId, opt.args[1]) { - continue - } - } - fmt.Printf("WorkerID :%v\n", m.WorkerId) - fmt.Printf("Model Path: %s\n", m.ModelPath) - fmt.Println("Parameters:") - for _, p := range m.Parameters { - fmt.Fprintf(w, " %s:\t%v\n", p.Name, p.Value) - } - w.Flush() - fmt.Println("Metrics:") - for _, m := range m.Metrics { - fmt.Fprintf(w, " %s:\t%v\n", m.Name, m.Value) - } - w.Flush() - } - } else { - fmt.Fprintln(w, "TrialID\tParamNum\tMetricsNum") - for _, m := range mr.Models { - if len(opt.args) > 1 { - if !strings.HasPrefix(m.WorkerId, opt.args[1]) { - continue - } - } - fmt.Fprintf(w, "%s\t%d\t%d\n", - string([]rune(m.WorkerId)[:7]), - len(m.Parameters), - len(m.Metrics), - ) - } - w.Flush() - } - } -} diff --git a/cmd/cli/command/get-study.go b/cmd/cli/command/get-study.go deleted file mode 100644 index 4c05a61893d..00000000000 --- a/cmd/cli/command/get-study.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "context" - "fmt" - "log" - "os" - "strings" - "text/tabwriter" - "unicode/utf8" - - "github.com/spf13/cobra" - "google.golang.org/grpc" - - "github.com/kubeflow/katib/pkg/api" -) - -//NewCommandGetStudy generate get studies cmd -func NewCommandGetStudy() *cobra.Command { - cmd := &cobra.Command{ - Use: "studies", - Args: cobra.MaximumNArgs(1), - Short: "Display Study lnfo", - Long: `Display Information of a studies`, - Aliases: []string{"st"}, - Run: getStudy, - } - return cmd -} - -func getStudy(cmd *cobra.Command, args []string) { - //check and get persistent flag volume - var pf *PersistentFlags - pf, err := CheckPersistentFlags() - if err != nil { - log.Fatalf("Fail to Check Flags: %v", err) - return - } - - conn, err := grpc.Dial(pf.server, grpc.WithInsecure()) - if err != nil { - log.Fatalf("could not connect: %v", err) - return - } - defer conn.Close() - - c := api.NewManagerClient(conn) - req := &api.GetStudyListRequest{} - r, err := c.GetStudyList(context.Background(), req) - if err != nil { - log.Fatalf("GetStudy failed: %v", err) - return - } - result := []*api.StudyOverview{} - // Search study if Study ID or name is set - if len(args) > 0 { - for _, si := range r.StudyOverviews { - if utf8.RuneCountInString(args[0]) >= 7 { - if strings.HasPrefix(si.Id, args[0]) { - result = append(result, si) - break - } - } - if si.Name == args[0] { - result = append(result, si) - break - } - } - } else { - result = r.StudyOverviews - } - w := new(tabwriter.Writer) - w.Init(os.Stdout, 0, 8, 0, '\t', tabwriter.TabIndent) - fmt.Fprintln(w, "StudyID\tName\tOwner") - for _, si := range result { - fmt.Fprintf(w, "%s\t%s\t%s\n", - string([]rune(si.Id)[:7]), - si.Name, - si.Owner, - ) - } - w.Flush() -} diff --git a/cmd/cli/command/get.go b/cmd/cli/command/get.go deleted file mode 100644 index 0765836f211..00000000000 --- a/cmd/cli/command/get.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "github.com/spf13/cobra" -) - -//NewCommandGet generate get cmd -func NewCommandGet() *cobra.Command { - cmd := &cobra.Command{ - Use: "get", - Short: "Display one or many resources", - Long: `list of resorces comannd can display includes: studies, study, trials, trial, models, model`, - } - - //set local flag - - //add subcommand - cmd.AddCommand(NewCommandGetStudy()) - // cmd.AddCommand(NewCommandGetTrial()) - cmd.AddCommand(NewCommandGetModel()) - - return cmd -} diff --git a/cmd/cli/command/pull-study.go b/cmd/cli/command/pull-study.go deleted file mode 100644 index 36dd3d06cc7..00000000000 --- a/cmd/cli/command/pull-study.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "context" - "fmt" - "io/ioutil" - "log" - "os" - "strings" - "unicode/utf8" - - "github.com/spf13/cobra" - "google.golang.org/grpc" - yaml "gopkg.in/yaml.v2" - - "github.com/kubeflow/katib/pkg/api" -) - -type pullStudyOpt struct { - outfile string - args []string -} - -//NewCommandPullStudy generate pull studies cmd -func NewCommandPullStudy() *cobra.Command { - var opt pullStudyOpt - cmd := &cobra.Command{ - Use: "studies", - Args: cobra.ExactArgs(1), - Short: "Export a Study and its Models lnfo", - Long: `Export Information of a Study and its Models to yaml format`, - Run: func(cmd *cobra.Command, args []string) { - opt.args = args - pullStudy(cmd, &opt) - }, - Aliases: []string{"st"}, - } - cmd.Flags().StringVarP(&opt.outfile, "output", "o", "", "File path to export") - return cmd -} - -func pullStudy(cmd *cobra.Command, opt *pullStudyOpt) { - //check and get persistent flag volume - var pf *PersistentFlags - pf, err := CheckPersistentFlags() - if err != nil { - log.Fatalf("Fail to Check Flags: %v", err) - } - conn, err := grpc.Dial(pf.server, grpc.WithInsecure()) - if err != nil { - log.Fatalf("could not connect: %v", err) - } - defer conn.Close() - - c := api.NewManagerClient(conn) - listreq := &api.GetStudyListRequest{} - listr, err := c.GetStudyList(context.Background(), listreq) - if err != nil { - log.Fatalf("GetStudy failed: %v", err) - return - } - studyID := "" - // Search study by Study ID or name - for _, si := range listr.StudyOverviews { - if utf8.RuneCountInString(opt.args[0]) >= 7 { - if strings.HasPrefix(si.Id, opt.args[0]) { - studyID = si.Id - break - } - } - if si.Name == opt.args[0] { - studyID = si.Id - break - } - } - if studyID == "" { - log.Fatalf("Study %s is not found", opt.args[0]) - } - req := &api.GetStudyRequest{ - StudyId: studyID, - } - r, err := c.GetStudy(context.Background(), req) - if err != nil { - log.Fatalf("GetStudy failed: %v", err) - } - mreq := &api.GetSavedModelsRequest{ - StudyName: r.StudyConfig.Name, - } - mr, err := c.GetSavedModels(context.Background(), mreq) - if err != nil { - log.Fatalf("GetModel failed: %v", err) - } - sd := StudyData{ - StudyConf: r.StudyConfig, - Models: mr.Models, - } - yst, err := yaml.Marshal(sd) - if err != nil { - log.Fatalf("Failed to Marshal: %v", err) - } - if opt.outfile != "" { - ioutil.WriteFile(opt.outfile, yst, os.ModePerm) - } else { - fmt.Println(string(yst)) - } -} diff --git a/cmd/cli/command/pull.go b/cmd/cli/command/pull.go deleted file mode 100644 index 1ec4486a150..00000000000 --- a/cmd/cli/command/pull.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "github.com/spf13/cobra" -) - -//NewCommandPull generate run cmd -func NewCommandPull() *cobra.Command { - cmd := &cobra.Command{ - Use: "pull", - Short: "Pull a resource from a file or from stdin.", - Long: `YAML or JSON formats are accepted.`, - } - - cmd.AddCommand(NewCommandPullStudy()) - // cmd.AddCommand(NewCommandPullModel()) - - return cmd -} diff --git a/cmd/cli/command/push-model.go b/cmd/cli/command/push-model.go deleted file mode 100644 index 40a4487cddb..00000000000 --- a/cmd/cli/command/push-model.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "log" - - "github.com/spf13/cobra" - "google.golang.org/grpc" - yaml "gopkg.in/yaml.v2" - - "github.com/kubeflow/katib/pkg/api" -) - -type pushModelOpt struct { - file string - args []string -} - -//NewCommandPushModel generate push model cmd -func NewCommandPushModel() *cobra.Command { - var opt pushModelOpt - cmd := &cobra.Command{ - Use: "model", - Args: cobra.MaximumNArgs(1), - Short: "Push a model Info from a file or from stdin", - Long: "YAML or JSON formats are accepted.", - Aliases: []string{"md"}, - Run: func(cmd *cobra.Command, args []string) { - opt.args = args - pushModel(cmd, &opt) - }, - } - cmd.Flags().StringVarP(&opt.file, "file", "f", "", "File path of model config file") - return cmd -} - -func pushModel(cmd *cobra.Command, opt *pushModelOpt) { - //check and get persistent flag volume - var pf *PersistentFlags - pf, err := CheckPersistentFlags() - if err != nil { - log.Fatalf("Fail to Check Flags: %v", err) - return - } - var req []*api.SaveModelRequest - if opt.file != "" { - buf, _ := ioutil.ReadFile(opt.file) - err = yaml.Unmarshal(buf, &req) - if err != nil { - log.Fatalf("Fail to parse config: %v", err) - return - } - } else if len(opt.args) > 0 { - err := json.Unmarshal(([]byte)(opt.args[0]), &req) - if err != nil { - log.Fatalf("Fail to parse input: %v", err) - return - } - } else { - log.Fatalf("You shoud specify study config from a file or options: %v", err) - return - } - - conn, err := grpc.Dial(pf.server, grpc.WithInsecure()) - if err != nil { - log.Fatalf("could not connect: %v", err) - return - } - defer conn.Close() - c := api.NewManagerClient(conn) - for _, m := range req { - _, err = c.SaveModel(context.Background(), m) - if err != nil { - log.Fatalf("PushModel failed: %v", err) - } - fmt.Printf("Model %v is Pushed.\n", m.Model.WorkerId) - } -} diff --git a/cmd/cli/command/push-study.go b/cmd/cli/command/push-study.go deleted file mode 100644 index da4f8354919..00000000000 --- a/cmd/cli/command/push-study.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "log" - - "github.com/spf13/cobra" - "google.golang.org/grpc" - yaml "gopkg.in/yaml.v2" - - "github.com/kubeflow/katib/pkg/api" -) - -type pushStudyOpt struct { - file string - args []string -} - -//NewCommandPushStudy generate push model cmd -func NewCommandPushStudy() *cobra.Command { - var opt pushStudyOpt - cmd := &cobra.Command{ - Use: "study", - Args: cobra.MaximumNArgs(1), - Short: "Push a Study Info and its Models from a file or from stdin", - Long: "Push a Study Info and its Models from a file or from stdin\nYAML formats are accepted.", - Aliases: []string{"st"}, - Run: func(cmd *cobra.Command, args []string) { - opt.args = args - pushStudy(cmd, &opt) - }, - } - cmd.Flags().StringVarP(&opt.file, "file", "f", "", "File path of model config file") - return cmd -} - -func pushStudy(cmd *cobra.Command, opt *pushStudyOpt) { - //check and get persistent flag volume - var pf *PersistentFlags - pf, err := CheckPersistentFlags() - if err != nil { - log.Fatalf("Fail to Check Flags: %v", err) - return - } - var in StudyData - - if opt.file != "" { - buf, _ := ioutil.ReadFile(opt.file) - err = yaml.Unmarshal(buf, &in) - if err != nil { - log.Fatalf("Fail to parse config: %v", err) - return - } - } else if len(opt.args) > 0 { - err := json.Unmarshal(([]byte)(opt.args[0]), &in) - if err != nil { - log.Fatalf("Fail to parse input: %v", err) - return - } - } else { - log.Fatalf("You shoud specify study config from a file or options: %v", err) - return - } - - conn, err := grpc.Dial(pf.server, grpc.WithInsecure()) - if err != nil { - log.Fatalf("could not connect: %v", err) - return - } - defer conn.Close() - c := api.NewManagerClient(conn) - sreq := &api.CreateStudyRequest{ - StudyConfig: in.StudyConf, - } - sr, err := c.CreateStudy(context.Background(), sreq) - if err != nil { - log.Fatalf("CreateStudy failed: %v", err) - } - - for _, m := range in.Models { - req := &api.SaveModelRequest{ - Model: m, - } - _, err = c.SaveModel(context.Background(), req) - if err != nil { - log.Fatalf("PushModel failed: %v", err) - } - fmt.Printf("Model %v is Pushed.\n", m.WorkerId) - } - fmt.Printf("Study %s is Pushd. ID: %s", in.StudyConf.Name, sr.StudyId) -} diff --git a/cmd/cli/command/push.go b/cmd/cli/command/push.go deleted file mode 100644 index d1a95d5ab63..00000000000 --- a/cmd/cli/command/push.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "github.com/spf13/cobra" -) - -//NewCommandPush generate run cmd -func NewCommandPush() *cobra.Command { - cmd := &cobra.Command{ - Use: "push", - Short: "Push a resource from a file or from stdin.", - Long: `YAML or JSON formats are accepted.`, - } - - cmd.AddCommand(NewCommandPushStudy()) - - return cmd -} diff --git a/cmd/cli/command/root.go b/cmd/cli/command/root.go deleted file mode 100644 index e297f671030..00000000000 --- a/cmd/cli/command/root.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "errors" - - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -//PersistentFlags is used for pass persistent flag value to other command -type PersistentFlags struct { - server string -} - -// NewRootCommand represents the base command when called without any subcommands -func NewRootCommand() *cobra.Command { - - cmd := &cobra.Command{ - Use: "katib-cli", - Short: "katib cli", - Long: `This is katib cli client using cobra framework`, - } - - //initialize config - initFlag(cmd) - - //add command - cmd.AddCommand(NewCommandCreate()) - cmd.AddCommand(NewCommandGet()) - cmd.AddCommand(NewCommandPush()) - cmd.AddCommand(NewCommandPull()) - - // cmd.AddCommand(NewCommandModel()) - - //MISC - //cmd.AddCommand(NewCommandVersion()) - - //Generate bash completion file - //cmd.AddCommand(NewCommandBashCmp()) - - return cmd -} - -//CheckPersistentFlags check values is not empty and retun values -func CheckPersistentFlags() (*PersistentFlags, error) { - var err error - - //katib manager endpoint - s := viper.GetString("server") - if s == "" { - err = errors.New("katib manager endpoint is not specified,use --server provide value") - return nil, err - } - - rtn := PersistentFlags{ - server: s, - } - - return &rtn, err -} - -//initFlag manage persistent flags -func initFlag(cmd *cobra.Command) { - // cobra.OnInitialize(initConfig) - // add Pesistent flags - cmd.PersistentFlags().StringP("server", "s", "localhost:6789", "katib manager API endpoint") - - //bind viper - viper.BindPFlag("server", cmd.PersistentFlags().Lookup("server")) -} diff --git a/cmd/cli/command/type.go b/cmd/cli/command/type.go deleted file mode 100644 index e55c7523a33..00000000000 --- a/cmd/cli/command/type.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2018 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "github.com/kubeflow/katib/pkg/api" -) - -type StudyData struct { - StudyConf *api.StudyConfig - Models []*api.ModelInfo -} diff --git a/cmd/cli/katibctl.go b/cmd/cli/katibctl.go deleted file mode 100644 index 59217822d05..00000000000 --- a/cmd/cli/katibctl.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/kubeflow/katib/cmd/cli/command" -) - -//Entry point -func main() { - //init command - katibctl := command.NewRootCommand() - if err := katibctl.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } - -} diff --git a/docs/CLI/katib-cli.md b/docs/CLI/katib-cli.md deleted file mode 100644 index cff4c9814e5..00000000000 --- a/docs/CLI/katib-cli.md +++ /dev/null @@ -1,24 +0,0 @@ -## katib-cli - -katib cli - -### Synopsis - -This is katib cli client using cobra framework - -### Options - -``` - -h, --help help for katib-cli - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli create](katib-cli_create.md) - Create a resource from a file -* [katib-cli get](katib-cli_get.md) - Display one or many resources -* [katib-cli pull](katib-cli_pull.md) - Pull a resource from a file or from stdin. -* [katib-cli push](katib-cli_push.md) - Push a resource from a file or from stdin. -* [katib-cli stop](katib-cli_stop.md) - Stop a resource - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_create.md b/docs/CLI/katib-cli_create.md deleted file mode 100644 index 8b32391f00f..00000000000 --- a/docs/CLI/katib-cli_create.md +++ /dev/null @@ -1,26 +0,0 @@ -## katib-cli create - -Create a resource from a file - -### Synopsis - -YAML formats are accepted. - -### Options - -``` - -h, --help help for create -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli](katib-cli.md) - katib cli -* [katib-cli create study](katib-cli_create_study.md) - Create a study from a file - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_create_study.md b/docs/CLI/katib-cli_create_study.md deleted file mode 100644 index b08a39e47da..00000000000 --- a/docs/CLI/katib-cli_create_study.md +++ /dev/null @@ -1,30 +0,0 @@ -## katib-cli create study - -Create a study from a file - -### Synopsis - -YAML formats are accepted. - -``` -katib-cli create study [flags] -``` - -### Options - -``` - -f, --config string File path of study config(required) - -h, --help help for study -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli create](katib-cli_create.md) - Create a resource from a file - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_get.md b/docs/CLI/katib-cli_get.md deleted file mode 100644 index 71e7c19b182..00000000000 --- a/docs/CLI/katib-cli_get.md +++ /dev/null @@ -1,27 +0,0 @@ -## katib-cli get - -Display one or many resources - -### Synopsis - -list of resorces comannd can display includes: studies, study, trials, trial, models, model - -### Options - -``` - -h, --help help for get -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli](katib-cli.md) - katib cli -* [katib-cli get model](katib-cli_get_model.md) - Display Model Info -* [katib-cli get studies](katib-cli_get_studies.md) - Display Study lnfo - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_get_model.md b/docs/CLI/katib-cli_get_model.md deleted file mode 100644 index 07e040dc336..00000000000 --- a/docs/CLI/katib-cli_get_model.md +++ /dev/null @@ -1,30 +0,0 @@ -## katib-cli get model - -Display Model Info - -### Synopsis - -Display Information of saved model - -``` -katib-cli get model [flags] -``` - -### Options - -``` - -d, --detail Display detail information of Model - -h, --help help for model -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli get](katib-cli_get.md) - Display one or many resources - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_get_studies.md b/docs/CLI/katib-cli_get_studies.md deleted file mode 100644 index 2093cff1e8a..00000000000 --- a/docs/CLI/katib-cli_get_studies.md +++ /dev/null @@ -1,29 +0,0 @@ -## katib-cli get studies - -Display Study lnfo - -### Synopsis - -Display Information of a studies - -``` -katib-cli get studies [flags] -``` - -### Options - -``` - -h, --help help for studies -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli get](katib-cli_get.md) - Display one or many resources - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_pull.md b/docs/CLI/katib-cli_pull.md deleted file mode 100644 index a2102702482..00000000000 --- a/docs/CLI/katib-cli_pull.md +++ /dev/null @@ -1,26 +0,0 @@ -## katib-cli pull - -Pull a resource from a file or from stdin. - -### Synopsis - -YAML or JSON formats are accepted. - -### Options - -``` - -h, --help help for pull -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli](katib-cli.md) - katib cli -* [katib-cli pull studies](katib-cli_pull_studies.md) - Export a Study and its Models lnfo - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_pull_studies.md b/docs/CLI/katib-cli_pull_studies.md deleted file mode 100644 index 173855c3dd3..00000000000 --- a/docs/CLI/katib-cli_pull_studies.md +++ /dev/null @@ -1,30 +0,0 @@ -## katib-cli pull studies - -Export a Study and its Models lnfo - -### Synopsis - -Export Information of a Study and its Models to yaml format - -``` -katib-cli pull studies [flags] -``` - -### Options - -``` - -h, --help help for studies - -o, --output string File path to export -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli pull](katib-cli_pull.md) - Pull a resource from a file or from stdin. - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_push.md b/docs/CLI/katib-cli_push.md deleted file mode 100644 index f40a420a3c1..00000000000 --- a/docs/CLI/katib-cli_push.md +++ /dev/null @@ -1,26 +0,0 @@ -## katib-cli push - -Push a resource from a file or from stdin. - -### Synopsis - -YAML or JSON formats are accepted. - -### Options - -``` - -h, --help help for push -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli](katib-cli.md) - katib cli -* [katib-cli push study](katib-cli_push_study.md) - Push a Study Info and its Models from a file or from stdin - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_push_study.md b/docs/CLI/katib-cli_push_study.md deleted file mode 100644 index ea5de776d01..00000000000 --- a/docs/CLI/katib-cli_push_study.md +++ /dev/null @@ -1,31 +0,0 @@ -## katib-cli push study - -Push a Study Info and its Models from a file or from stdin - -### Synopsis - -Push a Study Info and its Models from a file or from stdin -YAML formats are accepted. - -``` -katib-cli push study [flags] -``` - -### Options - -``` - -f, --file string File path of model config file - -h, --help help for study -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli push](katib-cli_push.md) - Push a resource from a file or from stdin. - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_stop.md b/docs/CLI/katib-cli_stop.md deleted file mode 100644 index 7bbb2d103bc..00000000000 --- a/docs/CLI/katib-cli_stop.md +++ /dev/null @@ -1,26 +0,0 @@ -## katib-cli stop - -Stop a resource - -### Synopsis - -Specify resource ID or Name. - -### Options - -``` - -h, --help help for stop -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli](katib-cli.md) - katib cli -* [katib-cli stop study](katib-cli_stop_study.md) - Stop a study - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/docs/CLI/katib-cli_stop_study.md b/docs/CLI/katib-cli_stop_study.md deleted file mode 100644 index e1a178fffdb..00000000000 --- a/docs/CLI/katib-cli_stop_study.md +++ /dev/null @@ -1,29 +0,0 @@ -## katib-cli stop study - -Stop a study - -### Synopsis - -Stop study with study ID or study name - -``` -katib-cli stop study [flags] -``` - -### Options - -``` - -h, --help help for study -``` - -### Options inherited from parent commands - -``` - -s, --server string katib manager API endpoint (default "localhost:6789") -``` - -### SEE ALSO - -* [katib-cli stop](katib-cli_stop.md) - Stop a resource - -###### Auto generated by spf13/cobra on 5-Jun-2018 diff --git a/examples/README.md b/examples/README.md index a40a69ed302..403f2b0b3fc 100644 --- a/examples/README.md +++ b/examples/README.md @@ -394,53 +394,6 @@ They have same named metrics ('accuracy' and 'cross_entropy'). The accuracy in training and test will be saved in *train/* directory and *test/* directory respectively. In a studyjob, please add directry name to the name of metrics as a prefix e.g. `train/accuracy`, `test/accuracy`. -## ModelManagement - -You can export model data to yaml file with CLI. - -``` -katib-cli -s {{server-cli}} pull study {{study ID or name}} -o {{filename}} -``` - -And you can push your existing models to Katib with CLI. -`mnist-models.yaml` trains 22 models using random suggestion with the Parameter Config below. - -``` -configs: - - name: --lr - parametertype: 1 - feasible: - max: "0.07" - min: "0.03" - list: [] - - name: --lr-factor - parametertype: 1 - feasible: - max: "0.05" - min: "0.005" - list: [] - - name: --lr-step - parametertype: 2 - feasible: - max: "20" - min: "5" - list: [] - - name: --optimizer - parametertype: 4 - feasible: - max: "" - min: "" - list: - - sgd - - adam - - ftrl -``` -You can explore the model easily on KatibUI. - -``` -katib-cli push md -f mnist-models.yaml -``` - ## Clean Clean up with `./destroy.sh` script. It will stop port-forward process and delete minikube cluster. diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt deleted file mode 100644 index b92001fb4ed..00000000000 --- a/hack/boilerplate.go.txt +++ /dev/null @@ -1,14 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ \ No newline at end of file diff --git a/hack/docgen.go b/hack/docgen.go deleted file mode 100644 index 4c53588697c..00000000000 --- a/hack/docgen.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "log" - - "github.com/kubeflow/katib/cmd/cli/command" - "github.com/spf13/cobra/doc" -) - -func main() { - katibCLI := command.NewRootCommand() - err := doc.GenMarkdownTree(katibCLI, "./docs/CLI") - if err != nil { - log.Fatal(err) - } -} diff --git a/hack/gen-doc.sh b/hack/gen-doc.sh deleted file mode 100755 index 71e23d0be14..00000000000 --- a/hack/gen-doc.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Kubeflow Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -PREFIX="katib" -CMD_PREFIX="cmd" - -SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. - -cd ${SCRIPT_ROOT} -echo "Generating CLI documentation..." -go run hack/docgen.go -cd - > /dev/null diff --git a/scripts/build.sh b/scripts/build.sh index 9d895dbe935..e88e565abb4 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -47,6 +47,3 @@ docker build -t ${PREFIX}/earlystopping-medianstopping -f ${CMD_PREFIX}/earlysto echo "Building UI image..." docker build -t ${PREFIX}/katib-ui -f ${CMD_PREFIX}/ui/Dockerfile . -#echo "Building CLI image..." -#docker build -t ${PREFIX}/katib-cli -f ${CMD_PREFIX}/cli/Dockerfile . -#cd - > /dev/null diff --git a/test/scripts/build-cli.sh b/test/scripts/build-cli.sh deleted file mode 100755 index 0ca748087a2..00000000000 --- a/test/scripts/build-cli.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Kubeflow Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This shell script is used to build an image from our argo workflow - -set -o errexit -set -o nounset -set -o pipefail - -export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH} -REGISTRY="${GCP_REGISTRY}" -PROJECT="${GCP_PROJECT}" -GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME} -VERSION=$(git describe --tags --always --dirty) - -echo "Activating service-account" -gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} - -echo "Copy source to GOPATH" -mkdir -p ${GO_DIR} -cp -r cmd ${GO_DIR}/cmd -cp -r pkg ${GO_DIR}/pkg -cp -r vendor ${GO_DIR}/vendor - -cd ${GO_DIR} -go build -o bin/katib github.com/kubeflow/katib/cmd/cli diff --git a/test/scripts/unit-test.sh b/test/scripts/unit-test.sh index a169d09a9ae..6301c61168e 100755 --- a/test/scripts/unit-test.sh +++ b/test/scripts/unit-test.sh @@ -70,6 +70,12 @@ done kubectl port-forward $(kubectl get pod -l instance=mysql-ut -o=name) 3306:3306& export TEST_MYSQL=localhost +echo "Copy source to GOPATH" +mkdir -p ${GO_DIR} +cp -r cmd ${GO_DIR}/cmd +cp -r pkg ${GO_DIR}/pkg +cp -r vendor ${GO_DIR}/vendor + echo "Run unit test cases" cd ${GO_DIR} go test ./... diff --git a/test/workflows/components/workflows.libsonnet b/test/workflows/components/workflows.libsonnet index e5593228218..99fd0b53963 100644 --- a/test/workflows/components/workflows.libsonnet +++ b/test/workflows/components/workflows.libsonnet @@ -222,10 +222,6 @@ name: "build-studyjobctr", template: "build-studyjobctr", }, - { - name: "build-cli", - template: "build-cli", - }, { name: "build-suggestion-random", template: "build-suggestion-random", @@ -360,9 +356,6 @@ $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("build-studyjobctr", testWorkerImage, [ "test/scripts/build-studyjobctr.sh", ]), // build-studyjobctr - $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("build-cli", testWorkerImage, [ - "test/scripts/build-cli.sh", - ]), // build-cli $.parts(namespace, name, overrides).e2e(prow_env, bucket).buildTemplate("build-suggestion-random", testWorkerImage, [ "test/scripts/build-suggestion-random.sh", ]), // build-suggestion-random