Katib v1beta1 version (kubeflow#1197)

* Add v1beta1 version

* Swagger for v1alpha3 and v1beta1 versions
Fix format in bash scripts

* Change make build to make buildv1alpha3

* Add folder path to python test

* Fix folder in python test

* Add goptuna and darts suggestions to check-katib-ready

* Disable custom metrics collector e2e in v1beta1

.dockerignore

@@ -3,6 +3,7 @@
 docs
 examples
 !examples/v1alpha3/nas
+!examples/v1beta1/nas
 manifests
 pkg/ui/*/frontend/node_modules
 pkg/ui/*/frontend/build

.travis.yml

@@ -6,7 +6,7 @@ jobs:
       go_import_path: github.com/kubeflow/katib
       install:
         - curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v1.0.7/kubebuilder_1.0.7_linux_amd64.tar.gz"
-        - # extract the archive
+        -  # extract the archive
         - tar -zxvf kubebuilder_1.0.7_linux_amd64.tar.gz
         - sudo mv kubebuilder_1.0.7_linux_amd64 /usr/local/kubebuilder
         - export PATH=$PATH:/usr/local/kubebuilder/bin
@@ -22,4 +22,5 @@ jobs:
       install:
         - npm install --global prettier@1.19.1
       script:
+        - make prettier-check-v1alpha3
         - make prettier-check

Makefile

@@ -33,23 +33,41 @@ vet: depend generate
 update:
 	hack/update-gofmt.sh
 
-# Deploy katib v1alpha3 manifests into a k8s cluster
-deploy: 
+# Deploy Katib v1alpha3 manifests into a k8s cluster
+deployv1alpha3:
 	bash scripts/v1alpha3/deploy.sh
 
-# Undeploy katib v1alpha3 manifests into a k8s cluster
-undeploy:
+# Deploy Katib v1beta1 manifests into a k8s cluster
+deploy:
+	bash scripts/v1beta1/deploy.sh
+
+# Undeploy Katib v1alpha3 manifests from a k8s cluster
+undeployv1alpha3:
 	bash scripts/v1alpha3/undeploy.sh
 
+# Undeploy Katib v1beta1 manifests from a k8s cluster
+undeploy:
+	bash scripts/v1beta1/undeploy.sh
+
 # Generate code
 generate:
 ifndef GOPATH
 	$(error GOPATH not defined, please define GOPATH. Run "go help gopath" to learn more about GOPATH)
 endif
 	go generate ./pkg/... ./cmd/...
 
-build: depend generate
+# Build images for Katib v1alpha3 components
+buildv1alpha3: depend generate
 	bash scripts/v1alpha3/build.sh
 
-prettier-check:
+# Build images for Katib v1beta1 components
+build: depend generate
+	bash scripts/v1beta1/build.sh
+
+# Prettier UI format check for Katib v1alpha3
+prettier-check-v1alpha3:
 	npm run format:check --prefix pkg/ui/v1alpha3/frontend
+
+# Prettier UI format check for Katib v1beta1
+prettier-check:
+	npm run format:check --prefix pkg/ui/v1beta1/frontend

cmd/db-manager/v1beta1/Dockerfile

@@ -0,0 +1,26 @@
+FROM golang:alpine AS build-env
+# The GOPATH in the image is /go.
+ADD . /go/src/github.com/kubeflow/katib
+WORKDIR /go/src/github.com/kubeflow/katib/cmd/db-manager
+RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
+    apk --update add git gcc musl-dev && \
+    go build -o katib-db-manager ./v1beta1; \
+    else \
+    go build -o katib-db-manager ./v1beta1; \
+    fi
+RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
+    if [ "$(uname -m)" = "ppc64le" ]; then \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
+    elif [ "$(uname -m)" = "aarch64" ]; then \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \
+    else \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64; \
+    fi && \
+    chmod +x /bin/grpc_health_probe
+
+FROM alpine:3.7
+WORKDIR /app
+COPY --from=build-env /bin/grpc_health_probe /bin/
+COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/db-manager/katib-db-manager /app/
+ENTRYPOINT ["./katib-db-manager"]
+CMD ["-w", "kubernetes"]

cmd/db-manager/v1beta1/main.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+
+	health_pb "github.com/kubeflow/katib/pkg/apis/manager/health"
+	api_pb "github.com/kubeflow/katib/pkg/apis/manager/v1beta1"
+	db "github.com/kubeflow/katib/pkg/db/v1beta1"
+	"github.com/kubeflow/katib/pkg/db/v1beta1/common"
+	"k8s.io/klog"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
+)
+
+const (
+	port = "0.0.0.0:6789"
+)
+
+var dbIf common.KatibDBInterface
+
+type server struct {
+}
+
+// Report a log of Observations for a Trial.
+// The log consists of timestamp and value of metric.
+// Katib store every log of metrics.
+// You can see accuracy curve or other metric logs on UI.
+func (s *server) ReportObservationLog(ctx context.Context, in *api_pb.ReportObservationLogRequest) (*api_pb.ReportObservationLogReply, error) {
+	err := dbIf.RegisterObservationLog(in.TrialName, in.ObservationLog)
+	return &api_pb.ReportObservationLogReply{}, err
+}
+
+// Get all log of Observations for a Trial.
+func (s *server) GetObservationLog(ctx context.Context, in *api_pb.GetObservationLogRequest) (*api_pb.GetObservationLogReply, error) {
+	ol, err := dbIf.GetObservationLog(in.TrialName, in.MetricName, in.StartTime, in.EndTime)
+	return &api_pb.GetObservationLogReply{
+		ObservationLog: ol,
+	}, err
+}
+
+// Delete all log of Observations for a Trial.
+func (s *server) DeleteObservationLog(ctx context.Context, in *api_pb.DeleteObservationLogRequest) (*api_pb.DeleteObservationLogReply, error) {
+	err := dbIf.DeleteObservationLog(in.TrialName)
+	return &api_pb.DeleteObservationLogReply{}, err
+}
+
+func (s *server) Check(ctx context.Context, in *health_pb.HealthCheckRequest) (*health_pb.HealthCheckResponse, error) {
+	resp := health_pb.HealthCheckResponse{
+		Status: health_pb.HealthCheckResponse_SERVING,
+	}
+
+	// We only accept optional service name only if it's set to suggested format.
+	if in != nil && in.Service != "" && in.Service != "grpc.health.v1.Health" {
+		resp.Status = health_pb.HealthCheckResponse_UNKNOWN
+		return &resp, fmt.Errorf("grpc.health.v1.Health can only be accepted if you specify service name.")
+	}
+
+	// Check if connection to katib db driver is okay since otherwise manager could not serve most of its methods.
+	err := dbIf.SelectOne()
+	if err != nil {
+		resp.Status = health_pb.HealthCheckResponse_NOT_SERVING
+		return &resp, fmt.Errorf("Failed to execute `SELECT 1` probe: %v", err)
+	}
+
+	return &resp, nil
+}
+
+func main() {
+	flag.Parse()
+	var err error
+	dbNameEnvName := common.DBNameEnvName
+	dbName := os.Getenv(dbNameEnvName)
+	if dbName == "" {
+		klog.Fatal("DB_NAME env is not set. Exiting")
+	}
+	dbIf, err = db.NewKatibDBInterface(dbName)
+	if err != nil {
+		klog.Fatalf("Failed to open db connection: %v", err)
+	}
+	dbIf.DBInit()
+	listener, err := net.Listen("tcp", port)
+	if err != nil {
+		klog.Fatalf("Failed to listen: %v", err)
+	}
+
+	size := 1<<31 - 1
+	klog.Infof("Start Katib manager: %s", port)
+	s := grpc.NewServer(grpc.MaxRecvMsgSize(size), grpc.MaxSendMsgSize(size))
+	api_pb.RegisterManagerServer(s, &server{})
+	health_pb.RegisterHealthServer(s, &server{})
+	reflection.Register(s)
+	if err = s.Serve(listener); err != nil {
+		klog.Fatalf("Failed to serve: %v", err)
+	}
+}

cmd/db-manager/v1beta1/main_test.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+
+	health_pb "github.com/kubeflow/katib/pkg/apis/manager/health"
+	api_pb "github.com/kubeflow/katib/pkg/apis/manager/v1beta1"
+	mockdb "github.com/kubeflow/katib/pkg/mock/v1beta1/db"
+)
+
+func TestReportObservationLog(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	s := &server{}
+	mockDB := mockdb.NewMockKatibDBInterface(ctrl)
+	dbIf = mockDB
+
+	req := &api_pb.ReportObservationLogRequest{
+		TrialName: "test1-trial1",
+		ObservationLog: &api_pb.ObservationLog{
+			MetricLogs: []*api_pb.MetricLog{
+				{
+					TimeStamp: "2019-02-03T04:05:06+09:00",
+					Metric: &api_pb.Metric{
+						Name:  "f1_score",
+						Value: "88.95",
+					},
+				},
+				{
+					TimeStamp: "2019-02-03T04:05:06+09:00",
+					Metric: &api_pb.Metric{
+						Name:  "loss",
+						Value: "0.5",
+					},
+				},
+				{
+					TimeStamp: "2019-02-03T04:05:06+09:00",
+					Metric: &api_pb.Metric{
+						Name:  "precision",
+						Value: "88.7",
+					},
+				},
+				{
+					TimeStamp: "2019-02-03T04:05:06+09:00",
+					Metric: &api_pb.Metric{
+						Name:  "recall",
+						Value: "89.2",
+					},
+				},
+			},
+		},
+	}
+	mockDB.EXPECT().RegisterObservationLog(req.TrialName, req.ObservationLog).Return(nil)
+	_, err := s.ReportObservationLog(context.Background(), req)
+	if err != nil {
+		t.Fatalf("ReportObservationLog Error %v", err)
+	}
+}
+
+func TestGetObservationLog(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	s := &server{}
+	mockDB := mockdb.NewMockKatibDBInterface(ctrl)
+	dbIf = mockDB
+
+	req := &api_pb.GetObservationLogRequest{
+		TrialName: "test1-trial1",
+		StartTime: "2019-02-03T03:05:06+09:00",
+		EndTime:   "2019-02-03T05:05:06+09:00",
+	}
+
+	obs := &api_pb.ObservationLog{
+		MetricLogs: []*api_pb.MetricLog{
+			{
+				TimeStamp: "2019-02-03T04:05:06+09:00",
+				Metric: &api_pb.Metric{
+					Name:  "f1_score",
+					Value: "88.95",
+				},
+			},
+			{
+				TimeStamp: "2019-02-03T04:05:06+09:00",
+				Metric: &api_pb.Metric{
+					Name:  "loss",
+					Value: "0.5",
+				},
+			},
+			{
+				TimeStamp: "2019-02-03T04:05:06+09:00",
+				Metric: &api_pb.Metric{
+					Name:  "precision",
+					Value: "88.7",
+				},
+			},
+			{
+				TimeStamp: "2019-02-03T04:05:06+09:00",
+				Metric: &api_pb.Metric{
+					Name:  "recall",
+					Value: "89.2",
+				},
+			},
+		},
+	}
+
+	mockDB.EXPECT().GetObservationLog(req.TrialName, req.MetricName, req.StartTime, req.EndTime).Return(obs, nil)
+	ret, err := s.GetObservationLog(context.Background(), req)
+	if err != nil {
+		t.Fatalf("GetObservationLog Error %v", err)
+	}
+	if len(obs.MetricLogs) != len(ret.ObservationLog.MetricLogs) {
+		t.Fatalf("GetObservationLog Test fail expect metrics number %d got %d", len(obs.MetricLogs), len(ret.ObservationLog.MetricLogs))
+	}
+}
+
+func TestDeleteObservationLog(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	s := &server{}
+	mockDB := mockdb.NewMockKatibDBInterface(ctrl)
+	dbIf = mockDB
+
+	req := &api_pb.DeleteObservationLogRequest{
+		TrialName: "test1-trial1",
+	}
+	mockDB.EXPECT().DeleteObservationLog(req.TrialName).Return(nil)
+	_, err := s.DeleteObservationLog(context.Background(), req)
+	if err != nil {
+		t.Fatalf("DeleteExperiment Error %v", err)
+	}
+}
+
+func TestCheck(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	s := &server{}
+	mockDB := mockdb.NewMockKatibDBInterface(ctrl)
+	dbIf = mockDB
+	testCases := []struct {
+		Request        *health_pb.HealthCheckRequest
+		ExpectedStatus health_pb.HealthCheckResponse_ServingStatus
+		Name           string
+	}{
+		{
+			Request: &health_pb.HealthCheckRequest{
+				Service: "grpc.health.v1.Health",
+			},
+			ExpectedStatus: health_pb.HealthCheckResponse_SERVING,
+			Name:           "Valid Request",
+		},
+		{
+			Request: &health_pb.HealthCheckRequest{
+				Service: "grpc.health.v1.1.Health",
+			},
+			ExpectedStatus: health_pb.HealthCheckResponse_UNKNOWN,
+			Name:           "Invalid service name",
+		},
+	}
+
+	mockDB.EXPECT().SelectOne().Return(nil)
+
+	for _, tc := range testCases {
+		response, _ := s.Check(context.Background(), tc.Request)
+		if response.Status != tc.ExpectedStatus {
+			t.Fatalf("Case %v failed. ExpectedStatus %v, got %v", tc.Name, tc.ExpectedStatus, response.Status)
+		}
+	}
+}

cmd/katib-controller/v1beta1/Dockerfile

@@ -0,0 +1,22 @@
+# Build the manager binary
+FROM golang:alpine AS build-env
+
+# Copy in the go src
+ADD . /go/src/github.com/kubeflow/katib
+
+WORKDIR /go/src/github.com/kubeflow/katib/cmd/katib-controller
+# Build
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
+    CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build -a -o katib-controller  ./v1beta1; \
+    elif [ "$(uname -m)" = "aarch64" ]; then \
+    CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o katib-controller  ./v1beta1; \
+    else \
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o katib-controller  ./v1beta1; \
+    fi
+# Copy the controller-manager into a thin image
+FROM alpine:3.7
+WORKDIR /app
+RUN apk update && apk add ca-certificates
+COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/katib-controller/katib-controller .
+USER 1000
+ENTRYPOINT ["./katib-controller"]