feat: Rename algorithms (#794)

* feat: Rename algorithms

Signed-off-by: Ce Gao <gaoce@caicloud.io>

* fix: Remove prefix

Signed-off-by: Ce Gao <gaoce@caicloud.io>

* fix: Fix test cases

Signed-off-by: Ce Gao <gaoce@caicloud.io>

* fix: Fix algorithms

Signed-off-by: Ce Gao <gaoce@caicloud.io>

examples/v1alpha3/skopt-bayesian-optimization-example.yaml → examples/v1alpha3/bayesianoptimization-example.yaml

@@ -4,7 +4,7 @@ metadata:
   namespace: kubeflow
   labels:
     controller-tools.k8s.io: "1.0"
-  name: skopt-bayesian-optimization-example
+  name: bayesianoptimization-example
 spec:
   objective:
     type: maximize
@@ -13,7 +13,7 @@ spec:
     additionalMetricNames:
       - accuracy
   algorithm:
-    algorithmName: skopt-bayesian-optimization
+    algorithmName: bayesianoptimization
     algorithmSettings:
       - name: "random_state"
         value: "10"

examples/v1alpha3/chocolate-grid-example.yaml → examples/v1alpha3/grid-example.yaml

@@ -4,7 +4,7 @@ metadata:
   namespace: kubeflow
   labels:
     controller-tools.k8s.io: "1.0"
-  name: chocolate-grid-example
+  name: grid-example
 spec:
   objective:
     type: maximize
@@ -13,7 +13,7 @@ spec:
     additionalMetricNames:
       - accuracy
   algorithm:
-    algorithmName: chocolate-grid
+    algorithmName: grid
   parallelTrialCount: 3
   maxTrialCount: 12
   maxFailedTrialCount: 3

examples/v1alpha3/hyperopt-tpe-example.yaml → examples/v1alpha3/random-example.yaml

@@ -4,7 +4,7 @@ metadata:
   namespace: kubeflow
   labels:
     controller-tools.k8s.io: "1.0"
-  name: hyperopt-tpe-example
+  name: random-example
 spec:
   objective:
     type: maximize
@@ -13,7 +13,7 @@ spec:
     additionalMetricNames:
       - accuracy
   algorithm:
-    algorithmName: hyperopt-tpe
+    algorithmName: random
   parallelTrialCount: 3
   maxTrialCount: 12
   maxFailedTrialCount: 3

examples/v1alpha3/tfjob-example.yaml

@@ -12,7 +12,7 @@ spec:
     goal: 0.99
     objectiveMetricName: accuracy_1
   algorithm:
-    algorithmName: hyperopt-random
+    algorithmName: random
   metricsCollectorSpec:
     source:
       fileSystemPath:

examples/v1alpha3/hyperopt-random-example.yaml → examples/v1alpha3/tpe-example.yaml

@@ -4,7 +4,7 @@ metadata:
   namespace: kubeflow
   labels:
     controller-tools.k8s.io: "1.0"
-  name: hyperopt-random-example
+  name: tpe-example
 spec:
   objective:
     type: maximize
@@ -13,7 +13,7 @@ spec:
     additionalMetricNames:
       - accuracy
   algorithm:
-    algorithmName: hyperopt-random
+    algorithmName: tpe
   parallelTrialCount: 3
   maxTrialCount: 12
   maxFailedTrialCount: 3

manifests/v1alpha3/katib-controller/katib-config.yaml

@@ -19,30 +19,21 @@ data:
   suggestion: |-
     {
       "random": {
-        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-random"
+        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-hyperopt"
       },
       "grid": {
-        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-grid"
+        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-chocolate"
       },
       "hyperband": {
         "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-hyperband"
       },
       "bayesianoptimization": {
-        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-bayesianoptimization"
-      },
-      "nasrl": {
-        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-nasrl"
-      },
-      "chocolate-grid": {
-        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-chocolate"
-      },
-      "hyperopt-tpe": {
-        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-hyperopt"
+        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-skopt"
       },
-      "hyperopt-random": {
+      "tpe": {
         "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-hyperopt"
       },
-      "skopt-bayesian-optimization": {
-        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-skopt"
+      "nasrl": {
+        "image": "gcr.io/kubeflow-images-public/katib/v1alpha3/suggestion-nasrl"
       }
     }

pkg/suggestion/v1alpha3/chocolate/base_chocolate_service.py

@@ -46,8 +46,9 @@ def getSuggestions(self, search_space, trials, request_number):
 
         conn = choco.SQLiteConnection("sqlite:///my_db.db")
         # Refer to https://chocolate.readthedocs.io/tutorials/algo.html
-        if self.algorithm_name == "chocolate-grid":
+        if self.algorithm_name == "grid":
             sampler = choco.Grid(conn, chocolate_search_space, clear_db=True)
+        # hyperopt-random is the default option in katib.
         elif self.algorithm_name == "chocolate-random":
             sampler = choco.Random(conn, chocolate_search_space, clear_db=True)
         elif self.algorithm_name == "chocolate-quasirandom":
@@ -61,7 +62,8 @@ def getSuggestions(self, search_space, trials, request_number):
             mu = 1
             sampler = choco.MOCMAES(
                 conn, chocolate_search_space, mu=mu, clear_db=True)
-        logger.info("algortihm: %s", self.algorithm_name)
+        else:
+            raise Exception('"Failed to create the algortihm: {}'.format(self.algorithm_name))
 
         for index, trial in enumerate(trials):
             loss_for_choco = float(trial.target_metric.value)

pkg/suggestion/v1alpha3/hyperopt/base_hyperopt_service.py

@@ -9,9 +9,9 @@
 
 class BaseHyperoptService(object):
     def __init__(self, algorithm_name="tpe"):
-        if algorithm_name == 'hyperopt-tpe':
+        if algorithm_name == 'tpe':
             self.hyperopt_algorithm = hyperopt.tpe.suggest
-        elif algorithm_name == 'hyperopt-random':
+        elif algorithm_name == 'random':
             self.hyperopt_algorithm = hyperopt.rand.suggest
         # elif algorithm_name == 'hyperopt-anneal':
         #     self.hyperopt_algorithm = hyperopt.anneal.suggest_batch

pkg/suggestion/v1alpha3/skopt/base_skopt_service.py

@@ -45,8 +45,9 @@ def getSuggestions(self, search_space, trials, request_number):
                 skopt_search_space.append(
                     skopt.space.Categorical(param.list, name=param.name))
 
-        if self.algorithm_name != "skopt-bayesian-optimization":
-            raise Exception("Algorithm name is not supported by skopt service.")
+        if self.algorithm_name != "bayesianoptimization":
+            raise Exception(
+                '"Failed to create the algortihm: {}'.format(self.algorithm_name))
         skopt_optimizer = skopt.Optimizer(skopt_search_space,
                                           base_estimator=self.base_estimator,
                                           n_initial_points=self.n_initial_points,

pkg/suggestion/v1alpha3/skopt_service.py

@@ -62,20 +62,3 @@ def convertAlgorithmSpec(algorithm_spec):
             elif s.name == "random_state":
                 optmizer.random_state = int(s.value)
         return algorithm_spec.algorithm_name, optmizer
-
-
-class HyperoptService(
-        api_pb2_grpc.SuggestionServicer):
-    def GetSuggestions(self, request, context):
-        """
-        Main function to provide suggestion.
-        """
-        base_serice = BaseHyperoptService(
-            algorithm_name=request.experiment.spec.algorithm.algorithm_name)
-        search_space = HyperParameterSearchSpace.convert(request.experiment)
-        trials = Trial.convert(request.trials)
-        new_assignments = base_serice.getSuggestions(
-            search_space, trials, request.request_number)
-        return api_pb2.GetSuggestionsReply(
-            parameter_assignments=Assignment.generate(new_assignments)
-        )

test/scripts/v1alpha3/run-suggestion-bayesian.sh

@@ -61,7 +61,7 @@ cd ${GO_DIR}/test/e2e/v1alpha3
 
 echo "Running e2e skopt bayesian optimization experiment"
 export KUBECONFIG=$HOME/.kube/config
-go run run-e2e-experiment.go ../../../examples/v1alpha3/skopt-bayesian-optimization-example.yaml
+go run run-e2e-experiment.go ../../../examples/v1alpha3/bayesianoptimization-example.yaml
 kubectl -n kubeflow describe suggestion
-kubectl -n kubeflow delete experiment skopt-bayesian-optimization-example
+kubectl -n kubeflow delete experiment bayesianoptimization-example
 exit 0

test/scripts/v1alpha3/run-suggestion-grid.sh

@@ -61,7 +61,7 @@ cd ${GO_DIR}/test/e2e/v1alpha3
 
 echo "Running e2e chocolate grid experiment"
 export KUBECONFIG=$HOME/.kube/config
-go run run-e2e-experiment.go ../../../examples/v1alpha3/chocolate-grid-example.yaml
+go run run-e2e-experiment.go ../../../examples/v1alpha3/grid-example.yaml
 kubectl -n kubeflow describe suggestion
-kubectl -n kubeflow delete experiment chocolate-grid-example
+kubectl -n kubeflow delete experiment grid-example
 exit 0

test/scripts/v1alpha3/run-suggestion-random.sh

@@ -61,7 +61,7 @@ cd ${GO_DIR}/test/e2e/v1alpha3
 
 echo "Running e2e hyperopt random experiment"
 export KUBECONFIG=$HOME/.kube/config
-go run run-e2e-experiment.go ../../../examples/v1alpha3/hyperopt-random-example.yaml
+go run run-e2e-experiment.go ../../../examples/v1alpha3/random-example.yaml
 kubectl -n kubeflow describe suggestion
-kubectl -n kubeflow delete experiment hyperopt-random-example
+kubectl -n kubeflow delete experiment random-example
 exit 0

test/suggestion/v1alpha3/test_chocolate_service.py.failed

@@ -113,7 +113,7 @@ class TestHyperopt(unittest.TestCase):
             name="test",
             spec=api_pb2.ExperimentSpec(
                 algorithm=api_pb2.AlgorithmSpec(
-                    algorithm_name="chocolate-grid",
+                    algorithm_name="grid",
                     algorithm_setting=[
                         api_pb2.AlgorithmSetting(
                             name="random_state",

test/suggestion/v1alpha3/test_hyperopt_service.py

@@ -113,7 +113,7 @@ def test_get_suggestion(self):
             name="test",
             spec=api_pb2.ExperimentSpec(
                 algorithm=api_pb2.AlgorithmSpec(
-                    algorithm_name="hyperopt-tpe",
+                    algorithm_name="tpe",
                     algorithm_setting=[
                         api_pb2.AlgorithmSetting(
                             name="random_state",

test/suggestion/v1alpha3/test_skopt_service.py

@@ -8,7 +8,7 @@
 from pkg.suggestion.v1alpha3.skopt_service import SkoptService
 
 
-class TestHyperopt(unittest.TestCase):
+class TestSkopt(unittest.TestCase):
     def setUp(self):
         servicers = {
             api_pb2.DESCRIPTOR.services_by_name['Suggestion']: SkoptService(
@@ -113,7 +113,7 @@ def test_get_suggestion(self):
             name="test",
             spec=api_pb2.ExperimentSpec(
                 algorithm=api_pb2.AlgorithmSpec(
-                    algorithm_name="skopt-bayesian-optimization",
+                    algorithm_name="bayesianoptimization",
                     algorithm_setting=[
                         api_pb2.AlgorithmSetting(
                             name="random_state",