[enh]: validate for bayesian optimization algorithm settings (#1600)

* [enh]: validate for skopt algorithm settings

* [style]: refactor with reviews

- use staticmethod rather than classmethod
- change convertAlgorithmSpec method name to a snake_case
- use .format() rather than f-string

Signed-off-by: Jaeyeon Kim <anencore94@gmail.com>

pkg/suggestion/v1beta1/skopt/service.py

@@ -14,13 +14,14 @@
 
 import logging
 
+import grpc
+
 from pkg.apis.manager.v1beta1.python import api_pb2
 from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer
 from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace
 from pkg.suggestion.v1beta1.internal.trial import Trial, Assignment
 from pkg.suggestion.v1beta1.skopt.base_service import BaseSkoptService
-from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer
-
 
 logger = logging.getLogger(__name__)
 
@@ -36,10 +37,8 @@ def GetSuggestions(self, request, context):
         """
         Main function to provide suggestion.
         """
-        algorithm_name, config = OptimizerConfiguration.convertAlgorithmSpec(
+        algorithm_name, config = OptimizerConfiguration.convert_algorithm_spec(
             request.experiment.spec.algorithm)
-        if algorithm_name != "bayesianoptimization":
-            raise Exception("Failed to create the algorithm: {}".format(algorithm_name))
 
         if self.is_first_run:
             search_space = HyperParameterSearchSpace.convert(request.experiment)
@@ -58,6 +57,15 @@ def GetSuggestions(self, request, context):
             parameter_assignments=Assignment.generate(new_trials)
         )
 
+    def ValidateAlgorithmSettings(self, request, context):
+        is_valid, message = OptimizerConfiguration.validate_algorithm_spec(
+            request.experiment.spec.algorithm)
+        if not is_valid:
+            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
+            context.set_details(message)
+            logger.error(message)
+        return api_pb2.ValidateAlgorithmSettingsReply()
+
 
 class OptimizerConfiguration(object):
     def __init__(self, base_estimator="GP",
@@ -72,7 +80,7 @@ def __init__(self, base_estimator="GP",
         self.random_state = random_state
 
     @staticmethod
-    def convertAlgorithmSpec(algorithm_spec):
+    def convert_algorithm_spec(algorithm_spec):
         optimizer = OptimizerConfiguration()
         for s in algorithm_spec.algorithm_settings:
             if s.name == "base_estimator":
@@ -86,3 +94,39 @@ def convertAlgorithmSpec(algorithm_spec):
             elif s.name == "random_state":
                 optimizer.random_state = int(s.value)
         return algorithm_spec.algorithm_name, optimizer
+
+    @classmethod
+    def validate_algorithm_spec(cls, algorithm_spec):
+        algo_name = algorithm_spec.algorithm_name
+
+        if algo_name == "bayesianoptimization":
+            return cls._validate_bayesianoptimization_setting(algorithm_spec.algorithm_settings)
+        else:
+            return False, "unknown algorithm name {}".format(algo_name)
+
+    @classmethod
+    def _validate_bayesianoptimization_setting(cls, algorithm_settings):
+        for s in algorithm_settings:
+            try:
+                if s.name == "base_estimator":
+                    if s.value not in ["GP", "RF", "ET", "GBRT"]:
+                        return False, "base_estimator {} is not supported in Bayesian optimization".format(s.value)
+                elif s.name == "n_initial_points":
+                    if not (int(s.value) >= 0):
+                        return False, "n_initial_points should be great or equal than zero"
+                elif s.name == "acq_func":
+                    if s.value not in ["gp_hedge", "LCB", "EI", "PI", "EIps", "PIps"]:
+                        return False, "acq_func {} is not supported in Bayesian optimization".format(s.value)
+                elif s.name == "acq_optimizer":
+                    if s.value not in ["auto", "sampling", "lbfgs"]:
+                        return False, "acq_optimizer {} is not supported in Bayesian optimization".format(s.value)
+                elif s.name == "random_state":
+                    if not (int(s.value) >= 0):
+                        return False, "random_state should be great or equal than zero"
+                else:
+                    return False, "unknown setting {} for algorithm bayesianoptimization".format(s.name)
+            except Exception as e:
+                return False, "failed to validate {name}({value}): {exception}".format(name=s.name, value=s.value,
+                                                                                       exception=e)
+
+        return True, ""

test/suggestion/v1beta1/test_skopt_service.py

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import unittest
+
 import grpc
 import grpc_testing
-import unittest
 
 from pkg.apis.manager.v1beta1.python import api_pb2
-
 from pkg.suggestion.v1beta1.skopt.service import SkoptService
 
 
@@ -177,8 +177,8 @@ def test_get_suggestion(self):
 
         get_suggestion = self.test_server.invoke_unary_unary(
             method_descriptor=(api_pb2.DESCRIPTOR
-                               .services_by_name['Suggestion']
-                               .methods_by_name['GetSuggestions']),
+                .services_by_name['Suggestion']
+                .methods_by_name['GetSuggestions']),
             invocation_metadata={},
             request=request, timeout=1)
 
@@ -187,6 +187,125 @@ def test_get_suggestion(self):
         self.assertEqual(code, grpc.StatusCode.OK)
         self.assertEqual(2, len(response.parameter_assignments))
 
+    def test_validate_algorithm_settings(self):
+        experiment_spec = [None]
+
+        def call_validate():
+            experiment = api_pb2.Experiment(name="test", spec=experiment_spec[0])
+            request = api_pb2.ValidateAlgorithmSettingsRequest(experiment=experiment)
+
+            validate_algorithm_settings = self.test_server.invoke_unary_unary(
+                method_descriptor=(api_pb2.DESCRIPTOR
+                    .services_by_name['Suggestion']
+                    .methods_by_name['ValidateAlgorithmSettings']),
+                invocation_metadata={},
+                request=request, timeout=1)
+
+            return validate_algorithm_settings.termination()
+
+        # valid cases
+        algorithm_spec = api_pb2.AlgorithmSpec(
+            algorithm_name="bayesianoptimization",
+            algorithm_settings=[
+                api_pb2.AlgorithmSetting(
+                    name="random_state",
+                    value="10"
+                )
+            ],
+        )
+        experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec)
+        self.assertEqual(call_validate()[2], grpc.StatusCode.OK)
+
+        # invalid cases
+        # unknown algorithm name
+        experiment_spec[0] = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
+        _, _, code, details = call_validate()
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details, 'unknown algorithm name unknown')
+
+        # unknown config name
+        experiment_spec[0] = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="bayesianoptimization",
+                algorithm_settings=[
+                    api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")]
+            ))
+        _, _, code, details = call_validate()
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details, 'unknown setting unknown_conf for algorithm bayesianoptimization')
+
+        # unknown base_estimator
+        experiment_spec[0] = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="bayesianoptimization",
+                algorithm_settings=[
+                    api_pb2.AlgorithmSetting(name="base_estimator", value="unknown estimator")]
+            ))
+        _, _, code, details = call_validate()
+        wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details,
+                         "{name} {value} is not supported in Bayesian optimization".format(
+                             name=wrong_algorithm_setting.name,
+                             value=wrong_algorithm_setting.value))
+
+        # wrong n_initial_points
+        experiment_spec[0] = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="bayesianoptimization",
+                algorithm_settings=[
+                    api_pb2.AlgorithmSetting(name="n_initial_points", value="-1")]
+            ))
+        _, _, code, details = call_validate()
+        wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details, "{name} should be great or equal than zero".format(name=wrong_algorithm_setting.name))
+
+        # unknown acq_func
+        experiment_spec[0] = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="bayesianoptimization",
+                algorithm_settings=[
+                    api_pb2.AlgorithmSetting(name="acq_func", value="unknown")]
+            ))
+        _, _, code, details = call_validate()
+        wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details,
+                         "{name} {value} is not supported in Bayesian optimization".format(
+                             name=wrong_algorithm_setting.name,
+                             value=wrong_algorithm_setting.value
+                         ))
+
+        # unknown acq_optimizer
+        experiment_spec[0] = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="bayesianoptimization",
+                algorithm_settings=[
+                    api_pb2.AlgorithmSetting(name="acq_optimizer", value="unknown")]
+            ))
+        _, _, code, details = call_validate()
+        wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details,
+                         "{name} {value} is not supported in Bayesian optimization".format(
+                             name=wrong_algorithm_setting.name,
+                             value=wrong_algorithm_setting.value
+                         ))
+
+        # wrong random_state
+        experiment_spec[0] = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="bayesianoptimization",
+                algorithm_settings=[
+                    api_pb2.AlgorithmSetting(name="random_state", value="-1")]
+            ))
+        _, _, code, details = call_validate()
+        wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details, "{name} should be great or equal than zero".format(name=wrong_algorithm_setting.name))
+
 
 if __name__ == '__main__':
     unittest.main()