Refactor suggestion services folder structure (#1166)

* Refactor suggestion services folder structure

* Move base health service to internal

* Rename service and base service files

cmd/suggestion/chocolate/v1alpha3/main.py

@@ -2,7 +2,7 @@
 import time
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.health.python import health_pb2_grpc
-from pkg.suggestion.v1alpha3.chocolate_service import ChocolateService
+from pkg.suggestion.v1alpha3.chocolate.service import ChocolateService
 from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24

cmd/suggestion/hyperband/v1alpha3/main.py

@@ -2,7 +2,7 @@
 import time
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.health.python import health_pb2_grpc
-from pkg.suggestion.v1alpha3.hyperband_service import HyperbandService
+from pkg.suggestion.v1alpha3.hyperband.service import HyperbandService
 from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24

cmd/suggestion/hyperopt/v1alpha3/main.py

@@ -2,7 +2,7 @@
 import time
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.health.python import health_pb2_grpc
-from pkg.suggestion.v1alpha3.hyperopt_service import HyperoptService
+from pkg.suggestion.v1alpha3.hyperopt.service import HyperoptService
 from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24

cmd/suggestion/nas/enas/v1alpha3/main.py

@@ -4,7 +4,7 @@
 
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.health.python import health_pb2_grpc
-from pkg.suggestion.v1alpha3.nas.enas_service import EnasService
+from pkg.suggestion.v1alpha3.nas.enas.service import EnasService
 
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24

cmd/suggestion/skopt/v1alpha3/main.py

@@ -2,7 +2,7 @@
 import time
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.health.python import health_pb2_grpc
-from pkg.suggestion.v1alpha3.skopt_service import SkoptService
+from pkg.suggestion.v1alpha3.skopt.service import SkoptService
 from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24

docs/new-algorithm-service.md

@@ -17,7 +17,7 @@ from pkg.apis.manager.v1alpha3.python import api_pb2
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.suggestion.v1alpha3.internal.search_space import HyperParameter, HyperParameterSearchSpace
 from pkg.suggestion.v1alpha3.internal.trial import Trial, Assignment
-from pkg.suggestion.v1alpha3.hyperopt.base_hyperopt_service import BaseHyperoptService
+from pkg.suggestion.v1alpha3.hyperopt.base_service import BaseHyperoptService
 from pkg.suggestion.v1alpha3.base_health_service import HealthServicer
 
 
@@ -122,7 +122,7 @@ import unittest
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.v1alpha3.python import api_pb2
 
-from pkg.suggestion.v1alpha3.hyperopt_service import HyperoptService
+from pkg.suggestion.v1alpha3.hyperopt.service import HyperoptService
 
 class TestHyperopt(unittest.TestCase):
     def setUp(self):

pkg/suggestion/v1alpha3/bayesianoptimization/src/algorithm_manager.py → pkg/suggestion/v1alpha3/bayesianoptimization/algorithm_manager.py

@@ -3,7 +3,7 @@
 
 from pkg.apis.manager.v1alpha3.python import api_pb2
 
-from .utils import get_logger
+from pkg.suggestion.v1alpha3.bayesianoptimization.utils import get_logger
 
 
 def deal_with_discrete(feasible_values, current_value):

pkg/suggestion/v1alpha3/bayesianoptimization/src/bayesian_optimization_algorithm.py → pkg/suggestion/v1alpha3/bayesianoptimization/bayesian_optimization_algorithm.py

@@ -2,7 +2,7 @@
 import numpy as np
 from sklearn.preprocessing import MinMaxScaler
 
-from .global_optimizer import GlobalOptimizer
+from pkg.suggestion.v1alpha3.bayesianoptimization.global_optimizer import GlobalOptimizer
 
 
 class BOAlgorithm:
@@ -31,7 +31,7 @@ def __init__(self, experiment_name, dim, N, lowerbound, upperbound, X_train, y_t
         else:
             self.current_optimal = max(self.y_train)
             self.logger.debug("create optimizer", extra={
-            "Experiment": self._experiment_name})
+                "Experiment": self._experiment_name})
             # initialize the global optimizer
             self.optimizer = GlobalOptimizer(
                 N,

pkg/suggestion/v1alpha3/bayesianoptimization/src/global_optimizer.py → pkg/suggestion/v1alpha3/bayesianoptimization/global_optimizer.py

@@ -5,10 +5,10 @@
 
 import numpy as np
 
-from .acquisition_func import AcquisitionFunc
-from .model.gp import GaussianProcessModel
-from .model.rf import RandomForestModel
-from .utils import get_logger
+from pkg.suggestion.v1alpha3.bayesianoptimization.acquisition_func import AcquisitionFunc
+from pkg.suggestion.v1alpha3.bayesianoptimization.model.gp import GaussianProcessModel
+from pkg.suggestion.v1alpha3.bayesianoptimization.model.rf import RandomForestModel
+from pkg.suggestion.v1alpha3.bayesianoptimization.utils import get_logger
 
 
 class RectPack:

pkg/suggestion/v1alpha3/chocolate_service.py → pkg/suggestion/v1alpha3/chocolate/service.py

@@ -4,11 +4,11 @@
 from pkg.apis.manager.v1alpha3.python import api_pb2
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 
-from pkg.suggestion.v1alpha3.internal.constant import *
+from pkg.suggestion.v1alpha3.internal.constant import DOUBLE
 from pkg.suggestion.v1alpha3.internal.search_space import HyperParameterSearchSpace
 from pkg.suggestion.v1alpha3.internal.trial import Trial, Assignment
-from pkg.suggestion.v1alpha3.chocolate.base_chocolate_service import BaseChocolateService
-from pkg.suggestion.v1alpha3.base_health_service import HealthServicer
+from pkg.suggestion.v1alpha3.chocolate.base_service import BaseChocolateService
+from pkg.suggestion.v1alpha3.internal.base_health_service import HealthServicer
 
 logger = logging.getLogger(__name__)
 

pkg/suggestion/v1alpha3/parsing_util.py → pkg/suggestion/v1alpha3/hyperband/parsing_util.py

@@ -6,7 +6,7 @@
 from collections.abc import Iterable
 from pkg.apis.manager.v1alpha3.python import api_pb2
 import numpy as np
-from .parameter import ParameterConfig
+from pkg.suggestion.v1alpha3.hyperband.parameter import ParameterConfig
 
 
 def _deal_with_discrete(feasible_values, current_value):

pkg/suggestion/v1alpha3/hyperband_service.py → pkg/suggestion/v1alpha3/hyperband/service.py

@@ -6,8 +6,8 @@
 
 from pkg.apis.manager.v1alpha3.python import api_pb2
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
-from pkg.suggestion.v1alpha3 import parsing_util
-from pkg.suggestion.v1alpha3.base_health_service import HealthServicer
+from pkg.suggestion.v1alpha3.hyperband import parsing_util
+from pkg.suggestion.v1alpha3.internal.base_health_service import HealthServicer
 
 logger = getLogger(__name__)
 FORMAT = '%(asctime)-15s Experiment %(experiment_name)s %(message)s'
@@ -43,7 +43,7 @@ def GetSuggestions(self, request, context):
             return reply
         except Exception as e:
             logger.error("Fail to generate trials: \n%s",
-                              traceback.format_exc(), extra={"experiment_name": experiment.name})
+                         traceback.format_exc(), extra={"experiment_name": experiment.name})
             raise e
 
     def _update_hbParameters(self, param):
@@ -72,25 +72,25 @@ def _make_bracket(self, experiment, param):
             param.evaluating_trials = 0
 
         logger.info("HyperBand Param eta %d.",
-                         param.eta, extra={"experiment_name": experiment.name})
+                    param.eta, extra={"experiment_name": experiment.name})
         logger.info("HyperBand Param R %d.",
-                         param.r_l, extra={"experiment_name": experiment.name})
+                    param.r_l, extra={"experiment_name": experiment.name})
         logger.info("HyperBand Param sMax %d.",
-                         param.s_max, extra={"experiment_name": experiment.name})
+                    param.s_max, extra={"experiment_name": experiment.name})
         logger.info("HyperBand Param B %d.",
-                         param.b_l, extra={"experiment_name": experiment.name})
+                    param.b_l, extra={"experiment_name": experiment.name})
         logger.info("HyperBand Param n %d.",
-                         param.n, extra={"experiment_name": experiment.name})
+                    param.n, extra={"experiment_name": experiment.name})
         logger.info("HyperBand Param r %d.",
-                         param.r, extra={"experiment_name": experiment.name})
+                    param.r, extra={"experiment_name": experiment.name})
         logger.info("HyperBand Param s %d.",
-                         param.current_s, extra={"experiment_name": experiment.name})
+                    param.current_s, extra={"experiment_name": experiment.name})
         logger.info("HyperBand Param i %d.",
-                         param.current_i, extra={"experiment_name": experiment.name})
+                    param.current_i, extra={"experiment_name": experiment.name})
         logger.info("HyperBand evaluating trials count %d.",
-                         param.evaluating_trials, extra={"experiment_name": experiment.name})
+                    param.evaluating_trials, extra={"experiment_name": experiment.name})
         logger.info("HyperBand budget resource name %s.",
-                         param.resource_name, extra={"experiment_name": experiment.name})
+                    param.resource_name, extra={"experiment_name": experiment.name})
         if param.evaluating_trials == 0:
             self._new_hbParameters(param)
 
@@ -107,7 +107,7 @@ def _make_child_bracket(self, experiment, param):
             last_trials, r_i, param.resource_name)
 
         logger.info("Generate %d trials by child bracket.",
-                         top_trials_num, extra={"experiment_name": experiment.name})
+                    top_trials_num, extra={"experiment_name": experiment.name})
         return trialSpecs
 
     def _get_last_trials(self, all_trials, latest_trials_num):
@@ -180,7 +180,7 @@ def _make_master_bracket(self, experiment, param):
                                                                  value=str(hp['value']))
             trial_specs.append(trial_spec)
         logger.info("Generate %d trials by master bracket.",
-                         n, extra={"experiment_name": experiment.name})
+                    n, extra={"experiment_name": experiment.name})
         return trial_specs
 
     def _set_validate_context_error(self, context, error_message):

pkg/suggestion/v1alpha3/hyperopt/base_hyperopt_service.py → pkg/suggestion/v1alpha3/hyperopt/base_service.py

@@ -2,7 +2,7 @@
 import numpy as np
 import logging
 
-from pkg.suggestion.v1alpha3.internal.constant import *
+from pkg.suggestion.v1alpha3.internal.constant import INTEGER, DOUBLE, CATEGORICAL, DISCRETE, MAX_GOAL
 from pkg.suggestion.v1alpha3.internal.trial import Assignment
 
 logger = logging.getLogger(__name__)
@@ -72,14 +72,13 @@ def getSuggestions(self, trials, request_number):
 
         recorded_trials_names = self.fmin.trials.specs
 
-
         hyperopt_trial_new_ids = []
         hyperopt_trial_specs = []
         hyperopt_trial_results = []
         hyperopt_trial_miscs = []
         # Update hyperopt FMin with new completed Trials
         for trial in trials:
-            if {"trial-name":trial.name} not in recorded_trials_names:
+            if {"trial-name": trial.name} not in recorded_trials_names:
                 # Produce new id for the new Trial
                 new_id = self.fmin.trials.new_trial_ids(1)
                 hyperopt_trial_new_ids.append(new_id[0])
@@ -107,7 +106,6 @@ def getSuggestions(self, trials, request_number):
                         hyperopt_trial_miscs_idxs[param.name] = new_id
                         hyperopt_trial_miscs_vals[param.name] = [index_of_value_in_list]
 
-
                 hyperopt_trial_misc["idxs"] = hyperopt_trial_miscs_idxs
                 hyperopt_trial_misc["vals"] = hyperopt_trial_miscs_vals
                 hyperopt_trial_miscs.append(hyperopt_trial_misc)

pkg/suggestion/v1alpha3/hyperopt_service.py → pkg/suggestion/v1alpha3/hyperopt/service.py

@@ -5,8 +5,8 @@
 
 from pkg.suggestion.v1alpha3.internal.search_space import HyperParameterSearchSpace
 from pkg.suggestion.v1alpha3.internal.trial import Trial, Assignment
-from pkg.suggestion.v1alpha3.hyperopt.base_hyperopt_service import BaseHyperoptService
-from pkg.suggestion.v1alpha3.base_health_service import HealthServicer
+from pkg.suggestion.v1alpha3.hyperopt.base_service import BaseHyperoptService
+from pkg.suggestion.v1alpha3.internal.base_health_service import HealthServicer
 
 logger = logging.getLogger(__name__)
 

pkg/suggestion/v1alpha3/internal/search_space.py

@@ -1,7 +1,7 @@
 import logging
 from pkg.apis.manager.v1alpha3.python import api_pb2 as api
 
-from .constant import *
+from pkg.suggestion.v1alpha3.internal.constant import *
 
 
 logging.basicConfig(level=logging.DEBUG)

pkg/suggestion/v1alpha3/nas/enas_service.py → pkg/suggestion/v1alpha3/nas/enas/service.py

@@ -11,7 +11,7 @@
 from pkg.suggestion.v1alpha3.nas.enas.Operation import SearchSpace
 from pkg.suggestion.v1alpha3.nas.enas.AlgorithmSettings import (
     parseAlgorithmSettings, algorithmSettingsValidator, enableNoneSettingsList)
-from pkg.suggestion.v1alpha3.base_health_service import HealthServicer
+from pkg.suggestion.v1alpha3.internal.base_health_service import HealthServicer
 
 
 class EnasExperiment:

pkg/suggestion/v1alpha3/skopt/base_skopt_service.py → pkg/suggestion/v1alpha3/skopt/base_service.py

@@ -2,7 +2,7 @@
 import logging
 import skopt
 
-from pkg.suggestion.v1alpha3.internal.constant import *
+from pkg.suggestion.v1alpha3.internal.constant import INTEGER, DOUBLE, CATEGORICAL, DISCRETE, MAX_GOAL
 from pkg.suggestion.v1alpha3.internal.trial import Assignment
 
 logger = logging.getLogger(__name__)

pkg/suggestion/v1alpha3/skopt_service.py → pkg/suggestion/v1alpha3/skopt/service.py

@@ -4,8 +4,8 @@
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.suggestion.v1alpha3.internal.search_space import HyperParameterSearchSpace
 from pkg.suggestion.v1alpha3.internal.trial import Trial, Assignment
-from pkg.suggestion.v1alpha3.skopt.base_skopt_service import BaseSkoptService
-from pkg.suggestion.v1alpha3.base_health_service import HealthServicer
+from pkg.suggestion.v1alpha3.skopt.base_service import BaseSkoptService
+from pkg.suggestion.v1alpha3.internal.base_health_service import HealthServicer
 
 
 logger = logging.getLogger(__name__)

test/suggestion/v1alpha3/test_chocolate_service.py.failed

@@ -5,7 +5,7 @@ import unittest
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.v1alpha3.python import api_pb2
 
-from pkg.suggestion.v1alpha3.chocolate_service import ChocolateService
+from pkg.suggestion.v1alpha3.chocolate.service import ChocolateService
 
 
 class TestHyperopt(unittest.TestCase):

test/suggestion/v1alpha3/test_enas_service.py

@@ -5,7 +5,7 @@
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.v1alpha3.python import api_pb2
 
-from pkg.suggestion.v1alpha3.nas.enas_service import EnasService
+from pkg.suggestion.v1alpha3.nas.enas.service import EnasService
 
 
 class TestEnas(unittest.TestCase):

test/suggestion/v1alpha3/test_hyperband_service.py

@@ -5,7 +5,7 @@
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.v1alpha3.python import api_pb2
 
-from pkg.suggestion.v1alpha3.hyperband_service import HyperbandService
+from pkg.suggestion.v1alpha3.hyperband.service import HyperbandService
 
 
 class TestHyperband(unittest.TestCase):

test/suggestion/v1alpha3/test_hyperopt_service.py

@@ -5,7 +5,7 @@
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.v1alpha3.python import api_pb2
 
-from pkg.suggestion.v1alpha3.hyperopt_service import HyperoptService
+from pkg.suggestion.v1alpha3.hyperopt.service import HyperoptService
 
 
 class TestHyperopt(unittest.TestCase):

test/suggestion/v1alpha3/test_skopt_service.py

@@ -5,7 +5,7 @@
 from pkg.apis.manager.v1alpha3.python import api_pb2_grpc
 from pkg.apis.manager.v1alpha3.python import api_pb2
 
-from pkg.suggestion.v1alpha3.skopt_service import SkoptService
+from pkg.suggestion.v1alpha3.skopt.service import SkoptService
 
 
 class TestSkopt(unittest.TestCase):