diff --git a/pkg/suggestion/v1beta1/optuna/service.py b/pkg/suggestion/v1beta1/optuna/service.py index 21fa330e691..7ebd8b154e0 100644 --- a/pkg/suggestion/v1beta1/optuna/service.py +++ b/pkg/suggestion/v1beta1/optuna/service.py @@ -134,19 +134,10 @@ def _tell(self, trials): optuna_trial_numbers = self.assignments_to_optuna_number[assignments_key] if len(optuna_trial_numbers) != 0: - # The trial has been suggested by the Optuna study. - # The objective value is reported using study.tell() with the corresponding trial number. trial_number = optuna_trial_numbers.pop(0) self.study.tell(trial_number, value) else: - # The trial has not been suggested by the Optuna study. - # A new trial object is created and reported using study.add_trial() with the assignments and the search space. - optuna_trial = optuna.create_trial( - params={a.name:self._get_casted_assignment_value(a) for a in trial.assignments}, - distributions=self._get_optuna_search_space(), - value=value, - ) - self.study.add_trial(optuna_trial) + raise ValueError("An unknown trial has been passed in the GetSuggestion request.") def _get_assignments_key(self, assignments): assignments = sorted(assignments, key=lambda a: a.name) diff --git a/test/suggestion/v1beta1/test_optuna_service.py b/test/suggestion/v1beta1/test_optuna_service.py index c1295db534a..17f061c5cf5 100644 --- a/test/suggestion/v1beta1/test_optuna_service.py +++ b/test/suggestion/v1beta1/test_optuna_service.py @@ -14,7 +14,6 @@ import grpc import grpc_testing -import unittest import pytest @@ -43,6 +42,72 @@ def setup_method(self): ], ) def test_get_suggestion(self, algorithm_name, algorithm_settings): + experiment = api_pb2.Experiment( + name="test", + spec=api_pb2.ExperimentSpec( + algorithm=api_pb2.AlgorithmSpec( + algorithm_name=algorithm_name, + algorithm_settings = [ + api_pb2.AlgorithmSetting( + name=name, + value=value + ) for name, value in algorithm_settings.items() + ], + ), + objective=api_pb2.ObjectiveSpec( + type=api_pb2.MAXIMIZE, + goal=0.9 + ), + parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs( + parameters=[ + api_pb2.ParameterSpec( + name="param-1", + parameter_type=api_pb2.INT, + feasible_space=api_pb2.FeasibleSpace( + max="5", min="1", list=[]), + ), + api_pb2.ParameterSpec( + name="param-2", + parameter_type=api_pb2.CATEGORICAL, + feasible_space=api_pb2.FeasibleSpace( + max=None, min=None, list=["cat1", "cat2", "cat3"]) + ), + api_pb2.ParameterSpec( + name="param-3", + parameter_type=api_pb2.DISCRETE, + feasible_space=api_pb2.FeasibleSpace( + max=None, min=None, list=["3", "2", "6"]) + ), + api_pb2.ParameterSpec( + name="param-4", + parameter_type=api_pb2.DOUBLE, + feasible_space=api_pb2.FeasibleSpace( + max="5", min="1", list=[]) + ) + ] + ) + ) + ) + + # Run the first suggestion with no previous trials in the request + request = api_pb2.GetSuggestionsRequest( + experiment=experiment, + trials=[], + request_number=2, + ) + + get_suggestion = self.test_server.invoke_unary_unary( + method_descriptor=(api_pb2.DESCRIPTOR + .services_by_name['Suggestion'] + .methods_by_name['GetSuggestions']), + invocation_metadata={}, + request=request, timeout=1) + + response, metadata, code, details = get_suggestion.termination() + assert code == grpc.StatusCode.OK + assert 2 == len(response.parameter_assignments) + + # Run the second suggestion with trials whose parameters are assigned in the first request trials = [ api_pb2.Trial( name="test-asfjh", @@ -53,24 +118,7 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings): goal=0.9 ), parameter_assignments=api_pb2.TrialSpec.ParameterAssignments( - assignments=[ - api_pb2.ParameterAssignment( - name="param-1", - value="2", - ), - api_pb2.ParameterAssignment( - name="param-2", - value="cat1", - ), - api_pb2.ParameterAssignment( - name="param-3", - value="2", - ), - api_pb2.ParameterAssignment( - name="param-4", - value="3.44", - ) - ] + assignments=response.parameter_assignments[0].assignments ) ), status=api_pb2.TrialStatus( @@ -98,24 +146,7 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings): goal=0.9 ), parameter_assignments=api_pb2.TrialSpec.ParameterAssignments( - assignments=[ - api_pb2.ParameterAssignment( - name="param-1", - value="3", - ), - api_pb2.ParameterAssignment( - name="param-2", - value="cat2", - ), - api_pb2.ParameterAssignment( - name="param-3", - value="6", - ), - api_pb2.ParameterAssignment( - name="param-4", - value="4.44", - ) - ] + assignments=response.parameter_assignments[1].assignments ) ), status=api_pb2.TrialStatus( @@ -135,53 +166,6 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings): ) ) ] - experiment = api_pb2.Experiment( - name="test", - spec=api_pb2.ExperimentSpec( - - algorithm=api_pb2.AlgorithmSpec( - algorithm_name=algorithm_name, - algorithm_settings = [ - api_pb2.AlgorithmSetting( - name=name, - value=value - ) for name, value in algorithm_settings.items() - ], - ), - objective=api_pb2.ObjectiveSpec( - type=api_pb2.MAXIMIZE, - goal=0.9 - ), - parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs( - parameters=[ - api_pb2.ParameterSpec( - name="param-1", - parameter_type=api_pb2.INT, - feasible_space=api_pb2.FeasibleSpace( - max="5", min="1", list=[]), - ), - api_pb2.ParameterSpec( - name="param-2", - parameter_type=api_pb2.CATEGORICAL, - feasible_space=api_pb2.FeasibleSpace( - max=None, min=None, list=["cat1", "cat2", "cat3"]) - ), - api_pb2.ParameterSpec( - name="param-3", - parameter_type=api_pb2.DISCRETE, - feasible_space=api_pb2.FeasibleSpace( - max=None, min=None, list=["3", "2", "6"]) - ), - api_pb2.ParameterSpec( - name="param-4", - parameter_type=api_pb2.DOUBLE, - feasible_space=api_pb2.FeasibleSpace( - max="5", min="1", list=[]) - ) - ] - ) - ) - ) request = api_pb2.GetSuggestionsRequest( experiment=experiment, @@ -202,4 +186,4 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings): if __name__ == '__main__': - unittest.main() + pytest.main()