Skip to content

Commit

Permalink
Fix the logic of handling unknown trials
Browse files Browse the repository at this point in the history
  • Loading branch information
@g-votte
g-votte committed Aug 14, 2021
1 parent a119fb2 commit 1c0e15f
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 95 deletions.
11 changes: 1 addition & 10 deletions pkg/suggestion/v1beta1/optuna/service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,19 +134,10 @@ def _tell(self, trials):
optuna_trial_numbers = self.assignments_to_optuna_number[assignments_key]

if len(optuna_trial_numbers) != 0:
# The trial has been suggested by the Optuna study.
# The objective value is reported using study.tell() with the corresponding trial number.
trial_number = optuna_trial_numbers.pop(0)
self.study.tell(trial_number, value)
else:
# The trial has not been suggested by the Optuna study.
# A new trial object is created and reported using study.add_trial() with the assignments and the search space.
optuna_trial = optuna.create_trial(
params={a.name:self._get_casted_assignment_value(a) for a in trial.assignments},
distributions=self._get_optuna_search_space(),
value=value,
)
self.study.add_trial(optuna_trial)
raise ValueError("An unknown trial has been passed in the GetSuggestion request.")

def _get_assignments_key(self, assignments):
assignments = sorted(assignments, key=lambda a: a.name)
Expand Down
154 changes: 69 additions & 85 deletions test/suggestion/v1beta1/test_optuna_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

import grpc
import grpc_testing
import unittest

import pytest

Expand Down Expand Up @@ -43,6 +42,72 @@ def setup_method(self):
],
)
def test_get_suggestion(self, algorithm_name, algorithm_settings):
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name=algorithm_name,
algorithm_settings = [
api_pb2.AlgorithmSetting(
name=name,
value=value
) for name, value in algorithm_settings.items()
],
),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2", "cat3"])
),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[])
)
]
)
)
)

# Run the first suggestion with no previous trials in the request
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=[],
request_number=2,
)

get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request, timeout=1)

response, metadata, code, details = get_suggestion.termination()
assert code == grpc.StatusCode.OK
assert 2 == len(response.parameter_assignments)

# Run the second suggestion with trials whose parameters are assigned in the first request
trials = [
api_pb2.Trial(
name="test-asfjh",
Expand All @@ -53,24 +118,7 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings):
goal=0.9
),
parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="2",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat1",
),
api_pb2.ParameterAssignment(
name="param-3",
value="2",
),
api_pb2.ParameterAssignment(
name="param-4",
value="3.44",
)
]
assignments=response.parameter_assignments[0].assignments
)
),
status=api_pb2.TrialStatus(
Expand Down Expand Up @@ -98,24 +146,7 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings):
goal=0.9
),
parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="3",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat2",
),
api_pb2.ParameterAssignment(
name="param-3",
value="6",
),
api_pb2.ParameterAssignment(
name="param-4",
value="4.44",
)
]
assignments=response.parameter_assignments[1].assignments
)
),
status=api_pb2.TrialStatus(
Expand All @@ -135,53 +166,6 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings):
)
)
]
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(

algorithm=api_pb2.AlgorithmSpec(
algorithm_name=algorithm_name,
algorithm_settings = [
api_pb2.AlgorithmSetting(
name=name,
value=value
) for name, value in algorithm_settings.items()
],
),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2", "cat3"])
),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[])
)
]
)
)
)

request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
Expand All @@ -202,4 +186,4 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings):


if __name__ == '__main__':
unittest.main()
pytest.main()

0 comments on commit 1c0e15f

Please sign in to comment.