From 7d0f0268a6da21a818966548632d37abe445281a Mon Sep 17 00:00:00 2001 From: zhangyuge Date: Wed, 15 Jan 2020 12:16:31 +0800 Subject: [PATCH 1/2] strengthen UT and fix hyperopt randint --- .../nni/gridsearch_tuner/gridsearch_tuner.py | 3 + .../nni/hyperopt_tuner/hyperopt_tuner.py | 4 +- src/sdk/pynni/tests/assets/search_space.json | 6 +- src/sdk/pynni/tests/test_builtin_tuners.py | 55 +++++++++++++------ 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py index f9dee6cdd8..8a9ab0a4ed 100644 --- a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py +++ b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py @@ -102,6 +102,9 @@ def _parse_randint(self, param_value): """ Parse type of randint parameter and return a list """ + if param_value[0] >= param_value[1]: + raise ValueError("Randint should contain at least 1 candidate, but [%s, %s) contains none.", + param_value[0], param_value[1]) return np.arange(param_value[0], param_value[1]).tolist() def _expand_parameters(self, para): diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py index 466c29c484..02de79ffb3 100644 --- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py +++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py @@ -118,6 +118,8 @@ def json2vals(in_x, vals, out_y, name=NodeType.ROOT): vals[NodeType.VALUE], out_y, name=name + '[%d]' % _index) + if _type == 'randint': + out_y[name] -= in_x[NodeType.VALUE][0] else: for key in in_x.keys(): json2vals(in_x[key], vals[key], out_y, @@ -258,7 +260,7 @@ def generate_parameters(self, parameter_id, **kwargs): Parameters ---------- - parameter_id : int + parameter_id : list of int Returns ------- diff --git a/src/sdk/pynni/tests/assets/search_space.json b/src/sdk/pynni/tests/assets/search_space.json index 0e7c7ba9cc..21b6f90996 100644 --- a/src/sdk/pynni/tests/assets/search_space.json +++ b/src/sdk/pynni/tests/assets/search_space.json @@ -1,8 +1,7 @@ { "choice_str": { "_type": "choice", - "_value": ["cat", "dog", "elephant", "cow", "sheep", "panda"], - "fail": ["metis", "gp"] + "_value": ["cat", "dog", "elephant", "cow", "sheep", "panda"] }, "choice_int": { "_type": "choice", @@ -10,8 +9,7 @@ }, "choice_mixed": { "_type": "choice", - "_value": [0.3, "cat", 1, null], - "fail": ["metis", "gp"] + "_value": [0.3, "cat", 1, null] }, "choice_float": { "_type": "choice", diff --git a/src/sdk/pynni/tests/test_builtin_tuners.py b/src/sdk/pynni/tests/test_builtin_tuners.py index 8dd3731983..2fe2c8345c 100644 --- a/src/sdk/pynni/tests/test_builtin_tuners.py +++ b/src/sdk/pynni/tests/test_builtin_tuners.py @@ -5,6 +5,7 @@ import json import logging import os +import random import shutil import sys from unittest import TestCase, main @@ -15,6 +16,7 @@ from nni.gridsearch_tuner.gridsearch_tuner import GridSearchTuner from nni.hyperopt_tuner.hyperopt_tuner import HyperoptTuner from nni.metis_tuner.metis_tuner import MetisTuner + try: from nni.smac_tuner.smac_tuner import SMACTuner except ImportError: @@ -34,20 +36,28 @@ class BuiltinTunersTestCase(TestCase): - [X] generate_multiple_parameters - [ ] import_data - [ ] trial_end - - [ ] receive_trial_result + - [x] receive_trial_result """ + def setUp(self): + self.test_round = 3 + self.params_each_round = 50 + self.exhaustive = False + def search_space_test_one(self, tuner_factory, search_space): tuner = tuner_factory() self.assertIsInstance(tuner, Tuner) tuner.update_search_space(search_space) - parameters = tuner.generate_multiple_parameters(list(range(0, 50))) - logger.info(parameters) - self.check_range(parameters, search_space) - if not parameters: # TODO: not strict - raise ValueError("No parameters generated") - return parameters + for i in range(self.test_round): + parameters = tuner.generate_multiple_parameters(list(range(i * self.params_each_round, + (i + 1) * self.params_each_round))) + logger.debug(parameters) + self.check_range(parameters, search_space) + for k in range(min(len(parameters), self.params_each_round)): + tuner.receive_trial_result(self.params_each_round * i + k, parameters[k], random.uniform(-100, 100)) + if not parameters and not self.exhaustive: + raise ValueError("No parameters generated") def check_range(self, generated_params, search_space): EPS = 1E-6 @@ -91,7 +101,8 @@ def check_range(self, generated_params, search_space): for layer_name in item["_value"].keys(): self.assertIn(v[layer_name]["chosen_layer"], item["layer_choice"]) - def search_space_test_all(self, tuner_factory, supported_types=None, ignore_types=None): + def search_space_test_all(self, tuner_factory, supported_types=None, ignore_types=None, fail_types=None): + # Three types: 1. supported; 2. ignore; 3. fail. # NOTE(yuge): ignore types # Supported types are listed in the table. They are meant to be supported and should be correct. # Other than those, all the rest are "unsupported", which are expected to produce ridiculous results @@ -103,16 +114,18 @@ def search_space_test_all(self, tuner_factory, supported_types=None, ignore_type if supported_types is None: supported_types = ["choice", "randint", "uniform", "quniform", "loguniform", "qloguniform", "normal", "qnormal", "lognormal", "qlognormal"] + if fail_types is None: + fail_types = [] + if ignore_types is None: + ignore_types = [] full_supported_search_space = dict() for single in search_space_all: - single_keyword = single.split("_") space = search_space_all[single] - expected_fail = not any([t in single_keyword for t in supported_types]) or "fail" in single_keyword - if ignore_types is not None and any([t in ignore_types for t in single_keyword]): + if any(single.startswith(t) for t in ignore_types): continue - if "fail" in space: - if self._testMethodName.split("_", 1)[1] in space.pop("fail"): - expected_fail = True + expected_fail = not any(single.startswith(t) for t in supported_types) or \ + any(single.startswith(t) for t in fail_types) or \ + "fail" in single # name contains fail (fail on all) single_search_space = {single: space} if not expected_fail: # supports this key @@ -129,11 +142,14 @@ def search_space_test_all(self, tuner_factory, supported_types=None, ignore_type self.search_space_test_one(tuner_factory, full_supported_search_space) def test_grid_search(self): + self.exhaustive = True self.search_space_test_all(lambda: GridSearchTuner(), supported_types=["choice", "randint", "quniform"]) def test_tpe(self): - self.search_space_test_all(lambda: HyperoptTuner("tpe")) + self.search_space_test_all(lambda: HyperoptTuner("tpe"), + ignore_types=["uniform_equal", "qloguniform_equal", "loguniform_equal", "quniform_clip_2"]) + # NOTE: types are ignored because `tpe.py line 465, in adaptive_parzen_normal assert prior_sigma > 0` def test_random_search(self): self.search_space_test_all(lambda: HyperoptTuner("random_search")) @@ -148,6 +164,7 @@ def test_smac(self): supported_types=["choice", "randint", "uniform", "quniform", "loguniform"]) def test_batch(self): + self.exhaustive = True self.search_space_test_all(lambda: BatchTuner(), supported_types=["choice"]) @@ -156,14 +173,18 @@ def test_evolution(self): self.search_space_test_all(lambda: EvolutionTuner(population_size=100)) def test_gp(self): + self.test_round = 1 # NOTE: GP tuner got hanged for multiple testing round self.search_space_test_all(lambda: GPTuner(), supported_types=["choice", "randint", "uniform", "quniform", "loguniform", "qloguniform"], - ignore_types=["normal", "lognormal", "qnormal", "qlognormal"]) + ignore_types=["normal", "lognormal", "qnormal", "qlognormal"], + fail_types=["choice_str", "choice_mixed"]) def test_metis(self): + self.test_round = 1 # NOTE: Metis tuner got hanged for multiple testing round self.search_space_test_all(lambda: MetisTuner(), - supported_types=["choice", "randint", "uniform", "quniform"]) + supported_types=["choice", "randint", "uniform", "quniform"], + fail_types=["choice_str", "choice_mixed"]) def test_networkmorphism(self): pass From 01d5d4a4353b6b64a83fd9b699e75db49001c47a Mon Sep 17 00:00:00 2001 From: zhangyuge Date: Wed, 15 Jan 2020 12:39:48 +0800 Subject: [PATCH 2/2] fix a ridiculous mistake --- src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py index 02de79ffb3..c7e168191f 100644 --- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py +++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py @@ -260,7 +260,7 @@ def generate_parameters(self, parameter_id, **kwargs): Parameters ---------- - parameter_id : list of int + parameter_id : int Returns -------