From a2e45f23d39433c9d3f712cbaf65aa18c5b6d714 Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Sat, 27 Jul 2019 14:56:11 +0800 Subject: [PATCH 1/6] fix quniform issue ; change related doc --- docs/en_US/Tutorial/SearchSpaceSpec.md | 3 ++- src/sdk/pynni/nni/parameter_expressions.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index f5a1d34d4f..a822bccfb8 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -45,7 +45,8 @@ All types of sampling strategies and their parameter are listed here: * When optimizing, this variable is constrained to a two-sided interval. * {"_type":"quniform","_value":[low, high, q]} - * Which means the variable value is a value like round(uniform(low, high) / q) * q + * Which means the variable value is a value like randint(np.floor((high-low)/q)+1)*q + low. For example, for _value specified as [0, 10, 2.5], possible values are [0, 2.5, 5.0, 7.5, 10.0]; For _value specified as [2, 10, 5], possible values are [2, 7]. All possible values have the same possibility to be selected. + * Suitable for a discrete value with respect to which the objective is still somewhat "smooth", but which should be bounded both above and below. If you want to uniformly choose integer from a range [low, high], you can write `_value` like this: `[low, high, 1]`. * {"_type":"loguniform","_value":[low, high]} diff --git a/src/sdk/pynni/nni/parameter_expressions.py b/src/sdk/pynni/nni/parameter_expressions.py index e0629d8e60..2e2b34cfeb 100644 --- a/src/sdk/pynni/nni/parameter_expressions.py +++ b/src/sdk/pynni/nni/parameter_expressions.py @@ -57,7 +57,7 @@ def quniform(low, high, q, random_state): q: sample step random_state: an object of numpy.random.RandomState ''' - return np.round(uniform(low, high, random_state) / q) * q + return randint(np.floor((high-low)/q)+1, random_state) * q + low def loguniform(low, high, random_state): From 9a00a14c70ad7df67e793cfcee6c99df47f77410 Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Sun, 28 Jul 2019 10:51:30 +0800 Subject: [PATCH 2/6] fix qloguniform issue --- docs/en_US/Tutorial/SearchSpaceSpec.md | 4 ++-- src/sdk/pynni/nni/parameter_expressions.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index a822bccfb8..cefee8b801 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -45,7 +45,7 @@ All types of sampling strategies and their parameter are listed here: * When optimizing, this variable is constrained to a two-sided interval. * {"_type":"quniform","_value":[low, high, q]} - * Which means the variable value is a value like randint(np.floor((high-low)/q)+1)*q + low. For example, for _value specified as [0, 10, 2.5], possible values are [0, 2.5, 5.0, 7.5, 10.0]; For _value specified as [2, 10, 5], possible values are [2, 7]. All possible values have the same possibility to be selected. + * Which means the variable value is a value like randint(floor((high-low)/q)+1)*q + low. For example, for _value specified as [0, 10, 2.5], possible values are [0, 2.5, 5.0, 7.5, 10.0]; For _value specified as [2, 10, 5], possible values are [2, 7]. All possible values have the same possibility to be selected. * Suitable for a discrete value with respect to which the objective is still somewhat "smooth", but which should be bounded both above and below. If you want to uniformly choose integer from a range [low, high], you can write `_value` like this: `[low, high, 1]`. @@ -54,7 +54,7 @@ All types of sampling strategies and their parameter are listed here: * When optimizing, this variable is constrained to be positive. * {"_type":"qloguniform","_value":[low, high, q]} - * Which means the variable value is a value like round(loguniform(low, high)) / q) * q + * Which means the variable value is a value like floor((loguniform(low, high) - low) / q) * q + low * Suitable for a discrete variable with respect to which the objective is "smooth" and gets smoother with the size of the value, but which should be bounded both above and below. * {"_type":"normal","_value":[mu, sigma]} diff --git a/src/sdk/pynni/nni/parameter_expressions.py b/src/sdk/pynni/nni/parameter_expressions.py index 2e2b34cfeb..384cac91b9 100644 --- a/src/sdk/pynni/nni/parameter_expressions.py +++ b/src/sdk/pynni/nni/parameter_expressions.py @@ -77,7 +77,7 @@ def qloguniform(low, high, q, random_state): q: sample step random_state: an object of numpy.random.RandomState ''' - return np.round(loguniform(low, high, random_state) / q) * q + return np.floor((loguniform(low, high, random_state) - low) / q) * q + low def normal(mu, sigma, random_state): From 591fa1d7993851735f396969d3bc68cfb3eb4a2a Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Mon, 29 Jul 2019 13:31:38 +0800 Subject: [PATCH 3/6] update annotation doc; update smartparam sync --- docs/en_US/Tutorial/AnnotationSpec.md | 4 ++-- src/sdk/pynni/nni/parameter_expressions.py | 1 + src/sdk/pynni/nni/smartparam.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/en_US/Tutorial/AnnotationSpec.md b/docs/en_US/Tutorial/AnnotationSpec.md index a6bdf887f6..32861e3b8b 100644 --- a/docs/en_US/Tutorial/AnnotationSpec.md +++ b/docs/en_US/Tutorial/AnnotationSpec.md @@ -41,11 +41,11 @@ There are 10 types to express your search space as follows: * `@nni.variable(nni.uniform(low, high),name=variable)` Which means the variable value is a value uniformly between low and high. * `@nni.variable(nni.quniform(low, high, q),name=variable)` - Which means the variable value is a value like round(uniform(low, high) / q) * q + Which means the variable value is a value like randint(floor((high-low)/q)+1)*q + low * `@nni.variable(nni.loguniform(low, high),name=variable)` Which means the variable value is a value drawn according to exp(uniform(low, high)) so that the logarithm of the return value is uniformly distributed. * `@nni.variable(nni.qloguniform(low, high, q),name=variable)` - Which means the variable value is a value like round(exp(uniform(low, high)) / q) * q + Which means the variable value is a value like floor((loguniform(low, high) - low) / q) * q + low * `@nni.variable(nni.normal(mu, sigma),name=variable)` Which means the variable value is a real value that's normally-distributed with mean mu and standard deviation sigma. * `@nni.variable(nni.qnormal(mu, sigma, q),name=variable)` diff --git a/src/sdk/pynni/nni/parameter_expressions.py b/src/sdk/pynni/nni/parameter_expressions.py index 384cac91b9..cb13a9c9ef 100644 --- a/src/sdk/pynni/nni/parameter_expressions.py +++ b/src/sdk/pynni/nni/parameter_expressions.py @@ -57,6 +57,7 @@ def quniform(low, high, q, random_state): q: sample step random_state: an object of numpy.random.RandomState ''' + assert high > low, 'Upper bound must be larger than lower bound' return randint(np.floor((high-low)/q)+1, random_state) * q + low diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index b6d87d04e6..dce1f3c931 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -57,14 +57,14 @@ def uniform(low, high, name=None): def quniform(low, high, q, name=None): assert high > low, 'Upper bound must be larger than lower bound' - return round(random.uniform(low, high) / q) * q + return randint(np.floor((high-low)/q)+1) * q + low def loguniform(low, high, name=None): assert low > 0, 'Lower bound must be positive' return np.exp(random.uniform(np.log(low), np.log(high))) def qloguniform(low, high, q, name=None): - return round(loguniform(low, high) / q) * q + return np.floor((loguniform(low, high) - low) / q) * q + low def normal(mu, sigma, name=None): return random.gauss(mu, sigma) From 4feb466c339601e6e775256730eb095593c17b92 Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Sun, 4 Aug 2019 23:06:46 +0800 Subject: [PATCH 4/6] fix hyperopt quniform/qloguniform issue --- src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py index a1f3bc37d5..ab562b383a 100644 --- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py +++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py @@ -91,7 +91,10 @@ def json2parameter(in_x, parameter, name=NodeType.ROOT): name=name + '[%d]' % _index) } else: - out_y = parameter[name] + if _type in ['quniform', 'qloguniform']: + out_y = np.clip(parameter[name], in_x[NodeType.VALUE][0], in_x[NodeType.VALUE][1]) + else: + out_y = parameter[name] else: out_y = dict() for key in in_x.keys(): From e3c8552f3cad8b5ab0128a6eb27e3190506fe9d8 Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Mon, 5 Aug 2019 09:35:06 +0800 Subject: [PATCH 5/6] change qtype format to conforme with BOHB --- docs/en_US/Tutorial/AnnotationSpec.md | 4 ++-- docs/en_US/Tutorial/SearchSpaceSpec.md | 4 ++-- src/sdk/pynni/nni/parameter_expressions.py | 5 ++--- src/sdk/pynni/nni/smartparam.py | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/en_US/Tutorial/AnnotationSpec.md b/docs/en_US/Tutorial/AnnotationSpec.md index 32861e3b8b..d8d73605ea 100644 --- a/docs/en_US/Tutorial/AnnotationSpec.md +++ b/docs/en_US/Tutorial/AnnotationSpec.md @@ -41,11 +41,11 @@ There are 10 types to express your search space as follows: * `@nni.variable(nni.uniform(low, high),name=variable)` Which means the variable value is a value uniformly between low and high. * `@nni.variable(nni.quniform(low, high, q),name=variable)` - Which means the variable value is a value like randint(floor((high-low)/q)+1)*q + low + Which means the variable value is a value like clip(round(uniform(low, high) / q) * q, low, high), where the clip operation is used to constraint the generated value in the bound. * `@nni.variable(nni.loguniform(low, high),name=variable)` Which means the variable value is a value drawn according to exp(uniform(low, high)) so that the logarithm of the return value is uniformly distributed. * `@nni.variable(nni.qloguniform(low, high, q),name=variable)` - Which means the variable value is a value like floor((loguniform(low, high) - low) / q) * q + low + Which means the variable value is a value like clip(round(loguniform(low, high) / q) * q, low, high), where the clip operation is used to constraint the generated value in the bound. * `@nni.variable(nni.normal(mu, sigma),name=variable)` Which means the variable value is a real value that's normally-distributed with mean mu and standard deviation sigma. * `@nni.variable(nni.qnormal(mu, sigma, q),name=variable)` diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index cefee8b801..6746db95bd 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -45,7 +45,7 @@ All types of sampling strategies and their parameter are listed here: * When optimizing, this variable is constrained to a two-sided interval. * {"_type":"quniform","_value":[low, high, q]} - * Which means the variable value is a value like randint(floor((high-low)/q)+1)*q + low. For example, for _value specified as [0, 10, 2.5], possible values are [0, 2.5, 5.0, 7.5, 10.0]; For _value specified as [2, 10, 5], possible values are [2, 7]. All possible values have the same possibility to be selected. + * Which means the variable value is a value like clip(round(uniform(low, high) / q) * q, low, high), where the clip operation is used to constraint the generated value in the bound. For example, for _value specified as [0, 10, 2.5], possible values are [0, 2.5, 5.0, 7.5, 10.0]; For _value specified as [2, 10, 5], possible values are [2, 5, 10]. * Suitable for a discrete value with respect to which the objective is still somewhat "smooth", but which should be bounded both above and below. If you want to uniformly choose integer from a range [low, high], you can write `_value` like this: `[low, high, 1]`. @@ -54,7 +54,7 @@ All types of sampling strategies and their parameter are listed here: * When optimizing, this variable is constrained to be positive. * {"_type":"qloguniform","_value":[low, high, q]} - * Which means the variable value is a value like floor((loguniform(low, high) - low) / q) * q + low + * Which means the variable value is a value like clip(round(loguniform(low, high)) / q) * q, low, high), where the clip operation is used to constraint the generated value in the bound. * Suitable for a discrete variable with respect to which the objective is "smooth" and gets smoother with the size of the value, but which should be bounded both above and below. * {"_type":"normal","_value":[mu, sigma]} diff --git a/src/sdk/pynni/nni/parameter_expressions.py b/src/sdk/pynni/nni/parameter_expressions.py index cb13a9c9ef..8aef0e9062 100644 --- a/src/sdk/pynni/nni/parameter_expressions.py +++ b/src/sdk/pynni/nni/parameter_expressions.py @@ -57,8 +57,7 @@ def quniform(low, high, q, random_state): q: sample step random_state: an object of numpy.random.RandomState ''' - assert high > low, 'Upper bound must be larger than lower bound' - return randint(np.floor((high-low)/q)+1, random_state) * q + low + return np.clip(np.round(uniform(low, high, random_state) / q) * q, low, high) def loguniform(low, high, random_state): @@ -78,7 +77,7 @@ def qloguniform(low, high, q, random_state): q: sample step random_state: an object of numpy.random.RandomState ''' - return np.floor((loguniform(low, high, random_state) - low) / q) * q + low + return np.clip(np.round(loguniform(low, high, random_state) / q) * q, low, high) def normal(mu, sigma, random_state): diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index dce1f3c931..bad771f1ae 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -57,14 +57,14 @@ def uniform(low, high, name=None): def quniform(low, high, q, name=None): assert high > low, 'Upper bound must be larger than lower bound' - return randint(np.floor((high-low)/q)+1) * q + low + return np.clip(round(random.uniform(low, high) / q) * q, low, high) def loguniform(low, high, name=None): assert low > 0, 'Lower bound must be positive' return np.exp(random.uniform(np.log(low), np.log(high))) def qloguniform(low, high, q, name=None): - return np.floor((loguniform(low, high) - low) / q) * q + low + return np.clip(round(loguniform(low, high) / q) * q, low, high) def normal(mu, sigma, name=None): return random.gauss(mu, sigma) From afb4e78c5c9c4782482777fd8587c636711ab2e5 Mon Sep 17 00:00:00 2001 From: suiguoxin Date: Mon, 5 Aug 2019 09:57:18 +0800 Subject: [PATCH 6/6] update gridsearch quniform to conform with BOHB --- src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py index f6af325c56..66e74e6a45 100644 --- a/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py +++ b/src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py @@ -97,9 +97,8 @@ def json2parameter(self, ss_spec): def _parse_quniform(self, param_value): '''parse type of quniform parameter and return a list''' - low, high, interval = param_value[0], param_value[1], param_value[2] - count = int(np.floor((high - low) / interval)) + 1 - return [low + interval * i for i in range(count)] + low, high, q = param_value[0], param_value[1], param_value[2] + return np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high) def _parse_randint(self, param_value): '''parse type of randint parameter and return a list'''