From 21efd64ee755cfb649305a809c3e6af9dc299cf8 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Thu, 13 Jun 2019 09:37:01 +0800 Subject: [PATCH 01/18] update --- src/sdk/pynni/nni/smartparam.py | 49 ++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index 175df204cd..0c6b94ee6b 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -124,7 +124,8 @@ def mutable_layer( funcs_args, fixed_inputs, optional_inputs, - optional_input_size): + optional_input_size, + tf=None): '''execute the chosen function and inputs. Below is an example of chosen function and inputs: { @@ -144,13 +145,47 @@ def mutable_layer( fixed_inputs: optional_inputs: dict of optional inputs optional_input_size: number of candidate inputs to be chosen + tf: tensorflow module ''' - mutable_block = _get_param(mutable_id) - chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] - chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] - real_chosen_inputs = [optional_inputs[input_name] for input_name in chosen_inputs] - layer_out = funcs[chosen_layer]([fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) - + if tf is None: + mutable_block = _get_param(mutable_id) + chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] + chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] + real_chosen_inputs = [optional_inputs[input_name] for input_name in chosen_inputs] + layer_out = funcs[chosen_layer]([fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) + else: + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # store namespace + global name_space + name_space[name_prefix] = dict() + name_space[name_prefix]['funcs'] = list(funcs) + name_space[name_prefix]['optional_inputs'] = list(optional_inputs) + # create tensorflow variables as signals of selections + name_for_optional_inputs = name_prefix + '_optional_inputs' + name_for_funcs = name_prefix + '_funcs' + + global tf_variables + tf_variables[name_prefix] = dict() + tf_variables[name_prefix]['optional_inputs'] = tf.get_variable(name_for_optional_inputs, + [len(optional_inputs)], + dtype=tf.bool, + trainable=False) + tf_variables[name_prefix]['funcs'] = tf.get_variable(name_for_funcs, [], dtype=tf.int64, trainable=False) + + # get real values using their variable names + real_optional_inputs_value = [optional_inputs[name] for name in name_space[name_prefix]['optional_inputs']] + real_func_value = [funcs[name] for name in name_space[name_prefix]['funcs']] + real_funcs_args = [funcs_args[name] for name in name_space[name_prefix]['funcs']] + # build tensorflow graph of geting chosen inputs by masking + real_chosen_inputs = tf.boolean_mask(real_optional_inputs_value, tf_variables[name_prefix]['optional_inputs']) + # build tensorflow graph of different branches by using tf.case + branches = dict() + for func_id in range(len(funcs)): + func_output = real_func_value[func_id]( + [fixed_inputs, real_chosen_inputs], *real_funcs_args[func_id]) + branches[tf.equal(tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output + layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) + return layer_out def _get_param(key): From 1318d51c2e836951232f737ae5d0faa4bc5251aa Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Tue, 18 Jun 2019 13:50:01 +0800 Subject: [PATCH 02/18] dev-tf-master --- src/sdk/pynni/nni/smartparam.py | 60 ++++++++++++++++++-------- tools/nni_annotation/code_generator.py | 17 ++++++-- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index 0c6b94ee6b..9c81f61434 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -37,7 +37,8 @@ 'lognormal', 'qlognormal', 'function_choice', - 'mutable_layer' + 'mutable_layer', + 'reload_tensorflow_variables' ] @@ -82,6 +83,10 @@ def function_choice(*funcs, name=None): def mutable_layer(): raise RuntimeError('Cannot call nni.mutable_layer in this mode') + def reload_tensorflow_variables(session, tf): + raise RuntimeError( + 'Cannot call nni.reload_tensorflow_variables in this mode') + else: def choice(options, name=None, key=None): @@ -151,8 +156,10 @@ def mutable_layer( mutable_block = _get_param(mutable_id) chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] - real_chosen_inputs = [optional_inputs[input_name] for input_name in chosen_inputs] - layer_out = funcs[chosen_layer]([fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) + real_chosen_inputs = [optional_inputs[input_name] + for input_name in chosen_inputs] + layer_out = funcs[chosen_layer]( + [fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) else: name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) # store namespace @@ -160,34 +167,53 @@ def mutable_layer( name_space[name_prefix] = dict() name_space[name_prefix]['funcs'] = list(funcs) name_space[name_prefix]['optional_inputs'] = list(optional_inputs) - # create tensorflow variables as signals of selections + # create tensorflow variables as 1/0 signals used to form subgraph + global tf_variables name_for_optional_inputs = name_prefix + '_optional_inputs' name_for_funcs = name_prefix + '_funcs' - - global tf_variables tf_variables[name_prefix] = dict() tf_variables[name_prefix]['optional_inputs'] = tf.get_variable(name_for_optional_inputs, - [len(optional_inputs)], - dtype=tf.bool, - trainable=False) - tf_variables[name_prefix]['funcs'] = tf.get_variable(name_for_funcs, [], dtype=tf.int64, trainable=False) + [len(optional_inputs)], + dtype=tf.bool, + trainable=False) + tf_variables[name_prefix]['funcs'] = tf.get_variable( + name_for_funcs, [], dtype=tf.int64, trainable=False) # get real values using their variable names - real_optional_inputs_value = [optional_inputs[name] for name in name_space[name_prefix]['optional_inputs']] - real_func_value = [funcs[name] for name in name_space[name_prefix]['funcs']] - real_funcs_args = [funcs_args[name] for name in name_space[name_prefix]['funcs']] + real_optional_inputs_value = [optional_inputs[name] + for name in name_space[name_prefix]['optional_inputs']] + real_func_value = [funcs[name] + for name in name_space[name_prefix]['funcs']] + real_funcs_args = [funcs_args[name] + for name in name_space[name_prefix]['funcs']] # build tensorflow graph of geting chosen inputs by masking - real_chosen_inputs = tf.boolean_mask(real_optional_inputs_value, tf_variables[name_prefix]['optional_inputs']) + real_chosen_inputs = tf.boolean_mask( + real_optional_inputs_value, tf_variables[name_prefix]['optional_inputs']) # build tensorflow graph of different branches by using tf.case branches = dict() for func_id in range(len(funcs)): - func_output = real_func_value[func_id]( - [fixed_inputs, real_chosen_inputs], *real_funcs_args[func_id]) + func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], *real_funcs_args[func_id]) branches[tf.equal(tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output - layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) + layer_out = tf.case(branches, exclusive=True, + default=lambda: func_output) return layer_out + def reload_tensorflow_variables(session, tf): + subgraph_from_tuner = trial.get_next_parameter() + for mutable_id, mutable_block in subgraph_from_tuner.items(): + for mutable_layer_id, mutable_layer in mutable_block.items(): + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # extract layer information from the subgraph sampled by tuner + chosen_layer = name_space[name_prefix]['funcs'].index( + mutable_layer["chosen_layer"]) + chosen_inputs = [1 if inp in mutable_layer["chosen_inputs"] + else 0 for inp in name_space[name_prefix]['funcs']] + # load these information into pre-defined tensorflow variables + tf_variables[name_prefix]['funcs'].load(chosen_layer, session) + tf_variables[name_prefix]['optional_inputs'].load( + chosen_inputs, session) + def _get_param(key): if trial._params is None: trial.get_next_parameter() diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index 4cc2f1f261..14f836b5a1 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -27,7 +27,7 @@ def parse_annotation_mutable_layers(code, lineno): """Parse the string of mutable layers in annotation. - Return a list of AST Expr nodes + Return a list of AST Expr nodes and NAS mode code: annotation string (excluding '@') """ module = ast.parse(code) @@ -38,7 +38,11 @@ def parse_annotation_mutable_layers(code, lineno): nodes = [] mutable_id = 'mutable_block_' + str(lineno) mutable_layer_cnt = 0 + mode = 'general' for arg in call.args: + if type(arg) is ast.Str: + assert arg.s in ['general', 'tensorflow'], 'Unsupported mode %s' % arg.s + mode = arg.s fields = {'layer_choice': False, 'fixed_inputs': False, 'optional_inputs': False, @@ -110,10 +114,12 @@ def parse_annotation_mutable_layers(code, lineno): else: target_call_args.append(ast.Dict(keys=[], values=[])) target_call_args.append(ast.Num(n=0)) + if mode == 'tensorflow': + target_call_args.append(ast.Name('tensorflow')) target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) node = ast.Assign(targets=[layer_output], value=target_call) nodes.append(node) - return nodes + return nodes, mode def parse_annotation(code): """Parse an annotation string. @@ -281,6 +287,7 @@ def __init__(self): self.stack = [] self.last_line = 0 self.annotated = False + self.mode = 'general' def visit(self, node): if isinstance(node, (ast.expr, ast.stmt)): @@ -325,7 +332,8 @@ def _visit_string(self, node): return parse_annotation(string[1:]) # expand annotation string to code if string.startswith('@nni.mutable_layers'): - return parse_annotation_mutable_layers(string[1:], node.lineno) + nodes, self.mode = parse_annotation_mutable_layers(string[1:], node.lineno) + return nodes if string.startswith('@nni.variable') \ or string.startswith('@nni.function_choice'): @@ -369,5 +377,8 @@ def parse(code): if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': last_future_import = i nodes.insert(last_future_import + 1, import_nni) + if transformer.mode == 'tensorflow': + import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) + nodes.insert(last_future_import + 1, import_tf) return astor.to_source(ast_tree) From 0fe0c4732873d765030af06e64e539d4e17a3695 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 19 Jun 2019 11:00:55 +0800 Subject: [PATCH 03/18] fix bugs --- tools/nni_annotation/code_generator.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index 14f836b5a1..0dabbdcb26 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -38,7 +38,13 @@ def parse_annotation_mutable_layers(code, lineno): nodes = [] mutable_id = 'mutable_block_' + str(lineno) mutable_layer_cnt = 0 - mode = 'general' + if call.keywords: + mode = call.keywords[0].value + assert isinstance(mode, (ast.Str, ast.Name)), 'Mode must be a string or Name' + mode = mode.s if isinstance(mode, ast.Str) else mode.id + assert mode in ['general', 'oneshot-tf', 'oneshot-pytorch'] + else: + mode = 'general' for arg in call.args: if type(arg) is ast.Str: assert arg.s in ['general', 'tensorflow'], 'Unsupported mode %s' % arg.s @@ -114,7 +120,7 @@ def parse_annotation_mutable_layers(code, lineno): else: target_call_args.append(ast.Dict(keys=[], values=[])) target_call_args.append(ast.Num(n=0)) - if mode == 'tensorflow': + if mode == 'oneshot-tf': target_call_args.append(ast.Name('tensorflow')) target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) node = ast.Assign(targets=[layer_output], value=target_call) @@ -323,8 +329,13 @@ def _visit_string(self, node): return node # not an annotation, ignore it if string.startswith('@nni.get_next_parameter'): - deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. Please remove this line in the trial code." - print_warning(deprecated_message) + call_node = parse_annotation(string[1:]).value + if call_node.args: + call_attr = ast.Attribute(value=ast.Name(id='nni', ctx=ast.Load()), attr='reload_tensorflow_variables', ctx=ast.Load()) + return ast.Call(func=call_attr, args=call_node.args, keywords=[]) + else: + deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. Please remove this line in the trial code." + print_warning(deprecated_message) if string.startswith('@nni.report_intermediate_result') \ or string.startswith('@nni.report_final_result') \ @@ -377,7 +388,7 @@ def parse(code): if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': last_future_import = i nodes.insert(last_future_import + 1, import_nni) - if transformer.mode == 'tensorflow': + if transformer.mode == 'oneshot-tf': import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) nodes.insert(last_future_import + 1, import_tf) From e36c7672139716d217ca37297b8b3363fa104f1f Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 19 Jun 2019 19:41:12 +0800 Subject: [PATCH 04/18] fix bugs --- src/sdk/pynni/nni/smartparam.py | 13 ++++++++++--- tools/nni_annotation/code_generator.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index 9c81f61434..0f40d9bdc9 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -164,11 +164,16 @@ def mutable_layer( name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) # store namespace global name_space + if 'name_space' not in globals(): + name_space = dict() + name_space[mutable_id] = True name_space[name_prefix] = dict() name_space[name_prefix]['funcs'] = list(funcs) name_space[name_prefix]['optional_inputs'] = list(optional_inputs) # create tensorflow variables as 1/0 signals used to form subgraph global tf_variables + if 'tf_variables' not in globals(): + tf_variables = dict() name_for_optional_inputs = name_prefix + '_optional_inputs' name_for_funcs = name_prefix + '_funcs' tf_variables[name_prefix] = dict() @@ -192,23 +197,25 @@ def mutable_layer( # build tensorflow graph of different branches by using tf.case branches = dict() for func_id in range(len(funcs)): - func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], *real_funcs_args[func_id]) + func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) branches[tf.equal(tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) return layer_out - def reload_tensorflow_variables(session, tf): + def reload_tensorflow_variables(session): subgraph_from_tuner = trial.get_next_parameter() for mutable_id, mutable_block in subgraph_from_tuner.items(): + if mutable_id not in name_space: + continue for mutable_layer_id, mutable_layer in mutable_block.items(): name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) # extract layer information from the subgraph sampled by tuner chosen_layer = name_space[name_prefix]['funcs'].index( mutable_layer["chosen_layer"]) chosen_inputs = [1 if inp in mutable_layer["chosen_inputs"] - else 0 for inp in name_space[name_prefix]['funcs']] + else 0 for inp in name_space[name_prefix]['optional_inputs']] # load these information into pre-defined tensorflow variables tf_variables[name_prefix]['funcs'].load(chosen_layer, session) tf_variables[name_prefix]['optional_inputs'].load( diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index 0dabbdcb26..05e9e640e2 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -121,7 +121,7 @@ def parse_annotation_mutable_layers(code, lineno): target_call_args.append(ast.Dict(keys=[], values=[])) target_call_args.append(ast.Num(n=0)) if mode == 'oneshot-tf': - target_call_args.append(ast.Name('tensorflow')) + target_call_args.append(ast.Name(id='tensorflow')) target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) node = ast.Assign(targets=[layer_output], value=target_call) nodes.append(node) @@ -332,7 +332,7 @@ def _visit_string(self, node): call_node = parse_annotation(string[1:]).value if call_node.args: call_attr = ast.Attribute(value=ast.Name(id='nni', ctx=ast.Load()), attr='reload_tensorflow_variables', ctx=ast.Load()) - return ast.Call(func=call_attr, args=call_node.args, keywords=[]) + return ast.Expr(value=ast.Call(func=call_attr, args=call_node.args, keywords=[])) else: deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. Please remove this line in the trial code." print_warning(deprecated_message) From 5a256b0db2a41526708ae0d60c62fe3e35026602 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Thu, 20 Jun 2019 10:30:10 +0800 Subject: [PATCH 05/18] remove unnecessary lines --- tools/nni_annotation/code_generator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index 05e9e640e2..00ef0c28ae 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -21,7 +21,6 @@ import ast import astor -from nni_cmd.common_utils import print_warning # pylint: disable=unidiomatic-typecheck @@ -333,9 +332,6 @@ def _visit_string(self, node): if call_node.args: call_attr = ast.Attribute(value=ast.Name(id='nni', ctx=ast.Load()), attr='reload_tensorflow_variables', ctx=ast.Load()) return ast.Expr(value=ast.Call(func=call_attr, args=call_node.args, keywords=[])) - else: - deprecated_message = "'@nni.get_next_parameter' is deprecated in annotation due to inconvenience. Please remove this line in the trial code." - print_warning(deprecated_message) if string.startswith('@nni.report_intermediate_result') \ or string.startswith('@nni.report_final_result') \ From d6f90cf47a97eccc5930a2c72b5659b9fb547fa8 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Sun, 23 Jun 2019 20:12:52 +0800 Subject: [PATCH 06/18] dev oneshot and darts --- src/sdk/pynni/nni/smartparam.py | 83 +++++++++++++++++++------- tools/nni_annotation/code_generator.py | 9 +-- 2 files changed, 65 insertions(+), 27 deletions(-) diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index 0f40d9bdc9..08290b1b51 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -130,7 +130,8 @@ def mutable_layer( fixed_inputs, optional_inputs, optional_input_size, - tf=None): + tf=None, + mode='general'): '''execute the chosen function and inputs. Below is an example of chosen function and inputs: { @@ -152,7 +153,9 @@ def mutable_layer( optional_input_size: number of candidate inputs to be chosen tf: tensorflow module ''' - if tf is None: + global train_mode + train_mode = mode + if mode == 'general': mutable_block = _get_param(mutable_id) chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] @@ -160,19 +163,20 @@ def mutable_layer( for input_name in chosen_inputs] layer_out = funcs[chosen_layer]( [fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) - else: + elif mode == 'general-tf': + assert tf is not None, 'Internal Error: Tensorflow should not be None in oneshot-tf mode' name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) # store namespace - global name_space if 'name_space' not in globals(): + global name_space name_space = dict() name_space[mutable_id] = True name_space[name_prefix] = dict() name_space[name_prefix]['funcs'] = list(funcs) name_space[name_prefix]['optional_inputs'] = list(optional_inputs) # create tensorflow variables as 1/0 signals used to form subgraph - global tf_variables if 'tf_variables' not in globals(): + global tf_variables tf_variables = dict() name_for_optional_inputs = name_prefix + '_optional_inputs' name_for_funcs = name_prefix + '_funcs' @@ -201,25 +205,62 @@ def mutable_layer( branches[tf.equal(tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) + elif mode == 'oneshot-tf': + inputs = optional_inputs + inputs_num = len(inputs) + # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs + if inputs_num > 0: + rate = 0.01 ** (1 / inputs_num) + noise_shape = [inputs_num] + [1] * inputs[0].get_shape().as_list() + inputs = tf.nn.dropout(inputs, rate=rate, noise_shape=noise_shape) + layer_outs = [func([fixed_inputs, inputs], **funcs_args[chosen_layer]) for func in funcs] + noise_shape = [len(layer_outs)] + [1] * layer_outs[0].get_shape().as_list() + layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) + layer_out = tf.add_n(layer_outs) + elif mode == 'darts-tf': + layer_outs = [func([fixed_inputs, inputs], **funcs_args[chosen_layer]) for func in funcs] + # Create architecture weights for every func(op) + var_name = "{}_{}_".format(mutable_id, mutable_layer_id, "arch_weights") + if 'arch_logits_list' not in globals(): + global arch_logits_list + arch_logits_list = list() + arch_logits = tf.get_variable(var_name, shape=[len[funcs]], trainable=False) + arch_logits_list.append(arch_logits) + arch_weights = tf.nn.softmax(arch_logits) + layer_out = tf.add_n([arch_weights[idx] * out for idx, out in enumerate(layer_outs)]) + elif mode == 'darts-pytorch': + raise NotImplementedError() return layer_out - def reload_tensorflow_variables(session): - subgraph_from_tuner = trial.get_next_parameter() - for mutable_id, mutable_block in subgraph_from_tuner.items(): - if mutable_id not in name_space: - continue - for mutable_layer_id, mutable_layer in mutable_block.items(): - name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) - # extract layer information from the subgraph sampled by tuner - chosen_layer = name_space[name_prefix]['funcs'].index( - mutable_layer["chosen_layer"]) - chosen_inputs = [1 if inp in mutable_layer["chosen_inputs"] - else 0 for inp in name_space[name_prefix]['optional_inputs']] - # load these information into pre-defined tensorflow variables - tf_variables[name_prefix]['funcs'].load(chosen_layer, session) - tf_variables[name_prefix]['optional_inputs'].load( - chosen_inputs, session) + def reload_tensorflow_variables(session, tf=None): + if train_mode == 'general-tf': + subgraph_from_tuner = trial.get_next_parameter() + for mutable_id, mutable_block in subgraph_from_tuner.items(): + if mutable_id not in name_space: + continue + for mutable_layer_id, mutable_layer in mutable_block.items(): + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # extract layer information from the subgraph sampled by tuner + chosen_layer = name_space[name_prefix]['funcs'].index( + mutable_layer["chosen_layer"]) + chosen_inputs = [1 if inp in mutable_layer["chosen_inputs"] + else 0 for inp in name_space[name_prefix]['optional_inputs']] + # load these information into pre-defined tensorflow variables + tf_variables[name_prefix]['funcs'].load(chosen_layer, session) + tf_variables[name_prefix]['optional_inputs'].load( + chosen_inputs, session) + elif train_mode == 'darts-tf': + if 'optimizer' not in globals(): + global arch_logits_list + global optimizer + global train_op + optimizer = tf.MomentumOptimizer(learning_rate=0.025) + # TODO: Calculate loss + grads_and_vars = optimizer.compute_gradients(0, arch_logits_list) + train_op = optimizer.apply_gradients(grads_and_vars) + session.run(train_op) + def _get_param(key): if trial._params is None: diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index 00ef0c28ae..5e7e15a4b8 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -41,13 +41,10 @@ def parse_annotation_mutable_layers(code, lineno): mode = call.keywords[0].value assert isinstance(mode, (ast.Str, ast.Name)), 'Mode must be a string or Name' mode = mode.s if isinstance(mode, ast.Str) else mode.id - assert mode in ['general', 'oneshot-tf', 'oneshot-pytorch'] + assert mode in ['general', 'general-tf', 'oneshot-tf', 'darts-tf'] else: mode = 'general' for arg in call.args: - if type(arg) is ast.Str: - assert arg.s in ['general', 'tensorflow'], 'Unsupported mode %s' % arg.s - mode = arg.s fields = {'layer_choice': False, 'fixed_inputs': False, 'optional_inputs': False, @@ -119,7 +116,7 @@ def parse_annotation_mutable_layers(code, lineno): else: target_call_args.append(ast.Dict(keys=[], values=[])) target_call_args.append(ast.Num(n=0)) - if mode == 'oneshot-tf': + if mode in ['general-tf', 'oneshot-tf', 'darts-tf']: target_call_args.append(ast.Name(id='tensorflow')) target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) node = ast.Assign(targets=[layer_output], value=target_call) @@ -384,7 +381,7 @@ def parse(code): if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': last_future_import = i nodes.insert(last_future_import + 1, import_nni) - if transformer.mode == 'oneshot-tf': + if transformer.mode in ['general-tf', 'oneshot-tf', 'darts-tf']: import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) nodes.insert(last_future_import + 1, import_tf) From b9e108fd23dd3d4c4dcd060409d989fd1e465cc6 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Mon, 24 Jun 2019 14:05:24 +0800 Subject: [PATCH 07/18] dev nas --- .../rest_server/restValidationSchemas.ts | 1 + src/sdk/pynni/nni/__init__.py | 1 + src/sdk/pynni/nni/smartparam.py | 146 ++++-------------- tools/nni_annotation/__init__.py | 3 +- tools/nni_annotation/code_generator.py | 28 ++-- tools/nni_cmd/config_schema.py | 3 +- tools/nni_cmd/launcher.py | 2 +- 7 files changed, 51 insertions(+), 133 deletions(-) diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index f794df4d70..5e9b03946d 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -51,6 +51,7 @@ export namespace ValidationSchemas { command: joi.string().min(1), virtualCluster: joi.string(), shmMB: joi.number(), + nasMode: joi.string().valid('classic_mode', 'enas_mode', 'oneshot_mode') worker: joi.object({ replicas: joi.number().min(1).required(), image: joi.string().min(1), diff --git a/src/sdk/pynni/nni/__init__.py b/src/sdk/pynni/nni/__init__.py index 0358b45bbf..8465846d4c 100644 --- a/src/sdk/pynni/nni/__init__.py +++ b/src/sdk/pynni/nni/__init__.py @@ -23,6 +23,7 @@ from .trial import * from .smartparam import * +from .nas_utils import reload_tensorflow_variables class NoMoreTrialError(Exception): def __init__(self,ErrorInfo): diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index 08290b1b51..8dfe67b06b 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -23,6 +23,7 @@ from .env_vars import trial_env_vars from . import trial +from .nas_utils import classic_mode, enas_mode, oneshot_mode __all__ = [ @@ -37,8 +38,7 @@ 'lognormal', 'qlognormal', 'function_choice', - 'mutable_layer', - 'reload_tensorflow_variables' + 'mutable_layer' ] @@ -83,10 +83,6 @@ def function_choice(*funcs, name=None): def mutable_layer(): raise RuntimeError('Cannot call nni.mutable_layer in this mode') - def reload_tensorflow_variables(session, tf): - raise RuntimeError( - 'Cannot call nni.reload_tensorflow_variables in this mode') - else: def choice(options, name=None, key=None): @@ -153,114 +149,36 @@ def mutable_layer( optional_input_size: number of candidate inputs to be chosen tf: tensorflow module ''' - global train_mode - train_mode = mode - if mode == 'general': - mutable_block = _get_param(mutable_id) - chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] - chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] - real_chosen_inputs = [optional_inputs[input_name] - for input_name in chosen_inputs] - layer_out = funcs[chosen_layer]( - [fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) - elif mode == 'general-tf': - assert tf is not None, 'Internal Error: Tensorflow should not be None in oneshot-tf mode' - name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) - # store namespace - if 'name_space' not in globals(): - global name_space - name_space = dict() - name_space[mutable_id] = True - name_space[name_prefix] = dict() - name_space[name_prefix]['funcs'] = list(funcs) - name_space[name_prefix]['optional_inputs'] = list(optional_inputs) - # create tensorflow variables as 1/0 signals used to form subgraph - if 'tf_variables' not in globals(): - global tf_variables - tf_variables = dict() - name_for_optional_inputs = name_prefix + '_optional_inputs' - name_for_funcs = name_prefix + '_funcs' - tf_variables[name_prefix] = dict() - tf_variables[name_prefix]['optional_inputs'] = tf.get_variable(name_for_optional_inputs, - [len(optional_inputs)], - dtype=tf.bool, - trainable=False) - tf_variables[name_prefix]['funcs'] = tf.get_variable( - name_for_funcs, [], dtype=tf.int64, trainable=False) - - # get real values using their variable names - real_optional_inputs_value = [optional_inputs[name] - for name in name_space[name_prefix]['optional_inputs']] - real_func_value = [funcs[name] - for name in name_space[name_prefix]['funcs']] - real_funcs_args = [funcs_args[name] - for name in name_space[name_prefix]['funcs']] - # build tensorflow graph of geting chosen inputs by masking - real_chosen_inputs = tf.boolean_mask( - real_optional_inputs_value, tf_variables[name_prefix]['optional_inputs']) - # build tensorflow graph of different branches by using tf.case - branches = dict() - for func_id in range(len(funcs)): - func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) - branches[tf.equal(tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output - layer_out = tf.case(branches, exclusive=True, - default=lambda: func_output) - elif mode == 'oneshot-tf': - inputs = optional_inputs - inputs_num = len(inputs) - # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs - if inputs_num > 0: - rate = 0.01 ** (1 / inputs_num) - noise_shape = [inputs_num] + [1] * inputs[0].get_shape().as_list() - inputs = tf.nn.dropout(inputs, rate=rate, noise_shape=noise_shape) - layer_outs = [func([fixed_inputs, inputs], **funcs_args[chosen_layer]) for func in funcs] - noise_shape = [len(layer_outs)] + [1] * layer_outs[0].get_shape().as_list() - layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) - layer_out = tf.add_n(layer_outs) - elif mode == 'darts-tf': - layer_outs = [func([fixed_inputs, inputs], **funcs_args[chosen_layer]) for func in funcs] - # Create architecture weights for every func(op) - var_name = "{}_{}_".format(mutable_id, mutable_layer_id, "arch_weights") - if 'arch_logits_list' not in globals(): - global arch_logits_list - arch_logits_list = list() - arch_logits = tf.get_variable(var_name, shape=[len[funcs]], trainable=False) - arch_logits_list.append(arch_logits) - arch_weights = tf.nn.softmax(arch_logits) - layer_out = tf.add_n([arch_weights[idx] * out for idx, out in enumerate(layer_outs)]) - elif mode == 'darts-pytorch': - raise NotImplementedError() - - return layer_out - - def reload_tensorflow_variables(session, tf=None): - if train_mode == 'general-tf': - subgraph_from_tuner = trial.get_next_parameter() - for mutable_id, mutable_block in subgraph_from_tuner.items(): - if mutable_id not in name_space: - continue - for mutable_layer_id, mutable_layer in mutable_block.items(): - name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) - # extract layer information from the subgraph sampled by tuner - chosen_layer = name_space[name_prefix]['funcs'].index( - mutable_layer["chosen_layer"]) - chosen_inputs = [1 if inp in mutable_layer["chosen_inputs"] - else 0 for inp in name_space[name_prefix]['optional_inputs']] - # load these information into pre-defined tensorflow variables - tf_variables[name_prefix]['funcs'].load(chosen_layer, session) - tf_variables[name_prefix]['optional_inputs'].load( - chosen_inputs, session) - elif train_mode == 'darts-tf': - if 'optimizer' not in globals(): - global arch_logits_list - global optimizer - global train_op - optimizer = tf.MomentumOptimizer(learning_rate=0.025) - # TODO: Calculate loss - grads_and_vars = optimizer.compute_gradients(0, arch_logits_list) - train_op = optimizer.apply_gradients(grads_and_vars) - session.run(train_op) - + if mode == 'classic_mode': # nas_mode + return classic_mode(mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size) + elif mode == 'enas_mode': # enas_mode + assert tf is not None, 'Internal Error: Tensorflow should not be None in enas_mode' + return enas_mode(mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf) + elif mode == 'oneshot_mode': # oneshot_mode + assert tf is not None, 'Internal Error: Tensorflow should not be None in oneshot_mode' + return oneshot_mode(mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf) + else: + raise RuntimeError('Unrecognized mode: %s' % mode) def _get_param(key): if trial._params is None: diff --git a/tools/nni_annotation/__init__.py b/tools/nni_annotation/__init__.py index 20ac69f659..48cbf6ea7f 100644 --- a/tools/nni_annotation/__init__.py +++ b/tools/nni_annotation/__init__.py @@ -76,11 +76,12 @@ def _generate_file_search_space(path, module): return search_space -def expand_annotations(src_dir, dst_dir, exp_id='', trial_id=''): +def expand_annotations(src_dir, dst_dir, exp_id='', trial_id='', nas_mode=None): """Expand annotations in user code. Return dst_dir if annotation detected; return src_dir if not. src_dir: directory path of user code (str) dst_dir: directory to place generated files (str) + nas_mode: the mode of NAS given that NAS interface is used """ if src_dir[-1] == slash: src_dir = src_dir[:-1] diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index 5e7e15a4b8..cae3105fc6 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -24,10 +24,11 @@ # pylint: disable=unidiomatic-typecheck -def parse_annotation_mutable_layers(code, lineno): +def parse_annotation_mutable_layers(code, lineno, add_tensorflow): """Parse the string of mutable layers in annotation. Return a list of AST Expr nodes and NAS mode code: annotation string (excluding '@') + add_tensorflow: whether to add tensorflow as an arg in the end """ module = ast.parse(code) assert type(module) is ast.Module, 'internal error #1' @@ -37,13 +38,6 @@ def parse_annotation_mutable_layers(code, lineno): nodes = [] mutable_id = 'mutable_block_' + str(lineno) mutable_layer_cnt = 0 - if call.keywords: - mode = call.keywords[0].value - assert isinstance(mode, (ast.Str, ast.Name)), 'Mode must be a string or Name' - mode = mode.s if isinstance(mode, ast.Str) else mode.id - assert mode in ['general', 'general-tf', 'oneshot-tf', 'darts-tf'] - else: - mode = 'general' for arg in call.args: fields = {'layer_choice': False, 'fixed_inputs': False, @@ -116,7 +110,7 @@ def parse_annotation_mutable_layers(code, lineno): else: target_call_args.append(ast.Dict(keys=[], values=[])) target_call_args.append(ast.Num(n=0)) - if mode in ['general-tf', 'oneshot-tf', 'darts-tf']: + if add_tensorflow: target_call_args.append(ast.Name(id='tensorflow')) target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) node = ast.Assign(targets=[layer_output], value=target_call) @@ -285,11 +279,11 @@ def visit_Call(self, node): # pylint: disable=invalid-name class Transformer(ast.NodeTransformer): """Transform original code to annotated code""" - def __init__(self): + def __init__(self, nas_mode=None): self.stack = [] self.last_line = 0 self.annotated = False - self.mode = 'general' + self.nas_mode = nas_mode def visit(self, node): if isinstance(node, (ast.expr, ast.stmt)): @@ -336,7 +330,8 @@ def _visit_string(self, node): return parse_annotation(string[1:]) # expand annotation string to code if string.startswith('@nni.mutable_layers'): - nodes, self.mode = parse_annotation_mutable_layers(string[1:], node.lineno) + add_tensorflow = self.mode in ['enas_mode', 'oneshot_mode'] + nodes = parse_annotation_mutable_layers(string[1:], node.lineno, add_tensorflow) return nodes if string.startswith('@nni.variable') \ @@ -355,17 +350,18 @@ def _visit_children(self, node): return node -def parse(code): +def parse(code, nas_mode=None): """Annotate user code. Return annotated code (str) if annotation detected; return None if not. - code: original user code (str) + code: original user code (str), + nas_mode: the mode of NAS given that NAS interface is used """ try: ast_tree = ast.parse(code) except Exception: raise RuntimeError('Bad Python code') - transformer = Transformer() + transformer = Transformer(nas_mode) try: transformer.visit(ast_tree) except AssertionError as exc: @@ -381,7 +377,7 @@ def parse(code): if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': last_future_import = i nodes.insert(last_future_import + 1, import_nni) - if transformer.mode in ['general-tf', 'oneshot-tf', 'darts-tf']: + if nas_mode in ['enas_mode', 'oneshot_mode']: import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) nodes.insert(last_future_import + 1, import_tf) diff --git a/tools/nni_cmd/config_schema.py b/tools/nni_cmd/config_schema.py index ef5173d8b9..8f95f07e3f 100644 --- a/tools/nni_cmd/config_schema.py +++ b/tools/nni_cmd/config_schema.py @@ -180,7 +180,8 @@ def setPathCheck(key): 'trial':{ 'command': setType('command', str), 'codeDir': setPathCheck('codeDir'), - 'gpuNum': setNumberRange('gpuNum', int, 0, 99999) + 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), + Optional('nasMode'): setChoice('nas_mode', 'enas_mode', 'oneshot_mode') } } diff --git a/tools/nni_cmd/launcher.py b/tools/nni_cmd/launcher.py index 8e0eb57fc6..e47246133b 100644 --- a/tools/nni_cmd/launcher.py +++ b/tools/nni_cmd/launcher.py @@ -380,7 +380,7 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen if not os.path.isdir(path): os.makedirs(path) path = tempfile.mkdtemp(dir=path) - code_dir = expand_annotations(experiment_config['trial']['codeDir'], path) + code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=experiment_config['trial']) experiment_config['trial']['codeDir'] = code_dir search_space = generate_search_space(code_dir) experiment_config['searchSpace'] = json.dumps(search_space) From d79b56560de5c8b19d5d4ae3ce5b1d1aed365388 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Mon, 24 Jun 2019 14:43:28 +0800 Subject: [PATCH 08/18] dev enas and oneshot --- src/sdk/pynni/nni/smartparam.py | 10 +++++----- tools/nni_annotation/__init__.py | 6 +++--- tools/nni_annotation/code_generator.py | 12 ++++++------ tools/nni_cmd/config_schema.py | 2 +- tools/nni_cmd/launcher.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index 8dfe67b06b..b42aa4ab57 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -126,8 +126,8 @@ def mutable_layer( fixed_inputs, optional_inputs, optional_input_size, - tf=None, - mode='general'): + mode='general', + tf=None): '''execute the chosen function and inputs. Below is an example of chosen function and inputs: { @@ -149,7 +149,7 @@ def mutable_layer( optional_input_size: number of candidate inputs to be chosen tf: tensorflow module ''' - if mode == 'classic_mode': # nas_mode + if mode == 'classic_mode': return classic_mode(mutable_id, mutable_layer_id, funcs, @@ -157,7 +157,7 @@ def mutable_layer( fixed_inputs, optional_inputs, optional_input_size) - elif mode == 'enas_mode': # enas_mode + elif mode == 'enas_mode': assert tf is not None, 'Internal Error: Tensorflow should not be None in enas_mode' return enas_mode(mutable_id, mutable_layer_id, @@ -167,7 +167,7 @@ def mutable_layer( optional_inputs, optional_input_size, tf) - elif mode == 'oneshot_mode': # oneshot_mode + elif mode == 'oneshot_mode': assert tf is not None, 'Internal Error: Tensorflow should not be None in oneshot_mode' return oneshot_mode(mutable_id, mutable_layer_id, diff --git a/tools/nni_annotation/__init__.py b/tools/nni_annotation/__init__.py index 48cbf6ea7f..ebfe73f5ba 100644 --- a/tools/nni_annotation/__init__.py +++ b/tools/nni_annotation/__init__.py @@ -109,7 +109,7 @@ def expand_annotations(src_dir, dst_dir, exp_id='', trial_id='', nas_mode=None): dst_path = os.path.join(dst_subdir, file_name) if file_name.endswith('.py'): if trial_id == '': - annotated |= _expand_file_annotations(src_path, dst_path) + annotated |= _expand_file_annotations(src_path, dst_path, nas_mode) else: module = package + file_name[:-3] annotated |= _generate_specific_file(src_path, dst_path, exp_id, trial_id, module) @@ -121,10 +121,10 @@ def expand_annotations(src_dir, dst_dir, exp_id='', trial_id='', nas_mode=None): return dst_dir if annotated else src_dir -def _expand_file_annotations(src_path, dst_path): +def _expand_file_annotations(src_path, dst_path, nas_mode): with open(src_path) as src, open(dst_path, 'w') as dst: try: - annotated_code = code_generator.parse(src.read()) + annotated_code = code_generator.parse(src.read(), nas_mode) if annotated_code is None: shutil.copyfile(src_path, dst_path) return False diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index cae3105fc6..d26dde3ad1 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -24,11 +24,11 @@ # pylint: disable=unidiomatic-typecheck -def parse_annotation_mutable_layers(code, lineno, add_tensorflow): +def parse_annotation_mutable_layers(code, lineno, nas_mode): """Parse the string of mutable layers in annotation. Return a list of AST Expr nodes and NAS mode code: annotation string (excluding '@') - add_tensorflow: whether to add tensorflow as an arg in the end + nas_mode: the mode of NAS """ module = ast.parse(code) assert type(module) is ast.Module, 'internal error #1' @@ -110,12 +110,13 @@ def parse_annotation_mutable_layers(code, lineno, add_tensorflow): else: target_call_args.append(ast.Dict(keys=[], values=[])) target_call_args.append(ast.Num(n=0)) - if add_tensorflow: + target_call_args.append(ast.Str(s=nas_mode)) + if nas_mode in ['enas_mode', 'oneshot_mode']: target_call_args.append(ast.Name(id='tensorflow')) target_call = ast.Call(func=target_call_attr, args=target_call_args, keywords=[]) node = ast.Assign(targets=[layer_output], value=target_call) nodes.append(node) - return nodes, mode + return nodes def parse_annotation(code): """Parse an annotation string. @@ -330,8 +331,7 @@ def _visit_string(self, node): return parse_annotation(string[1:]) # expand annotation string to code if string.startswith('@nni.mutable_layers'): - add_tensorflow = self.mode in ['enas_mode', 'oneshot_mode'] - nodes = parse_annotation_mutable_layers(string[1:], node.lineno, add_tensorflow) + nodes = parse_annotation_mutable_layers(string[1:], node.lineno, self.nas_mode) return nodes if string.startswith('@nni.variable') \ diff --git a/tools/nni_cmd/config_schema.py b/tools/nni_cmd/config_schema.py index 8f95f07e3f..84c7ad3bb5 100644 --- a/tools/nni_cmd/config_schema.py +++ b/tools/nni_cmd/config_schema.py @@ -181,7 +181,7 @@ def setPathCheck(key): 'command': setType('command', str), 'codeDir': setPathCheck('codeDir'), 'gpuNum': setNumberRange('gpuNum', int, 0, 99999), - Optional('nasMode'): setChoice('nas_mode', 'enas_mode', 'oneshot_mode') + Optional('nasMode'): setChoice('classic_mode', 'enas_mode', 'oneshot_mode') } } diff --git a/tools/nni_cmd/launcher.py b/tools/nni_cmd/launcher.py index e47246133b..8eaca55fc2 100644 --- a/tools/nni_cmd/launcher.py +++ b/tools/nni_cmd/launcher.py @@ -380,7 +380,7 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen if not os.path.isdir(path): os.makedirs(path) path = tempfile.mkdtemp(dir=path) - code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=experiment_config['trial']) + code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=experiment_config['trial']['nasMode']) experiment_config['trial']['codeDir'] = code_dir search_space = generate_search_space(code_dir) experiment_config['searchSpace'] = json.dumps(search_space) From 5bfefc6a6ba72eaac2f2e84a8398d70fb44a7937 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Mon, 24 Jun 2019 14:45:39 +0800 Subject: [PATCH 09/18] dev enas and oneshot --- src/nni_manager/rest_server/restValidationSchemas.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index 5e9b03946d..6a61c209c3 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -51,7 +51,7 @@ export namespace ValidationSchemas { command: joi.string().min(1), virtualCluster: joi.string(), shmMB: joi.number(), - nasMode: joi.string().valid('classic_mode', 'enas_mode', 'oneshot_mode') + nasMode: joi.string().valid('classic_mode', 'enas_mode', 'oneshot_mode'), worker: joi.object({ replicas: joi.number().min(1).required(), image: joi.string().min(1), From 39557bc862ee54fd6492f2e5cbe46cf99c0bfc8c Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Mon, 24 Jun 2019 14:50:01 +0800 Subject: [PATCH 10/18] dev enas and oneshot --- src/sdk/pynni/nni/nas_utils.py | 134 +++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/sdk/pynni/nni/nas_utils.py diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py new file mode 100644 index 0000000000..1a2bea4207 --- /dev/null +++ b/src/sdk/pynni/nni/nas_utils.py @@ -0,0 +1,134 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +# associated documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, +# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or +# substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT +# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# ================================================================================================== + +from . import trial + +def nas_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf=None,): + + if trial._params is None: + trial.get_next_parameter() + mutable_block = trial.get_current_parameter(mutable_id) + chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] + chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] + real_chosen_inputs = [optional_inputs[input_name] + for input_name in chosen_inputs] + layer_out = funcs[chosen_layer]( + [fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) + + return layer_out + +def enas_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size, + tf=None,): + + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # store namespace + if 'name_space' not in globals(): + global name_space + name_space = dict() + name_space[mutable_id] = True + name_space[name_prefix] = dict() + name_space[name_prefix]['funcs'] = list(funcs) + name_space[name_prefix]['optional_inputs'] = list(optional_inputs) + # create tensorflow variables as 1/0 signals used to form subgraph + if 'tf_variables' not in globals(): + global tf_variables + tf_variables = dict() + name_for_optional_inputs = name_prefix + '_optional_inputs' + name_for_funcs = name_prefix + '_funcs' + tf_variables[name_prefix] = dict() + tf_variables[name_prefix]['optional_inputs'] = tf.get_variable(name_for_optional_inputs, + [len(optional_inputs)], + dtype=tf.bool, + trainable=False) + tf_variables[name_prefix]['funcs'] = tf.get_variable( + name_for_funcs, [], dtype=tf.int64, trainable=False) + + # get real values using their variable names + real_optional_inputs_value = [optional_inputs[name] + for name in name_space[name_prefix]['optional_inputs']] + real_func_value = [funcs[name] + for name in name_space[name_prefix]['funcs']] + real_funcs_args = [funcs_args[name] + for name in name_space[name_prefix]['funcs']] + # build tensorflow graph of geting chosen inputs by masking + real_chosen_inputs = tf.boolean_mask( + real_optional_inputs_value, tf_variables[name_prefix]['optional_inputs']) + # build tensorflow graph of different branches by using tf.case + branches = dict() + for func_id in range(len(funcs)): + func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) + branches[tf.equal(tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output + layer_out = tf.case(branches, exclusive=True, + default=lambda: func_output) + + return layer_out + +def oneshot_mode( + mutable_id, + mutable_layer_id, + funcs, + funcs_args, + fixed_inputs, + optional_inputs, + optional_input_size): + inputs_num = len(optional_inputs) + # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs + if inputs_num > 0: + rate = 0.01 ** (1 / inputs_num) + noise_shape = [inputs_num] + [1] * optional_inputs[0].get_shape().as_list() + optional_inputs = tf.nn.dropout(optional_inputs, rate=rate, noise_shape=noise_shape) + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[chosen_layer]) for func in funcs] + noise_shape = [len(layer_outs)] + [1] * layer_outs[0].get_shape().as_list() + layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) + layer_out = tf.add_n(layer_outs) + + return layer_out + +def reload_tensorflow_variables(session, tf=None): + subgraph_from_tuner = trial.get_next_parameter() + for mutable_id, mutable_block in subgraph_from_tuner.items(): + if mutable_id not in name_space: + continue + for mutable_layer_id, mutable_layer in mutable_block.items(): + name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) + # extract layer information from the subgraph sampled by tuner + chosen_layer = name_space[name_prefix]['funcs'].index( + mutable_layer["chosen_layer"]) + chosen_inputs = [1 if inp in mutable_layer["chosen_inputs"] + else 0 for inp in name_space[name_prefix]['optional_inputs']] + # load these information into pre-defined tensorflow variables + tf_variables[name_prefix]['funcs'].load(chosen_layer, session) + tf_variables[name_prefix]['optional_inputs'].load( + chosen_inputs, session) \ No newline at end of file From 5f2e8a3e202217de10902f7e3178e5a854d8a115 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Mon, 24 Jun 2019 08:39:45 +0000 Subject: [PATCH 11/18] dev oneshot and enas --- src/sdk/pynni/nni/nas_utils.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py index 1a2bea4207..eb328cd0e3 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/src/sdk/pynni/nni/nas_utils.py @@ -20,15 +20,14 @@ from . import trial -def nas_mode( +def classic_mode( mutable_id, mutable_layer_id, funcs, funcs_args, fixed_inputs, optional_inputs, - optional_input_size, - tf=None,): + optional_input_size): if trial._params is None: trial.get_next_parameter() @@ -50,7 +49,7 @@ def enas_mode( fixed_inputs, optional_inputs, optional_input_size, - tf=None,): + tf): name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) # store namespace @@ -102,17 +101,21 @@ def oneshot_mode( funcs_args, fixed_inputs, optional_inputs, - optional_input_size): + optional_input_size, + tf): + optional_inputs = list(optional_inputs.values()) inputs_num = len(optional_inputs) # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs if inputs_num > 0: rate = 0.01 ** (1 / inputs_num) - noise_shape = [inputs_num] + [1] * optional_inputs[0].get_shape().as_list() + noise_shape = [inputs_num] + [1] * len(optional_inputs[0].get_shape()) optional_inputs = tf.nn.dropout(optional_inputs, rate=rate, noise_shape=noise_shape) - layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[chosen_layer]) for func in funcs] - noise_shape = [len(layer_outs)] + [1] * layer_outs[0].get_shape().as_list() + optional_inputs = [optional_inputs[idx] for idx in range(inputs_num)] + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) for func_name, func in funcs.items()] + rate = 0.01 ** (1 / len(layer_outs)) + noise_shape = [len(layer_outs)] + [1] * len(layer_outs[0].get_shape()) layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) - layer_out = tf.add_n(layer_outs) + layer_out = tf.reduce_sum(layer_outs, axis=0) return layer_out @@ -131,4 +134,4 @@ def reload_tensorflow_variables(session, tf=None): # load these information into pre-defined tensorflow variables tf_variables[name_prefix]['funcs'].load(chosen_layer, session) tf_variables[name_prefix]['optional_inputs'].load( - chosen_inputs, session) \ No newline at end of file + chosen_inputs, session) From 1bea6fb1d63d17616b30e414cb6736b756bbc829 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Mon, 24 Jun 2019 11:06:13 +0000 Subject: [PATCH 12/18] dev oneshot --- src/sdk/pynni/nni/nas_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py index eb328cd0e3..364d5402c3 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/src/sdk/pynni/nni/nas_utils.py @@ -103,6 +103,9 @@ def oneshot_mode( optional_inputs, optional_input_size, tf): + + if trial._params is None: + trial.get_next_parameter() optional_inputs = list(optional_inputs.values()) inputs_num = len(optional_inputs) # Calculate dropout rate according to the formular r^(1/k), where r is a hyper-parameter and k is the number of inputs From 1e245a78866e275cf4e963dfd27f88e7ff399c7f Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Tue, 25 Jun 2019 10:04:18 +0800 Subject: [PATCH 13/18] add ut --- src/sdk/pynni/nni/smartparam.py | 2 +- src/sdk/pynni/tests/test_smartparam.py | 17 ++++++++++++++++- tools/nni_annotation/code_generator.py | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index b42aa4ab57..07519b69ea 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -126,7 +126,7 @@ def mutable_layer( fixed_inputs, optional_inputs, optional_input_size, - mode='general', + mode='classic_mode', tf=None): '''execute the chosen function and inputs. Below is an example of chosen function and inputs: diff --git a/src/sdk/pynni/tests/test_smartparam.py b/src/sdk/pynni/tests/test_smartparam.py index 33fb783afc..001bd35032 100644 --- a/src/sdk/pynni/tests/test_smartparam.py +++ b/src/sdk/pynni/tests/test_smartparam.py @@ -38,7 +38,13 @@ def setUp(self): 'test_smartparam/choice3/choice': '[1, 2]', 'test_smartparam/choice4/choice': '{"a", 2}', 'test_smartparam/func/function_choice': 'bar', - 'test_smartparam/lambda_func/function_choice': "lambda: 2*3" + 'test_smartparam/lambda_func/function_choice': "lambda: 2*3", + 'mutable_block_66':{ + 'mutable_layer_0':{ + 'chosen_layer': 'conv2D(size=5)', + 'chosen_inputs': ['y'] + } + } } nni.trial._params = { 'parameter_id': 'test_trial', 'parameters': params } @@ -61,6 +67,13 @@ def test_lambda_func(self): val = nni.function_choice({"lambda: 2*3": lambda: 2*3, "lambda: 3*4": lambda: 3*4}, name = 'lambda_func', key='test_smartparam/lambda_func/function_choice') self.assertEqual(val, 6) + def test_mutable_layer(self): + layer_out = nni.mutable_layer('mutable_block_66', + 'mutable_layer_0', {'conv2D(size=3)': conv2D, 'conv2D(size=5)': conv2D}, {'conv2D(size=3)': + {'size':3}, 'conv2D(size=5)': {'size':5}}, [100], {'x':1,'y':2}, 1, 'classic_mode') + self.assertEqual(layer_out, [100, 2, 5]) + + def foo(): return 'foo' @@ -68,6 +81,8 @@ def foo(): def bar(): return 'bar' +def conv2D(inputs, size=3): + return inputs[0] + inputs[1] + [size] if __name__ == '__main__': main() diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index d26dde3ad1..5a77c7be34 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -26,7 +26,7 @@ def parse_annotation_mutable_layers(code, lineno, nas_mode): """Parse the string of mutable layers in annotation. - Return a list of AST Expr nodes and NAS mode + Return a list of AST Expr nodes code: annotation string (excluding '@') nas_mode: the mode of NAS """ From 790287d0e286ba72c03b78fc2dd05d0701298f74 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Tue, 25 Jun 2019 10:29:20 +0800 Subject: [PATCH 14/18] add docstring --- src/sdk/pynni/nni/nas_utils.py | 49 +++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py index 364d5402c3..cb6a7fcd9f 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/src/sdk/pynni/nni/nas_utils.py @@ -20,6 +20,7 @@ from . import trial + def classic_mode( mutable_id, mutable_layer_id, @@ -28,19 +29,20 @@ def classic_mode( fixed_inputs, optional_inputs, optional_input_size): - + '''Execute the chosen function and inputs directly''' if trial._params is None: trial.get_next_parameter() mutable_block = trial.get_current_parameter(mutable_id) chosen_layer = mutable_block[mutable_layer_id]["chosen_layer"] chosen_inputs = mutable_block[mutable_layer_id]["chosen_inputs"] real_chosen_inputs = [optional_inputs[input_name] - for input_name in chosen_inputs] + for input_name in chosen_inputs] layer_out = funcs[chosen_layer]( [fixed_inputs, real_chosen_inputs], **funcs_args[chosen_layer]) return layer_out + def enas_mode( mutable_id, mutable_layer_id, @@ -50,7 +52,9 @@ def enas_mode( optional_inputs, optional_input_size, tf): - + '''Build a tensorflow graph including all functions and optional_inputs using signal varaibles. + So that we can use those signal variables to change the whole graph into different subgraphs + in the `reload_tensorflow_variables` function''' name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) # store namespace if 'name_space' not in globals(): @@ -68,32 +72,36 @@ def enas_mode( name_for_funcs = name_prefix + '_funcs' tf_variables[name_prefix] = dict() tf_variables[name_prefix]['optional_inputs'] = tf.get_variable(name_for_optional_inputs, - [len(optional_inputs)], - dtype=tf.bool, - trainable=False) + [len( + optional_inputs)], + dtype=tf.bool, + trainable=False) tf_variables[name_prefix]['funcs'] = tf.get_variable( name_for_funcs, [], dtype=tf.int64, trainable=False) # get real values using their variable names real_optional_inputs_value = [optional_inputs[name] - for name in name_space[name_prefix]['optional_inputs']] + for name in name_space[name_prefix]['optional_inputs']] real_func_value = [funcs[name] - for name in name_space[name_prefix]['funcs']] + for name in name_space[name_prefix]['funcs']] real_funcs_args = [funcs_args[name] - for name in name_space[name_prefix]['funcs']] + for name in name_space[name_prefix]['funcs']] # build tensorflow graph of geting chosen inputs by masking real_chosen_inputs = tf.boolean_mask( real_optional_inputs_value, tf_variables[name_prefix]['optional_inputs']) # build tensorflow graph of different branches by using tf.case branches = dict() for func_id in range(len(funcs)): - func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) - branches[tf.equal(tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output + func_output = real_func_value[func_id]( + [fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) + branches[tf.equal(tf_variables[name_prefix]['funcs'], + func_id)] = lambda: func_output layer_out = tf.case(branches, exclusive=True, default=lambda: func_output) return layer_out + def oneshot_mode( mutable_id, mutable_layer_id, @@ -103,7 +111,9 @@ def oneshot_mode( optional_inputs, optional_input_size, tf): - + '''Execute all the functions using all the optional_inputs, where a dropout will be implemented + to optional_inputs. + ''' if trial._params is None: trial.get_next_parameter() optional_inputs = list(optional_inputs.values()) @@ -112,9 +122,11 @@ def oneshot_mode( if inputs_num > 0: rate = 0.01 ** (1 / inputs_num) noise_shape = [inputs_num] + [1] * len(optional_inputs[0].get_shape()) - optional_inputs = tf.nn.dropout(optional_inputs, rate=rate, noise_shape=noise_shape) + optional_inputs = tf.nn.dropout( + optional_inputs, rate=rate, noise_shape=noise_shape) optional_inputs = [optional_inputs[idx] for idx in range(inputs_num)] - layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) for func_name, func in funcs.items()] + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] rate = 0.01 ** (1 / len(layer_outs)) noise_shape = [len(layer_outs)] + [1] * len(layer_outs[0].get_shape()) layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) @@ -122,7 +134,14 @@ def oneshot_mode( return layer_out + def reload_tensorflow_variables(session, tf=None): + '''In Enas mode, this function reload every signal varaible created in `enas_mode` function so + the whole tensorflow graph will be changed into certain subgraph recerived from Tuner. + --------------- + session: the tensorflow session created by users + tf: tensorflow module + ''' subgraph_from_tuner = trial.get_next_parameter() for mutable_id, mutable_block in subgraph_from_tuner.items(): if mutable_id not in name_space: @@ -133,7 +152,7 @@ def reload_tensorflow_variables(session, tf=None): chosen_layer = name_space[name_prefix]['funcs'].index( mutable_layer["chosen_layer"]) chosen_inputs = [1 if inp in mutable_layer["chosen_inputs"] - else 0 for inp in name_space[name_prefix]['optional_inputs']] + else 0 for inp in name_space[name_prefix]['optional_inputs']] # load these information into pre-defined tensorflow variables tf_variables[name_prefix]['funcs'].load(chosen_layer, session) tf_variables[name_prefix]['optional_inputs'].load( From 891f8f50f0a1305bf773f5a5b573f276b6bfa210 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Tue, 25 Jun 2019 10:32:05 +0800 Subject: [PATCH 15/18] add docstring --- src/sdk/pynni/nni/nas_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py index cb6a7fcd9f..1067b4bddc 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/src/sdk/pynni/nni/nas_utils.py @@ -112,8 +112,7 @@ def oneshot_mode( optional_input_size, tf): '''Execute all the functions using all the optional_inputs, where a dropout will be implemented - to optional_inputs. - ''' + to optional_inputs.''' if trial._params is None: trial.get_next_parameter() optional_inputs = list(optional_inputs.values()) From 1461d5668088fb87bea7ba40f1ffeae5a19d1fc5 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Tue, 25 Jun 2019 02:35:33 +0000 Subject: [PATCH 16/18] fix --- src/sdk/pynni/nni/nas_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py index 364d5402c3..3e4526effc 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/src/sdk/pynni/nni/nas_utils.py @@ -115,10 +115,7 @@ def oneshot_mode( optional_inputs = tf.nn.dropout(optional_inputs, rate=rate, noise_shape=noise_shape) optional_inputs = [optional_inputs[idx] for idx in range(inputs_num)] layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) for func_name, func in funcs.items()] - rate = 0.01 ** (1 / len(layer_outs)) - noise_shape = [len(layer_outs)] + [1] * len(layer_outs[0].get_shape()) - layer_outs = tf.nn.dropout(layer_outs, rate=rate, noise_shape=noise_shape) - layer_out = tf.reduce_sum(layer_outs, axis=0) + layer_out = tf.add_n(layer_outs) return layer_out From 8f83ea5bc4c5aec222356c814e69dc01b824a1b8 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Tue, 25 Jun 2019 11:36:51 +0800 Subject: [PATCH 17/18] resolve comments by changing docstring --- src/sdk/pynni/nni/nas_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py index 42b3527e18..d26f4c9f44 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/src/sdk/pynni/nni/nas_utils.py @@ -29,7 +29,9 @@ def classic_mode( fixed_inputs, optional_inputs, optional_input_size): - '''Execute the chosen function and inputs directly''' + '''Execute the chosen function and inputs directly. + In this mode, the trial code is only running the chosen subgraph (i.e., the chosen ops and inputs), + without touching the full model graph.''' if trial._params is None: trial.get_next_parameter() mutable_block = trial.get_current_parameter(mutable_id) @@ -54,7 +56,7 @@ def enas_mode( tf): '''Build a tensorflow graph including all functions and optional_inputs using signal varaibles. So that we can use those signal variables to change the whole graph into different subgraphs - in the `reload_tensorflow_variables` function''' + in the `reload_tensorflow_variables` function.''' name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) # store namespace if 'name_space' not in globals(): @@ -124,7 +126,8 @@ def oneshot_mode( optional_inputs = tf.nn.dropout( optional_inputs, rate=rate, noise_shape=noise_shape) optional_inputs = [optional_inputs[idx] for idx in range(inputs_num)] - layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) for func_name, func in funcs.items()] + layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) + for func_name, func in funcs.items()] layer_out = tf.add_n(layer_outs) return layer_out From 8778910724b0e3d41e73799b4c545aa4facd2c16 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Tue, 25 Jun 2019 15:28:17 +0800 Subject: [PATCH 18/18] resolve comments --- src/sdk/pynni/nni/nas_utils.py | 13 ++++++++----- tools/nni_annotation/code_generator.py | 2 ++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/sdk/pynni/nni/nas_utils.py b/src/sdk/pynni/nni/nas_utils.py index d26f4c9f44..baca7cf30c 100644 --- a/src/sdk/pynni/nni/nas_utils.py +++ b/src/sdk/pynni/nni/nas_utils.py @@ -54,9 +54,10 @@ def enas_mode( optional_inputs, optional_input_size, tf): - '''Build a tensorflow graph including all functions and optional_inputs using signal varaibles. - So that we can use those signal variables to change the whole graph into different subgraphs - in the `reload_tensorflow_variables` function.''' + '''For enas mode, we build the full model graph in trial but only run a subgraph。 + This is implemented by masking inputs and branching ops. + Specifically, based on the received subgraph (through nni.get_next_parameter), + it can be known which inputs should be masked and which op should be executed.''' name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) # store namespace if 'name_space' not in globals(): @@ -113,8 +114,10 @@ def oneshot_mode( optional_inputs, optional_input_size, tf): - '''Execute all the functions using all the optional_inputs, where a dropout will be implemented - to optional_inputs.''' + '''Similar to enas mode, oneshot mode also builds the full model graph. + The difference is that oneshot mode does not receive subgraph. + Instead, it uses dropout to randomly dropout inputs and ops.''' + # NNI requires to get_next_parameter before report a result. But the parameter will not be used in this mode if trial._params is None: trial.get_next_parameter() optional_inputs = list(optional_inputs.values()) diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index 5a77c7be34..871308e5d4 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -322,6 +322,7 @@ def _visit_string(self, node): if string.startswith('@nni.get_next_parameter'): call_node = parse_annotation(string[1:]).value if call_node.args: + # it is used in enas mode as it needs to retrieve the next subgraph for training call_attr = ast.Attribute(value=ast.Name(id='nni', ctx=ast.Load()), attr='reload_tensorflow_variables', ctx=ast.Load()) return ast.Expr(value=ast.Call(func=call_attr, args=call_node.args, keywords=[])) @@ -377,6 +378,7 @@ def parse(code, nas_mode=None): if type(nodes[i]) is ast.ImportFrom and nodes[i].module == '__future__': last_future_import = i nodes.insert(last_future_import + 1, import_nni) + # enas and oneshot modes for tensorflow need tensorflow module, so we import it here if nas_mode in ['enas_mode', 'oneshot_mode']: import_tf = ast.Import(names=[ast.alias(name='tensorflow', asname=None)]) nodes.insert(last_future_import + 1, import_tf)