From 10d51b43b576a88ae39eec2e7d7d5dd1b7906865 Mon Sep 17 00:00:00 2001 From: xuehui Date: Mon, 8 Oct 2018 12:47:59 +0800 Subject: [PATCH 1/3] Update batch tuner (#158) * update readme in ga_squad * update readme * fix typo * Update README.md * Update README.md * Update README.md * update readme * update batch tuner --- src/nni_manager/rest_server/restValidationSchemas.ts | 2 +- src/sdk/pynni/nni/constants.py | 2 ++ tools/nnicmd/config_schema.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index 24b07836a7..a94f9f11b8 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -58,7 +58,7 @@ export namespace ValidationSchemas { searchSpace: joi.string().required(), maxExecDuration: joi.number().min(0).required(), tuner: joi.object({ - builtinTunerName: joi.string().valid('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC'), + builtinTunerName: joi.string().valid('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC', 'BatchTuner'), codeDir: joi.string(), classFileName: joi.string(), className: joi.string(), diff --git a/src/sdk/pynni/nni/constants.py b/src/sdk/pynni/nni/constants.py index cf611bebfd..0e4fff8663 100644 --- a/src/sdk/pynni/nni/constants.py +++ b/src/sdk/pynni/nni/constants.py @@ -24,6 +24,7 @@ 'Anneal': 'nni.hyperopt_tuner.hyperopt_tuner', 'Evolution': 'nni.evolution_tuner.evolution_tuner', 'SMAC': 'nni.smac_tuner.smac_tuner', + 'BatchTuner': 'nni.batch_tuner.batch_tuner', 'Medianstop': 'nni.medianstop_assessor.medianstop_assessor' } @@ -34,6 +35,7 @@ 'Anneal': 'HyperoptTuner', 'Evolution': 'EvolutionTuner', 'SMAC': 'SMACTuner', + 'BatchTuner': 'BatchTuner', 'Medianstop': 'MedianstopAssessor' } diff --git a/tools/nnicmd/config_schema.py b/tools/nnicmd/config_schema.py index d6299256bd..9eff5bfa66 100644 --- a/tools/nnicmd/config_schema.py +++ b/tools/nnicmd/config_schema.py @@ -31,7 +31,7 @@ Optional('searchSpacePath'): os.path.exists, 'useAnnotation': bool, 'tuner': Or({ - 'builtinTunerName': Or('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC'), + 'builtinTunerName': Or('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC', 'BatchTuner'), 'classArgs': { 'optimize_mode': Or('maximize', 'minimize'), Optional('speed'): int From 1c22c766a1ac5cdb06f969809e0698473ae5f354 Mon Sep 17 00:00:00 2001 From: xuehui Date: Mon, 8 Oct 2018 12:48:22 +0800 Subject: [PATCH 2/3] Quickly fix cascading search space bug in tuner (#156) * update readme in ga_squad * update readme * fix typo * Update README.md * Update README.md * Update README.md * update readme * quickly fix cascading searchspace bug in tuner --- src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py index 99b3006b0d..0f3ef7eb19 100644 --- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py +++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py @@ -127,7 +127,7 @@ def json2vals(in_x, vals, out_y, name=ROOT): json2vals(in_x[key], vals[key], out_y, name + '[%s]' % str(key)) elif isinstance(in_x, list): for i, temp in enumerate(in_x): - json2vals(i, vals[temp], out_y, name + '[%d]' % temp) + json2vals(temp, vals[i], out_y, name + '[%d]' % i) def _split_index(params): From 6ef6511736ccac0bce0f459d9c436b079174ed0d Mon Sep 17 00:00:00 2001 From: xuehui Date: Mon, 8 Oct 2018 13:47:39 +0800 Subject: [PATCH 3/3] Add iterative search space example (#119) * update readme in ga_squad * update readme * fix typo * Update README.md * Update README.md * Update README.md * update readme * add iterative search space example * update * update readme * change name --- .../mnist-cascading-search-space/config.yml | 20 +++ .../mnist-cascading-search-space/mnist.py | 164 ++++++++++++++++++ .../requirments.txt | 3 + .../mnist-cascading-search-space/sample.json | 1 + .../search_space.json | 62 +++++++ 5 files changed, 250 insertions(+) create mode 100644 examples/trials/mnist-cascading-search-space/config.yml create mode 100644 examples/trials/mnist-cascading-search-space/mnist.py create mode 100644 examples/trials/mnist-cascading-search-space/requirments.txt create mode 100644 examples/trials/mnist-cascading-search-space/sample.json create mode 100644 examples/trials/mnist-cascading-search-space/search_space.json diff --git a/examples/trials/mnist-cascading-search-space/config.yml b/examples/trials/mnist-cascading-search-space/config.yml new file mode 100644 index 0000000000..1c2a4643a4 --- /dev/null +++ b/examples/trials/mnist-cascading-search-space/config.yml @@ -0,0 +1,20 @@ +authorName: default +experimentName: mnist-cascading-search-space +trialConcurrency: 2 +maxExecDuration: 1h +maxTrialNum: 100 +#choice: local, remote +trainingServicePlatform: local +searchSpacePath: search_space.json +#choice: true, false +useAnnotation: false +tuner: + #choice: TPE, Random, Anneal, Evolution + builtinTunerName: TPE + classArgs: + #choice: maximize, minimize + optimize_mode: maximize +trial: + command: python3 mnist.py + codeDir: . + gpuNum: 0 diff --git a/examples/trials/mnist-cascading-search-space/mnist.py b/examples/trials/mnist-cascading-search-space/mnist.py new file mode 100644 index 0000000000..bd6dd35a5c --- /dev/null +++ b/examples/trials/mnist-cascading-search-space/mnist.py @@ -0,0 +1,164 @@ +''' +mnist.py is an example to show: how to use iterative search space to tune architecture network for mnist. +''' +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import codecs +import json +import logging +import math +import sys +import tempfile +import tensorflow as tf + +from tensorflow.examples.tutorials.mnist import input_data + +import nni + + +logger = logging.getLogger('mnist_cascading_search_space') +FLAGS = None + +class MnistNetwork(object): + def __init__(self, params, feature_size = 784): + config = [] + + for i in range(10): + config.append(params['layer'+str(i)]) + self.config = config + self.feature_size = feature_size + self.label_size = 10 + + + def is_expand_dim(self, input): + # input is a tensor + shape = len(input.get_shape().as_list()) + if shape < 4: + return True + return False + + + def is_flatten(self, input): + # input is a tensor + shape = len(input.get_shape().as_list()) + if shape > 2: + return True + return False + + + def get_layer(self, layer_config, input, in_height, in_width, id): + if layer_config[0] == 'Empty': + return input + + if self.is_expand_dim(input): + input = tf.reshape(input, [-1, in_height, in_width, 1]) + h, w = layer_config[1], layer_config[2] + + if layer_config[0] == 'Conv': + conv_filter = tf.Variable(tf.random_uniform([h, w, 1, 1]), name='id_%d_conv_%d_%d' % (id, h, w)) + return tf.nn.conv2d(input, filter=conv_filter, strides=[1, 1, 1, 1], padding='SAME') + if layer_config[0] == 'Max_pool': + return tf.nn.max_pool(input, ksize=[1, h, w, 1], strides=[1, 1, 1, 1], padding='SAME') + if layer_config[0] == 'Avg_pool': + return tf.nn.avg_pool(input, ksize=[1, h, w, 1], strides=[1, 1, 1, 1], padding='SAME') + + print('error:', layer_config) + raise Exception('%s layer is illegal'%layer_config[0]) + + + def build_network(self): + layer_configs = self.config + feature_size = 784 + + # define placeholder + self.x = tf.placeholder(tf.float32, [None, feature_size], name="input_x") + self.y = tf.placeholder(tf.int32, [None, self.label_size], name="input_y") + label_number = 10 + + # define network + input_layer = self.x + in_height = in_width = int(math.sqrt(feature_size)) + for i, layer_config in enumerate(layer_configs): + input_layer = tf.nn.relu(self.get_layer(layer_config, input_layer, in_height, in_width, i)) + + output_layer = input_layer + if self.is_flatten(output_layer): + output_layer = tf.contrib.layers.flatten(output_layer) # flatten + output_layer = tf.layers.dense(output_layer, label_number) + child_logit = tf.nn.softmax_cross_entropy_with_logits(logits=output_layer, labels=self.y) + child_loss = tf.reduce_mean(child_logit) + + self.train_step = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(child_loss) + child_accuracy = tf.equal(tf.argmax(output_layer, 1), tf.argmax(self.y, 1)) + self.accuracy = tf.reduce_mean(tf.cast(child_accuracy, "float")) # add a reduce_mean + +def main(params): + # Import data + mnist = input_data.read_data_sets(params['data_dir'], one_hot=True) + + # Create the model + # Build the graph for the deep net + mnist_network = MnistNetwork(params) + mnist_network.build_network() + print('build network done.') + + # Write log + graph_location = tempfile.mkdtemp() + #print('Saving graph to: %s' % graph_location) + train_writer = tf.summary.FileWriter(graph_location) + train_writer.add_graph(tf.get_default_graph()) + + test_acc = 0.0 + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + for i in range(params['batch_num']): + batch = mnist.train.next_batch(params['batch_size']) + mnist_network.train_step.run(feed_dict={mnist_network.x: batch[0], mnist_network.y: batch[1]}) + + if i % 100 == 0: + train_accuracy = mnist_network.accuracy.eval(feed_dict={ + mnist_network.x: batch[0], mnist_network.y: batch[1]}) + print('step %d, training accuracy %g' % (i, train_accuracy)) + + test_acc = mnist_network.accuracy.eval(feed_dict={ + mnist_network.x: mnist.test.images, mnist_network.y: mnist.test.labels}) + + nni.report_final_result(test_acc) + +def generate_defualt_params(): + params = {'data_dir': '/tmp/tensorflow/mnist/input_data', + 'batch_num': 1000, + 'batch_size': 200} + return params + + +def parse_init_json(data): + params = {} + for key in data: + value = data[key] + if value == 'Empty': + params[key] = ['Empty'] + else: + params[key] = [value[0], value[1]['_value'], value[1]['_value']] + return params + + +if __name__ == '__main__': + try: + # get parameters form tuner + data = nni.get_parameters() + logger.debug(data) + + RCV_PARAMS = parse_init_json(data) + logger.debug(RCV_PARAMS) + params = generate_defualt_params() + params.update(RCV_PARAMS) + print(RCV_PARAMS) + + main(params) + except Exception as exception: + logger.exception(exception) + raise diff --git a/examples/trials/mnist-cascading-search-space/requirments.txt b/examples/trials/mnist-cascading-search-space/requirments.txt new file mode 100644 index 0000000000..a77b0fa943 --- /dev/null +++ b/examples/trials/mnist-cascading-search-space/requirments.txt @@ -0,0 +1,3 @@ +tensorflow >= 1.3 +six == 1.11.0 +numpy == 1.13.3 diff --git a/examples/trials/mnist-cascading-search-space/sample.json b/examples/trials/mnist-cascading-search-space/sample.json new file mode 100644 index 0000000000..8781c2eeed --- /dev/null +++ b/examples/trials/mnist-cascading-search-space/sample.json @@ -0,0 +1 @@ +{"layer2": "Empty", "layer8": ["Conv", {"_index": 0, "_value": 2}], "layer3": ["Avg_pool", {"_index": 2, "_value": 5}], "layer0": ["Max_pool", {"_index": 2, "_value": 5}], "layer1": ["Conv", {"_index": 0, "_value": 2}], "layer6": ["Max_pool", {"_index": 1, "_value": 3}], "layer7": ["Max_pool", {"_index": 2, "_value": 5}], "layer9": ["Conv", {"_index": 0, "_value": 2}], "layer4": ["Avg_pool", {"_index": 1, "_value": 3}], "layer5": ["Avg_pool", {"_index": 2, "_value": 5}]} diff --git a/examples/trials/mnist-cascading-search-space/search_space.json b/examples/trials/mnist-cascading-search-space/search_space.json new file mode 100644 index 0000000000..caf9c36117 --- /dev/null +++ b/examples/trials/mnist-cascading-search-space/search_space.json @@ -0,0 +1,62 @@ +{ + "layer0":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer1":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer2":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer3":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer4":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer5":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer6":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer7":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer8":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]}, + "layer9":{"_type":"choice","_value":[ + "Empty", + ["Conv", {"_type":"choice","_value":[2,3,5]}], + ["Max_pool", {"_type":"choice","_value":[2,3,5]}], + ["Avg_pool", {"_type":"choice","_value":[2,3,5]}] + ]} +} \ No newline at end of file