From c5e6c6bed35fea8615661da7be1f354ab30b3d12 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Wed, 19 Sep 2018 13:53:25 +0800 Subject: [PATCH 1/7] support package install (#91) * fix nnictl bug * support package install * update * update package install logic --- tools/nnicmd/constants.py | 4 +++ tools/nnicmd/nnictl.py | 16 ++++++++++-- tools/nnicmd/package_management.py | 42 ++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 tools/nnicmd/package_management.py diff --git a/tools/nnicmd/constants.py b/tools/nnicmd/constants.py index 5f57da15f5..b03b1bdcbe 100644 --- a/tools/nnicmd/constants.py +++ b/tools/nnicmd/constants.py @@ -49,3 +49,7 @@ '4. nnictl trial kill kill a trial job by id\n' \ '5. nnictl --help get help information about nnictl\n' \ '6. nnictl webui url get the url of web ui' + +PACKAGE_REQUIREMENTS = { + 'SMAC': 'smac_tuner' +} diff --git a/tools/nnicmd/nnictl.py b/tools/nnicmd/nnictl.py index 73b2950a55..b677d10c6d 100644 --- a/tools/nnicmd/nnictl.py +++ b/tools/nnicmd/nnictl.py @@ -23,13 +23,14 @@ from .launcher import create_experiment, resume_experiment from .updater import update_searchspace, update_concurrency, update_duration from .nnictl_utils import * +from .package_management import * def nni_help_info(*args): print('please run "nnictl --help" to see nnictl guidance') def parse_args(): '''Definite the arguments users need to follow and input''' - parser = argparse.ArgumentParser(prog='nni ctl', description='use nni control') + parser = argparse.ArgumentParser(prog='nnictl', description='use nnictl command to control nni experiments') parser.set_defaults(func=nni_help_info) # create subparsers for args with sub values @@ -104,7 +105,7 @@ def parse_args(): #parse log command parser_log = subparsers.add_parser('log', help='get log information') - # add subparsers for parser_rest + # add subparsers for parser_log parser_log_subparsers = parser_log.add_subparsers() parser_log_stdout = parser_log_subparsers.add_parser('stdout', help='get stdout information') parser_log_stdout.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stdout') @@ -117,6 +118,17 @@ def parse_args(): parser_log_stderr.add_argument('--path', '-p', action='store_true', default=False, help='get the path of stderr file') parser_log_stderr.set_defaults(func=log_stderr) + #parse package command + parser_package = subparsers.add_parser('package', help='control nni tuner and assessor packages') + # add subparsers for parser_package + parser_package_subparsers = parser_package.add_subparsers() + parser_package_install = parser_package_subparsers.add_parser('install', help='install packages') + parser_package_install.add_argument('--name', '-n', dest='name', help='package name to be installed') + parser_package_install.set_defaults(func=package_install) + parser_package_show = parser_package_subparsers.add_parser('show', help='show the information of packages') + parser_package_show.set_defaults(func=package_show) + + args = parser.parse_args() args.func(args) diff --git a/tools/nnicmd/package_management.py b/tools/nnicmd/package_management.py new file mode 100644 index 0000000000..bb9b96b705 --- /dev/null +++ b/tools/nnicmd/package_management.py @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import nni +import os +from subprocess import call +from .constants import PACKAGE_REQUIREMENTS +from .common_utils import print_normal, print_error + +def process_install(package_name): + if PACKAGE_REQUIREMENTS.get(package_name) is None: + print_error('{0} is not supported!' % package_name) + else: + requirements_path = os.path.join(nni.__path__[0], PACKAGE_REQUIREMENTS[package_name], 'requirements.txt') + cmds = ['python3', '-m', 'pip', 'install', '--user', requirements_path] + call(cmds) + +def package_install(args): + '''install packages''' + process_install(args.name) + +def package_show(args): + '''show all packages''' + print(' '.join(PACKAGE_REQUIREMENTS.keys())) + \ No newline at end of file From 44be8996dc360b793ea6c68a509f7aeaa631078d Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Wed, 19 Sep 2018 16:02:50 +0800 Subject: [PATCH 2/7] Fix package install issue (#95) * fix nnictl bug * fix pakcage install --- tools/nnicmd/package_management.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/nnicmd/package_management.py b/tools/nnicmd/package_management.py index bb9b96b705..86f9007d8d 100644 --- a/tools/nnicmd/package_management.py +++ b/tools/nnicmd/package_management.py @@ -28,9 +28,9 @@ def process_install(package_name): if PACKAGE_REQUIREMENTS.get(package_name) is None: print_error('{0} is not supported!' % package_name) else: - requirements_path = os.path.join(nni.__path__[0], PACKAGE_REQUIREMENTS[package_name], 'requirements.txt') - cmds = ['python3', '-m', 'pip', 'install', '--user', requirements_path] - call(cmds) + requirements_path = os.path.join(nni.__path__[0], PACKAGE_REQUIREMENTS[package_name]) + cmds = 'cd ' + requirements_path + ' && python3 -m pip install --user -r requirements.txt' + call(cmds, shell=True) def package_install(args): '''install packages''' From 5ef618cb3a0cf0032f0168b82594e4c3e14d3f88 Mon Sep 17 00:00:00 2001 From: QuanluZhang Date: Fri, 21 Sep 2018 09:47:42 +0800 Subject: [PATCH 3/7] support SMAC as a tuner on nni (#81) * update doc * update doc * update doc * update hyperopt installation * update doc * update doc * update description in setup.py * update setup.py * modify encoding * encoding * add encoding * remove pymc3 * update doc * update builtin tuner spec * support smac in sdk, fix logging issue * support smac tuner * add optimize_mode * update config in nnictl * add __init__.py * update smac * update import path * update setup.py: remove entry_point * update rest server validation * fix bug in nnictl launcher * support classArgs: optimize_mode * quick fix bug * test travis * add dependency * add dependency * add dependency * add dependency * create smac python package * fix trivial points * optimize import of tuners, modify nnictl accordingly * fix bug: incorrect algorithm_name * trivial refactor * for debug * support virtual * update doc of SMAC * update smac requirements * update requirements * change debug mode --- setup.py | 8 +- .../rest_server/restValidationSchemas.ts | 2 +- src/sdk/pynni/nni/README.md | 32 ++- src/sdk/pynni/nni/__main__.py | 67 ++++-- src/sdk/pynni/nni/common.py | 22 +- src/sdk/pynni/nni/constants.py | 51 +++++ src/sdk/pynni/nni/smac_tuner/README.md | 1 + src/sdk/pynni/nni/smac_tuner/__init__.py | 0 .../nni/smac_tuner/convert_ss_to_scenario.py | 119 +++++++++++ src/sdk/pynni/nni/smac_tuner/requirements.txt | 2 + src/sdk/pynni/nni/smac_tuner/smac_tuner.py | 190 ++++++++++++++++++ src/sdk/pynni/setup.py | 4 +- tools/nnicmd/config_schema.py | 4 +- tools/nnicmd/launcher_utils.py | 38 ++-- 14 files changed, 465 insertions(+), 75 deletions(-) create mode 100644 src/sdk/pynni/nni/constants.py create mode 100644 src/sdk/pynni/nni/smac_tuner/README.md create mode 100644 src/sdk/pynni/nni/smac_tuner/__init__.py create mode 100644 src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py create mode 100644 src/sdk/pynni/nni/smac_tuner/requirements.txt create mode 100644 src/sdk/pynni/nni/smac_tuner/smac_tuner.py diff --git a/setup.py b/setup.py index eeee54d075..25997c78c8 100644 --- a/setup.py +++ b/setup.py @@ -81,16 +81,10 @@ def run(self): 'pyyaml', 'requests', 'scipy', - 'schema' - ], - dependency_links = [ - 'git+https://github.com/hyperopt/hyperopt.git' + 'schema' ], cmdclass={ 'install': CustomInstallCommand - }, - entry_points={ - 'console_scripts': ['nnictl = nnicmd.nnictl:parse_args'] } ) diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index 218a8c22c4..6c172349c6 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -48,7 +48,7 @@ export namespace ValidationSchemas { searchSpace: joi.string().required(), maxExecDuration: joi.number().min(0).required(), tuner: joi.object({ - builtinTunerName: joi.string().valid('TPE', 'Random', 'Anneal', 'Evolution'), + builtinTunerName: joi.string().valid('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC'), codeDir: joi.string(), classFileName: joi.string(), className: joi.string(), diff --git a/src/sdk/pynni/nni/README.md b/src/sdk/pynni/nni/README.md index 8c0c41278c..e26b4c3090 100644 --- a/src/sdk/pynni/nni/README.md +++ b/src/sdk/pynni/nni/README.md @@ -1,20 +1,21 @@ -# How to use Tuner that NNI support? +# How to use Tuner that NNI supports? -For now, NNI could support tuner algorithm as following: +For now, NNI could support tuner algorithms as following: - TPE - Random Search - Anneal - Naive Evolution - - ENAS (on going) + - SMAC + - ENAS (ongoing) + - Grid Search (ongoing) - - **1. Tuner algorithm introduction** + ## 1. Tuner algorithm introduction We will introduce some basic knowledge about tuner algorithm here. If you are an expert, you could skip this part and jump to how to use. -*1.1 TPE* +**TPE** The Tree-structured Parzen Estimator (TPE) is a sequential model-based optimization (SMBO) approach. SMBO methods sequentially construct models to approximate the performance of hyperparameters based on historical measurements, and then subsequently choose new hyperparameters to test based on this model. @@ -22,20 +23,27 @@ The TPE approach models P(x|y) and P(y) where x represents hyperparameters and y Comparing with other algorithm, TPE could be achieve better result when the number of trial experiment is small. Also TPE support continuous or discrete hyper-parameters. From a large amount of experiments, we could found that TPE is far better than Random Search. -*1.2 Random Search* +**Random Search** In [Random Search for Hyper-Parameter Optimization][2] show that Random Search might be surprsingly simple and effective. We suggests that we could use Random Search as basline when we have no knowledge about the prior distribution of hyper-parameters. -*1.3 Anneal* +**Anneal** -*1.4 Naive Evolution* +**Naive Evolution** + Naive Evolution comes from [Large-Scale Evolution of Image Classifiers][3]. Naive Evolution requir more experiments to works, but it's very simple and easily to expand new features. There are some tips for user: 1) large initial population could avoid to fall into local optimum 2) use some strategies to keep the deversity of population could be better. +**SMAC** + +[SMAC][4] is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by nni is a wrapper on [the SMAC3 github repo][5]. + +Note that SMAC only supports a subset of the types in [search space spec](../../../../docs/SearchSpaceSpec.md), including `choice`, `randint`, `uniform`, `loguniform`, `quniform(q=1)`. + - **2. How to use the tuner algorithm in NNI?** + ## 2. How to use the tuner algorithm in NNI? User only need to do one thing: choose a Tuner```config.yaml```. Here is an example: @@ -61,4 +69,6 @@ There are two filed you need to set: [1]: https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf [2]: http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf - [3]: https://arxiv.org/pdf/1703.01041.pdf \ No newline at end of file + [3]: https://arxiv.org/pdf/1703.01041.pdf + [4]: https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf + [5]: https://github.com/automl/SMAC3 \ No newline at end of file diff --git a/src/sdk/pynni/nni/__main__.py b/src/sdk/pynni/nni/__main__.py index e3a39bac96..b7caf8671b 100644 --- a/src/sdk/pynni/nni/__main__.py +++ b/src/sdk/pynni/nni/__main__.py @@ -27,28 +27,40 @@ import json import importlib +from .constants import ModuleName, ClassName, ClassArgs from nni.msg_dispatcher import MsgDispatcher -from nni.hyperopt_tuner.hyperopt_tuner import HyperoptTuner -from nni.evolution_tuner.evolution_tuner import EvolutionTuner -from nni.batch_tuner.batch_tuner import BatchTuner -from nni.medianstop_assessor.medianstop_assessor import MedianstopAssessor logger = logging.getLogger('nni.main') logger.debug('START') -BUILT_IN_CLASS_NAMES = ['HyperoptTuner', 'EvolutionTuner', 'BatchTuner', 'MedianstopAssessor'] +def augment_classargs(input_class_args, classname): + if classname in ClassArgs: + for key, value in ClassArgs[classname].items(): + if key not in input_class_args: + input_class_args[key] = value + return input_class_args def create_builtin_class_instance(classname, jsonstr_args): + if classname not in ModuleName or \ + importlib.util.find_spec(ModuleName[classname]) is None: + raise RuntimeError('Tuner module is not found: {}'.format(classname)) + class_module = importlib.import_module(ModuleName[classname]) + class_constructor = getattr(class_module, ClassName[classname]) if jsonstr_args: class_args = json.loads(jsonstr_args) - instance = eval(classname)(**class_args) + class_args = augment_classargs(class_args, classname) else: - instance = eval(classname)() + class_args = augment_classargs({}, classname) + if class_args: + instance = class_constructor(**class_args) + else: + instance = class_constructor() return instance def create_customized_class_instance(class_dir, class_filename, classname, jsonstr_args): if not os.path.isfile(os.path.join(class_dir, class_filename)): - raise ValueError('Class file not found: {}'.format(os.path.join(class_dir, class_filename))) + raise ValueError('Class file not found: {}'.format( + os.path.join(class_dir, class_filename))) sys.path.append(class_dir) module_name = class_filename.split('.')[0] class_module = importlib.import_module(module_name) @@ -64,12 +76,12 @@ def parse_args(): parser = argparse.ArgumentParser(description='parse command line parameters.') parser.add_argument('--tuner_class_name', type=str, required=True, help='Tuner class name, the class must be a subclass of nni.Tuner') + parser.add_argument('--tuner_class_filename', type=str, required=False, + help='Tuner class file path') parser.add_argument('--tuner_args', type=str, required=False, help='Parameters pass to tuner __init__ constructor') parser.add_argument('--tuner_directory', type=str, required=False, help='Tuner directory') - parser.add_argument('--tuner_class_filename', type=str, required=False, - help='Tuner class file path') parser.add_argument('--assessor_class_name', type=str, required=False, help='Assessor class name, the class must be a subclass of nni.Assessor') @@ -93,23 +105,34 @@ def main(): tuner = None assessor = None - if args.tuner_class_name is None: - raise ValueError('Tuner must be specified') - if args.tuner_class_name in BUILT_IN_CLASS_NAMES: - tuner = create_builtin_class_instance(args.tuner_class_name, args.tuner_args) + if args.tuner_class_name in ModuleName: + tuner = create_builtin_class_instance( + args.tuner_class_name, + args.tuner_args) else: - tuner = create_customized_class_instance(args.tuner_directory, args.tuner_class_filename, args.tuner_class_name, args.tuner_args) - - if args.assessor_class_name: - if args.assessor_class_name in BUILT_IN_CLASS_NAMES: - assessor = create_builtin_class_instance(args.assessor_class_name, args.assessor_args) - else: - assessor = create_customized_class_instance(args.assessor_directory, \ - args.assessor_class_filename, args.assessor_class_name, args.assessor_args) + tuner = create_customized_class_instance( + args.tuner_directory, + args.tuner_class_filename, + args.tuner_class_name, + args.tuner_args) if tuner is None: raise AssertionError('Failed to create Tuner instance') + if args.assessor_class_name: + if args.assessor_class_name in ModuleName: + assessor = create_builtin_class_instance( + args.assessor_class_name, + args.assessor_args) + else: + assessor = create_customized_class_instance( + args.assessor_directory, + args.assessor_class_filename, + args.assessor_class_name, + args.assessor_args) + if assessor is None: + raise AssertionError('Failed to create Assessor instance') + dispatcher = MsgDispatcher(tuner, assessor) try: diff --git a/src/sdk/pynni/nni/common.py b/src/sdk/pynni/nni/common.py index 644f13d15b..79ee214aa2 100644 --- a/src/sdk/pynni/nni/common.py +++ b/src/sdk/pynni/nni/common.py @@ -20,6 +20,7 @@ from collections import namedtuple +from datetime import datetime from io import TextIOBase import logging import os @@ -39,13 +40,16 @@ def _load_env_args(): '''Arguments passed from environment''' -class _LoggerFile(TextIOBase): - def __init__(self, logger): - self.logger = logger +_time_format = '%Y-%m-%d %H:%M:%S' +class _LoggerFileWrapper(TextIOBase): + def __init__(self, logger_file): + self.file = logger_file def write(self, s): - if s != '\n': # ignore line break, since logger will add it - self.logger.info(s) + if s != '\n': + time = datetime.now().strftime(_time_format) + self.file.write('[{}] PRINT '.format(time) + s + '\n') + self.file.flush() return len(s) @@ -58,12 +62,12 @@ def init_logger(logger_file_path): logger_file_path = 'unittest.log' elif env_args.log_dir is not None: logger_file_path = os.path.join(env_args.log_dir, logger_file_path) + logger_file = open(logger_file_path, 'w') fmt = '[%(asctime)s] %(levelname)s (%(name)s) %(message)s' - datefmt = '%Y-%m-%d %H:%M:%S' - formatter = logging.Formatter(fmt, datefmt) + formatter = logging.Formatter(fmt, _time_format) - handler = logging.FileHandler(logger_file_path) + handler = logging.StreamHandler(logger_file) handler.setFormatter(formatter) root_logger = logging.getLogger() @@ -73,4 +77,4 @@ def init_logger(logger_file_path): # these modules are too verbose logging.getLogger('matplotlib').setLevel(logging.INFO) - sys.stdout = _LoggerFile(logging.getLogger('print')) + sys.stdout = _LoggerFileWrapper(logger_file) diff --git a/src/sdk/pynni/nni/constants.py b/src/sdk/pynni/nni/constants.py new file mode 100644 index 0000000000..cf611bebfd --- /dev/null +++ b/src/sdk/pynni/nni/constants.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +ModuleName = { + 'TPE': 'nni.hyperopt_tuner.hyperopt_tuner', + 'Random': 'nni.hyperopt_tuner.hyperopt_tuner', + 'Anneal': 'nni.hyperopt_tuner.hyperopt_tuner', + 'Evolution': 'nni.evolution_tuner.evolution_tuner', + 'SMAC': 'nni.smac_tuner.smac_tuner', + + 'Medianstop': 'nni.medianstop_assessor.medianstop_assessor' +} + +ClassName = { + 'TPE': 'HyperoptTuner', + 'Random': 'HyperoptTuner', + 'Anneal': 'HyperoptTuner', + 'Evolution': 'EvolutionTuner', + 'SMAC': 'SMACTuner', + + 'Medianstop': 'MedianstopAssessor' +} + +ClassArgs = { + 'TPE': { + 'algorithm_name': 'tpe' + }, + 'Random': { + 'algorithm_name': 'random_search' + }, + 'Anneal': { + 'algorithm_name': 'anneal' + } +} diff --git a/src/sdk/pynni/nni/smac_tuner/README.md b/src/sdk/pynni/nni/smac_tuner/README.md new file mode 100644 index 0000000000..a1a8b37190 --- /dev/null +++ b/src/sdk/pynni/nni/smac_tuner/README.md @@ -0,0 +1 @@ +# Integration doc: SMAC on nni \ No newline at end of file diff --git a/src/sdk/pynni/nni/smac_tuner/__init__.py b/src/sdk/pynni/nni/smac_tuner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py b/src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py new file mode 100644 index 0000000000..ab2523a7a5 --- /dev/null +++ b/src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py @@ -0,0 +1,119 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import json + +def get_json_content(file_path): + '''Load json file content''' + try: + with open(file_path, 'r') as file: + return json.load(file) + except TypeError as err: + print('Error: ', err) + return None + +def generate_pcs(nni_search_space_content): + # parameter_name categorical {value_1, ..., value_N} [default value] + # parameter_name ordinal {value_1, ..., value_N} [default value] + # parameter_name integer [min_value, max_value] [default value] + # parameter_name integer [min_value, max_value] [default value] log + # parameter_name real [min_value, max_value] [default value] + # parameter_name real [min_value, max_value] [default value] log + # https://automl.github.io/SMAC3/stable/options.html + search_space = nni_search_space_content + with open('param_config_space.pcs', 'w') as pcs_fd: + if isinstance(search_space, dict): + for key in search_space.keys(): + if isinstance(search_space[key], dict): + try: + if search_space[key]['_type'] == 'choice': + pcs_fd.write('%s categorical {%s} [%s]\n' % ( + key, + str(search_space[key]['_value'])[1:-1], + search_space[key]['_value'][0])) + elif search_space[key]['_type'] == 'randint': + # TODO: support lower bound in randint + pcs_fd.write('%s integer [0, %d] [%d]\n' % ( + key, + search_space[key]['_value'][0], + search_space[key]['_value'][0])) + elif search_space[key]['_type'] == 'uniform': + pcs_fd.write('%s real %s [%f]\n' % ( + key, + search_space[key]['_value'], + search_space[key]['_value'][0])) + elif search_space[key]['_type'] == 'loguniform': + pcs_fd.write('%s real %s [%f] log\n' % ( + key, + search_space[key]['_value'], + search_space[key]['_value'][0])) + elif search_space[key]['_type'] == 'quniform' \ + and search_space[key]['_value'][2] == 1: + pcs_fd.write('%s integer [%d, %d] [%d]\n' % ( + key, + search_space[key]['_value'][0], + search_space[key]['_value'][1], + search_space[key]['_value'][0])) + else: + raise RuntimeError('unsupported _type %s' % search_space[key]['_type']) + except: + raise RuntimeError('_type or _value error.') + else: + raise RuntimeError('incorrect search space.') + +def generate_scenario(ss_content): + # deterministic, 1/0 + # output_dir, + # paramfile, + # run_obj, 'quality' + sce_fd = open('scenario.txt', 'w') + sce_fd.write('deterministic = 0\n') + #sce_fd.write('output_dir = \n') + sce_fd.write('paramfile = param_config_space.pcs\n') + sce_fd.write('run_obj = quality\n') + sce_fd.close() + + generate_pcs(ss_content) + + # the following keys use default value or empty + # algo, not required by tuner, but required by nni's training service for running trials + # abort_on_first_run_crash, because trials reported to nni tuner would always in success state + # always_race_default, + # cost_for_crash, trials reported to nni tuner would always in success state + # cutoff_time, + # execdir, trials are executed by nni's training service + # feature_file, no features specified or feature file is not supported + # initial_incumbent, use default value + # input_psmac_dirs, parallelism is supported by nni + # instance_file, not supported + # intensification_percentage, not supported, trials are controlled by nni's training service and kill be assessor + # maxR, use default, 2000 + # minR, use default, 1 + # overall_obj, timeout is not supported + # shared_model, parallelism is supported by nni + # test_instance_file, instance is not supported + # tuner-timeout, not supported + # runcount_limit, default: inf., use default because this is controlled by nni + # wallclock_limit,default: inf., use default because this is controlled by nni + # please refer to https://automl.github.io/SMAC3/stable/options.html + +if __name__ == '__main__': + generate_scenario('search_space.json') diff --git a/src/sdk/pynni/nni/smac_tuner/requirements.txt b/src/sdk/pynni/nni/smac_tuner/requirements.txt new file mode 100644 index 0000000000..a3027fb6fe --- /dev/null +++ b/src/sdk/pynni/nni/smac_tuner/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/QuanluZhang/ConfigSpace.git +git+https://github.com/QuanluZhang/SMAC3.git diff --git a/src/sdk/pynni/nni/smac_tuner/smac_tuner.py b/src/sdk/pynni/nni/smac_tuner/smac_tuner.py new file mode 100644 index 0000000000..87b9b977ba --- /dev/null +++ b/src/sdk/pynni/nni/smac_tuner/smac_tuner.py @@ -0,0 +1,190 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, +# to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +''' +smac_tuner.py +''' + +from nni.tuner import Tuner + +import sys +import logging +import numpy as np +import json_tricks +from enum import Enum, unique +from .convert_ss_to_scenario import generate_scenario + +from smac.utils.io.cmd_reader import CMDReader +from smac.scenario.scenario import Scenario +from smac.facade.smac_facade import SMAC +from smac.facade.roar_facade import ROAR +from smac.facade.epils_facade import EPILS + +@unique +class OptimizeMode(Enum): + ''' + Oprimize Mode class + ''' + Minimize = 'minimize' + Maximize = 'maximize' + +class SMACTuner(Tuner): + def __init__(self, optimize_mode): + ''' + Constructor + ''' + self.logger = logging.getLogger( + self.__module__ + "." + self.__class__.__name__) + self.optimize_mode = OptimizeMode(optimize_mode) + self.total_data = {} + self.optimizer = None + self.smbo_solver = None + self.first_one = True + self.update_ss_done = False + + def _main_cli(self): + ''' + Main function of SMAC for CLI interface + ''' + self.logger.info("SMAC call: %s" % (" ".join(sys.argv))) + + cmd_reader = CMDReader() + args_, _ = cmd_reader.read_cmd() + + root_logger = logging.getLogger() + root_logger.setLevel(args_.verbose_level) + logger_handler = logging.StreamHandler( + stream=sys.stdout) + if root_logger.level >= logging.INFO: + formatter = logging.Formatter( + "%(levelname)s:\t%(message)s") + else: + formatter = logging.Formatter( + "%(asctime)s:%(levelname)s:%(name)s:%(message)s", + "%Y-%m-%d %H:%M:%S") + logger_handler.setFormatter(formatter) + root_logger.addHandler(logger_handler) + # remove default handler + root_logger.removeHandler(root_logger.handlers[0]) + + # Create defaults + rh = None + initial_configs = None + stats = None + incumbent = None + + # Create scenario-object + scen = Scenario(args_.scenario_file, []) + + if args_.mode == "SMAC": + optimizer = SMAC( + scenario=scen, + rng=np.random.RandomState(args_.seed), + runhistory=rh, + initial_configurations=initial_configs, + stats=stats, + restore_incumbent=incumbent, + run_id=args_.seed) + elif args_.mode == "ROAR": + optimizer = ROAR( + scenario=scen, + rng=np.random.RandomState(args_.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args_.seed) + elif args_.mode == "EPILS": + optimizer = EPILS( + scenario=scen, + rng=np.random.RandomState(args_.seed), + runhistory=rh, + initial_configurations=initial_configs, + run_id=args_.seed) + else: + optimizer = None + + return optimizer + + def update_search_space(self, search_space): + ''' + TODO: this is urgly, we put all the initialization work in this method, + because initialization relies on search space, also because update_search_space is called at the beginning. + NOTE: updating search space is not supported. + ''' + if not self.update_ss_done: + generate_scenario(search_space) + self.optimizer = self._main_cli() + self.smbo_solver = self.optimizer.solver + self.update_ss_done = True + else: + pass + + def receive_trial_result(self, parameter_id, parameters, reward): + ''' + receive_trial_result + ''' + if self.optimize_mode is OptimizeMode.Maximize: + reward = -reward + + if parameter_id not in self.total_data: + raise RuntimeError('Received parameter_id not in total_data.') + if self.first_one: + self.smbo_solver.nni_smac_receive_first_run(self.total_data[parameter_id], reward) + self.first_one = False + else: + self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward) + + def generate_parameters(self, parameter_id): + ''' + generate one instance of hyperparameters + ''' + if self.first_one: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[parameter_id] = init_challenger + json_tricks.dumps(init_challenger.get_dictionary()) + return init_challenger.get_dictionary() + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + for challenger in challengers: + self.total_data[parameter_id] = challenger + json_tricks.dumps(challenger.get_dictionary()) + return challenger.get_dictionary() + + def generate_multiple_parameters(self, parameter_id_list): + ''' + generate mutiple instances of hyperparameters + ''' + if self.first_one: + params = [] + for one_id in parameter_id_list: + init_challenger = self.smbo_solver.nni_smac_start() + self.total_data[one_id] = init_challenger + json_tricks.dumps(init_challenger.get_dictionary()) + params.append(init_challenger.get_dictionary()) + else: + challengers = self.smbo_solver.nni_smac_request_challengers() + cnt = 0 + params = [] + for challenger in challengers: + if cnt >= len(parameter_id_list): + break + self.total_data[parameter_id_list[cnt]] = challenger + json_tricks.dumps(challenger.get_dictionary()) + params.append(challenger.get_dictionary()) + cnt += 1 + return params diff --git a/src/sdk/pynni/setup.py b/src/sdk/pynni/setup.py index a24758e9db..fee463e371 100644 --- a/src/sdk/pynni/setup.py +++ b/src/sdk/pynni/setup.py @@ -35,7 +35,7 @@ def read(fname): 'hyperopt', 'json_tricks', 'numpy', - 'scipy', + 'scipy' ], test_suite = 'tests', @@ -43,7 +43,7 @@ def read(fname): author = 'Microsoft NNI Team', author_email = 'nni@microsoft.com', description = 'Python SDK for Neural Network Intelligence project', - license = 'MIT', + license = 'MIT', url = 'https://msrasrg.visualstudio.com/NeuralNetworkIntelligence', long_description = read('README.md') diff --git a/tools/nnicmd/config_schema.py b/tools/nnicmd/config_schema.py index 8cd8431151..aec020d181 100644 --- a/tools/nnicmd/config_schema.py +++ b/tools/nnicmd/config_schema.py @@ -31,7 +31,7 @@ Optional('searchSpacePath'): os.path.exists, 'useAnnotation': bool, 'tuner': Or({ - 'builtinTunerName': Or('TPE', 'Random', 'Anneal', 'Evolution'), + 'builtinTunerName': Or('TPE', 'Random', 'Anneal', 'Evolution', 'SMAC'), 'classArgs': { 'optimize_mode': Or('maximize', 'minimize'), Optional('speed'): int @@ -109,4 +109,4 @@ "gpuType": str, "retryCount": And(int, lambda x: 0 <= x <= 99999) } -}) \ No newline at end of file +}) diff --git a/tools/nnicmd/launcher_utils.py b/tools/nnicmd/launcher_utils.py index dc97910244..28d3bc341c 100644 --- a/tools/nnicmd/launcher_utils.py +++ b/tools/nnicmd/launcher_utils.py @@ -21,6 +21,7 @@ import os import json from .config_schema import CONFIG_SCHEMA +from .common_utils import get_json_content def expand_path(experiment_config, key): '''Change '~' to user home directory''' @@ -87,34 +88,29 @@ def validate_common_content(experiment_config): def parse_tuner_content(experiment_config): '''Validate whether tuner in experiment_config is valid''' - tuner_class_name_dict = {'TPE': 'HyperoptTuner',\ - 'Random': 'HyperoptTuner',\ - 'Anneal': 'HyperoptTuner',\ - 'Evolution': 'EvolutionTuner',\ - 'BatchTuner': 'BatchTuner'} - - tuner_algorithm_name_dict = {'TPE': 'tpe',\ - 'Random': 'random_search',\ - 'Anneal': 'anneal'} - - if experiment_config['tuner'].get('builtinTunerName') and experiment_config['tuner'].get('classArgs'): - experiment_config['tuner']['className'] = tuner_class_name_dict.get(experiment_config['tuner']['builtinTunerName']) - experiment_config['tuner']['classArgs']['algorithm_name'] = tuner_algorithm_name_dict.get(experiment_config['tuner']['builtinTunerName']) - elif experiment_config['tuner'].get('codeDir') and experiment_config['tuner'].get('classFileName') and experiment_config['tuner'].get('className'): - if not os.path.exists(os.path.join(experiment_config['tuner']['codeDir'], experiment_config['tuner']['classFileName'])): + if experiment_config['tuner'].get('builtinTunerName'): + experiment_config['tuner']['className'] = experiment_config['tuner']['builtinTunerName'] + elif experiment_config['tuner'].get('codeDir') and \ + experiment_config['tuner'].get('classFileName') and \ + experiment_config['tuner'].get('className'): + if not os.path.exists(os.path.join( + experiment_config['tuner']['codeDir'], + experiment_config['tuner']['classFileName'])): raise ValueError('Tuner file directory is not valid!') else: raise ValueError('Tuner format is not valid!') def parse_assessor_content(experiment_config): '''Validate whether assessor in experiment_config is valid''' - assessor_class_name_dict = {'Medianstop': 'MedianstopAssessor'} - if experiment_config.get('assessor'): - if experiment_config['assessor'].get('builtinAssessorName') and experiment_config['assessor'].get('classArgs'): - experiment_config['assessor']['className'] = assessor_class_name_dict.get(experiment_config['assessor']['builtinAssessorName']) - elif experiment_config['assessor'].get('codeDir') and experiment_config['assessor'].get('classFileName') and experiment_config['assessor'].get('className') and experiment_config['assessor'].get('classArgs'): - if not os.path.exists(os.path.join(experiment_config['assessor']['codeDir'], experiment_config['assessor']['classFileName'])): + if experiment_config['assessor'].get('builtinAssessorName'): + experiment_config['assessor']['className'] = experiment_config['assessor']['builtinAssessorName'] + elif experiment_config['assessor'].get('codeDir') and \ + experiment_config['assessor'].get('classFileName') and \ + experiment_config['assessor'].get('className'): + if not os.path.exists(os.path.join( + experiment_config['assessor']['codeDir'], + experiment_config['assessor']['classFileName'])): raise ValueError('Assessor file directory is not valid!') else: raise ValueError('Assessor format is not valid!') From 8cbe98cc37ea36405837d9e84c0744dcc4c7b54e Mon Sep 17 00:00:00 2001 From: quzha Date: Tue, 25 Sep 2018 11:29:15 +0800 Subject: [PATCH 4/7] update doc --- docs/SearchSpaceSpec.md | 2 ++ src/sdk/pynni/nni/README.md | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/SearchSpaceSpec.md b/docs/SearchSpaceSpec.md index c26d79b376..836363891c 100644 --- a/docs/SearchSpaceSpec.md +++ b/docs/SearchSpaceSpec.md @@ -66,3 +66,5 @@ The candidate type and value for variable is here: * Which means the variable value is a value like round(exp(normal(mu, sigma)) / q) * q * Suitable for a discrete variable with respect to which the objective is smooth and gets smoother with the size of the variable, which is bounded from one side.
+ +Note that SMAC only supports a subset of the types above, including `choice`, `randint`, `uniform`, `loguniform`, `quniform(q=1)`. \ No newline at end of file diff --git a/src/sdk/pynni/nni/README.md b/src/sdk/pynni/nni/README.md index e26b4c3090..1fbb12a1b6 100644 --- a/src/sdk/pynni/nni/README.md +++ b/src/sdk/pynni/nni/README.md @@ -8,7 +8,7 @@ For now, NNI could support tuner algorithms as following: - Naive Evolution - SMAC - ENAS (ongoing) - - Grid Search (ongoing) + - Batch (ongoing) ## 1. Tuner algorithm introduction @@ -42,6 +42,10 @@ Naive Evolution comes from [Large-Scale Evolution of Image Classifiers][3]. Naiv Note that SMAC only supports a subset of the types in [search space spec](../../../../docs/SearchSpaceSpec.md), including `choice`, `randint`, `uniform`, `loguniform`, `quniform(q=1)`. +**Batch** + +Batch allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. + ## 2. How to use the tuner algorithm in NNI? From bdc1664f20834cb8faee2d42d5a66cb1cca5f040 Mon Sep 17 00:00:00 2001 From: quzha Date: Tue, 25 Sep 2018 11:34:53 +0800 Subject: [PATCH 5/7] update doc --- docs/SearchSpaceSpec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/SearchSpaceSpec.md b/docs/SearchSpaceSpec.md index 836363891c..ad59e36f16 100644 --- a/docs/SearchSpaceSpec.md +++ b/docs/SearchSpaceSpec.md @@ -67,4 +67,4 @@ The candidate type and value for variable is here: * Suitable for a discrete variable with respect to which the objective is smooth and gets smoother with the size of the variable, which is bounded from one side.
-Note that SMAC only supports a subset of the types above, including `choice`, `randint`, `uniform`, `loguniform`, `quniform(q=1)`. \ No newline at end of file +Note that SMAC only supports a subset of the types above, including `choice`, `randint`, `uniform`, `loguniform`, `quniform(q=1)`. In the current version, SMAC does not support cascaded search space (i.e., conditional variable in SMAC). \ No newline at end of file From 05ad032be3950ba8ef505f59e762364057c1facf Mon Sep 17 00:00:00 2001 From: quzha Date: Tue, 25 Sep 2018 14:02:00 +0800 Subject: [PATCH 6/7] refactor based on comments --- src/sdk/pynni/nni/__main__.py | 3 +-- src/sdk/pynni/nni/smac_tuner/smac_tuner.py | 26 +++++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/sdk/pynni/nni/__main__.py b/src/sdk/pynni/nni/__main__.py index b7caf8671b..206cd1f5c1 100644 --- a/src/sdk/pynni/nni/__main__.py +++ b/src/sdk/pynni/nni/__main__.py @@ -47,8 +47,7 @@ def create_builtin_class_instance(classname, jsonstr_args): class_module = importlib.import_module(ModuleName[classname]) class_constructor = getattr(class_module, ClassName[classname]) if jsonstr_args: - class_args = json.loads(jsonstr_args) - class_args = augment_classargs(class_args, classname) + class_args = augment_classargs(json.loads(jsonstr_args), classname) else: class_args = augment_classargs({}, classname) if class_args: diff --git a/src/sdk/pynni/nni/smac_tuner/smac_tuner.py b/src/sdk/pynni/nni/smac_tuner/smac_tuner.py index 87b9b977ba..36c14b330a 100644 --- a/src/sdk/pynni/nni/smac_tuner/smac_tuner.py +++ b/src/sdk/pynni/nni/smac_tuner/smac_tuner.py @@ -65,10 +65,10 @@ def _main_cli(self): self.logger.info("SMAC call: %s" % (" ".join(sys.argv))) cmd_reader = CMDReader() - args_, _ = cmd_reader.read_cmd() + args, _ = cmd_reader.read_cmd() root_logger = logging.getLogger() - root_logger.setLevel(args_.verbose_level) + root_logger.setLevel(args.verbose_level) logger_handler = logging.StreamHandler( stream=sys.stdout) if root_logger.level >= logging.INFO: @@ -90,31 +90,31 @@ def _main_cli(self): incumbent = None # Create scenario-object - scen = Scenario(args_.scenario_file, []) + scen = Scenario(args.scenario_file, []) - if args_.mode == "SMAC": + if args.mode == "SMAC": optimizer = SMAC( scenario=scen, - rng=np.random.RandomState(args_.seed), + rng=np.random.RandomState(args.seed), runhistory=rh, initial_configurations=initial_configs, stats=stats, restore_incumbent=incumbent, - run_id=args_.seed) - elif args_.mode == "ROAR": + run_id=args.seed) + elif args.mode == "ROAR": optimizer = ROAR( scenario=scen, - rng=np.random.RandomState(args_.seed), + rng=np.random.RandomState(args.seed), runhistory=rh, initial_configurations=initial_configs, - run_id=args_.seed) - elif args_.mode == "EPILS": + run_id=args.seed) + elif args.mode == "EPILS": optimizer = EPILS( scenario=scen, - rng=np.random.RandomState(args_.seed), + rng=np.random.RandomState(args.seed), runhistory=rh, initial_configurations=initial_configs, - run_id=args_.seed) + run_id=args.seed) else: optimizer = None @@ -132,7 +132,7 @@ def update_search_space(self, search_space): self.smbo_solver = self.optimizer.solver self.update_ss_done = True else: - pass + self.logger.warning('update search space is not supported.') def receive_trial_result(self, parameter_id, parameters, reward): ''' From fb5f8c4192466c67b634605104d80700fabdf7f7 Mon Sep 17 00:00:00 2001 From: quzha Date: Tue, 25 Sep 2018 14:11:40 +0800 Subject: [PATCH 7/7] fix comments --- .../nni/smac_tuner/convert_ss_to_scenario.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py b/src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py index ab2523a7a5..bd2694386f 100644 --- a/src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py +++ b/src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py @@ -31,6 +31,7 @@ def get_json_content(file_path): return None def generate_pcs(nni_search_space_content): + ''' # parameter_name categorical {value_1, ..., value_N} [default value] # parameter_name ordinal {value_1, ..., value_N} [default value] # parameter_name integer [min_value, max_value] [default value] @@ -38,6 +39,7 @@ def generate_pcs(nni_search_space_content): # parameter_name real [min_value, max_value] [default value] # parameter_name real [min_value, max_value] [default value] log # https://automl.github.io/SMAC3/stable/options.html + ''' search_space = nni_search_space_content with open('param_config_space.pcs', 'w') as pcs_fd: if isinstance(search_space, dict): @@ -80,18 +82,11 @@ def generate_pcs(nni_search_space_content): raise RuntimeError('incorrect search space.') def generate_scenario(ss_content): + ''' # deterministic, 1/0 # output_dir, # paramfile, # run_obj, 'quality' - sce_fd = open('scenario.txt', 'w') - sce_fd.write('deterministic = 0\n') - #sce_fd.write('output_dir = \n') - sce_fd.write('paramfile = param_config_space.pcs\n') - sce_fd.write('run_obj = quality\n') - sce_fd.close() - - generate_pcs(ss_content) # the following keys use default value or empty # algo, not required by tuner, but required by nni's training service for running trials @@ -114,6 +109,14 @@ def generate_scenario(ss_content): # runcount_limit, default: inf., use default because this is controlled by nni # wallclock_limit,default: inf., use default because this is controlled by nni # please refer to https://automl.github.io/SMAC3/stable/options.html + ''' + with open('scenario.txt', 'w') as sce_fd: + sce_fd.write('deterministic = 0\n') + #sce_fd.write('output_dir = \n') + sce_fd.write('paramfile = param_config_space.pcs\n') + sce_fd.write('run_obj = quality\n') + + generate_pcs(ss_content) if __name__ == '__main__': generate_scenario('search_space.json')