From db19946d112ff0659e228d9f5a9f05f3bbb83e41 Mon Sep 17 00:00:00 2001
From: SparkSnail <shinyang@microsoft.com>
Date: Wed, 6 Nov 2019 18:59:47 +0800
Subject: [PATCH 01/10] Support AAD token login in PAI mode (#1660)

---
 docs/en_US/TrainingService/PaiMode.md         | 16 ++++++++++
 .../rest_server/restValidationSchemas.ts      |  3 +-
 .../training_service/pai/paiConfig.ts         |  7 +++--
 .../pai/paiTrainingService.ts                 | 31 ++++++++++++-------
 tools/nni_cmd/config_schema.py                |  8 +++--
 5 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/docs/en_US/TrainingService/PaiMode.md b/docs/en_US/TrainingService/PaiMode.md
index 3787f7165d..a5926c6484 100644
--- a/docs/en_US/TrainingService/PaiMode.md
+++ b/docs/en_US/TrainingService/PaiMode.md
@@ -82,6 +82,22 @@ Compared with [LocalMode](LocalMode.md) and [RemoteMachineMode](RemoteMachineMod
         portNumber: 1
     ```
 
+NNI support two kind of authorization method in PAI, including password and PAI token, [refer](https://github.com/microsoft/pai/blob/b6bd2ab1c8890f91b7ac5859743274d2aa923c22/docs/rest-server/API.md#2-authentication). The authorization is configured in `paiConfig` field.  
+For password authorization, the `paiConfig` schema is:
+```
+paiConfig:
+  userName: your_pai_nni_user
+  passWord: your_pai_password
+  host: 10.1.1.1
+```  
+For pai token authorization, the `paiConfig` schema is:
+```
+paiConfig:
+  userName: your_pai_nni_user
+  token: your_pai_token
+  host: 10.1.1.1
+```
+
 Once complete to fill NNI experiment config file and save (for example, save as exp_pai.yml), then run the following command
 ```
 nnictl create --config exp_pai.yml
diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts
index 99bbe4bb96..69a7ec1d90 100644
--- a/src/nni_manager/rest_server/restValidationSchemas.ts
+++ b/src/nni_manager/rest_server/restValidationSchemas.ts
@@ -107,7 +107,8 @@ export namespace ValidationSchemas {
             }),
             pai_config: joi.object({
                 userName: joi.string().min(1).required(),
-                passWord: joi.string().min(1).required(),
+                passWord: joi.string().min(1),
+                token: joi.string().min(1),
                 host: joi.string().min(1).required()
             }),
             kubeflow_config: joi.object({
diff --git a/src/nni_manager/training_service/pai/paiConfig.ts b/src/nni_manager/training_service/pai/paiConfig.ts
index 43f95f7f9c..d25da7513f 100644
--- a/src/nni_manager/training_service/pai/paiConfig.ts
+++ b/src/nni_manager/training_service/pai/paiConfig.ts
@@ -107,19 +107,22 @@ export class PAIJobConfig {
  */
 export class PAIClusterConfig {
     public readonly userName: string;
-    public readonly passWord: string;
+    public readonly passWord?: string;
     public readonly host: string;
+    public readonly token?: string;
 
     /**
      * Constructor
      * @param userName User name of PAI Cluster
      * @param passWord password of PAI Cluster
      * @param host Host IP of PAI Cluster
+     * @param token PAI token of PAI Cluster
      */
-    constructor(userName: string, passWord : string, host : string) {
+    constructor(userName: string, host : string, passWord?: string, token?: string) {
         this.userName = userName;
         this.passWord = passWord;
         this.host = host;
+        this.token = token;
     }
 }
 
diff --git a/src/nni_manager/training_service/pai/paiTrainingService.ts b/src/nni_manager/training_service/pai/paiTrainingService.ts
index 27cc8be976..d741931b29 100644
--- a/src/nni_manager/training_service/pai/paiTrainingService.ts
+++ b/src/nni_manager/training_service/pai/paiTrainingService.ts
@@ -208,7 +208,7 @@ class PAITrainingService implements TrainingService {
 
         const stopJobRequest: request.Options = {
             uri: `http://${this.paiClusterConfig.host}/rest-server/api/v1/user/${this.paiClusterConfig.userName}\
-/jobs/${trialJobDetail.paiJobName}/executionType`,
+/jobs/${trialJobDetail.paiJobName}/executionType`, 
             method: 'PUT',
             json: true,
             body: {value: 'STOP'},
@@ -256,9 +256,15 @@ class PAITrainingService implements TrainingService {
                     path: '/webhdfs/api/v1',
                     host: this.paiClusterConfig.host
                 });
+                if(this.paiClusterConfig.passWord) {
+                    // Get PAI authentication token
+                    await this.updatePaiToken();
+                } else if(this.paiClusterConfig.token) {
+                    this.paiToken = this.paiClusterConfig.token;
+                } else {
+                    deferred.reject(new Error('pai cluster config format error, please set password or token!'));
+                }
 
-                // Get PAI authentication token
-                await this.updatePaiToken();
                 deferred.resolve();
                 break;
 
@@ -483,8 +489,7 @@ class PAITrainingService implements TrainingService {
         request(submitJobRequest, (error: Error, response: request.Response, body: any) => {
             if ((error !== undefined && error !== null) || response.statusCode >= 400) {
                 const errorMessage : string = (error !== undefined && error !== null) ? error.message :
-                    `Submit trial ${trialJobId} failed, http code:${response.statusCode}, http body: ${response.body}`;
-                this.log.error(errorMessage);
+                    `Submit trial ${trialJobId} failed, http code:${response.statusCode}, http body: ${response.body.message}`;
                 trialJobDetail.status = 'FAILED';
                 deferred.resolve(true);
             } else {
@@ -498,13 +503,15 @@ class PAITrainingService implements TrainingService {
 
     private async statusCheckingLoop(): Promise<void> {
         while (!this.stopping) {
-            try {
-                await this.updatePaiToken();
-            } catch (error) {
-                this.log.error(`${error}`);
-                //only throw error when initlize paiToken first time
-                if (this.paiToken === undefined) {
-                    throw new Error(error);
+            if(this.paiClusterConfig && this.paiClusterConfig.passWord) {
+                try {
+                    await this.updatePaiToken();
+                } catch (error) {
+                    this.log.error(`${error}`);
+                    //only throw error when initlize paiToken first time
+                    if (this.paiToken === undefined) {
+                        throw new Error(error);
+                    }
                 }
             }
             await this.paiJobCollector.retrieveTrialStatus(this.paiToken, this.paiClusterConfig);
diff --git a/tools/nni_cmd/config_schema.py b/tools/nni_cmd/config_schema.py
index dded8d1e95..5eb9538ad8 100644
--- a/tools/nni_cmd/config_schema.py
+++ b/tools/nni_cmd/config_schema.py
@@ -265,11 +265,15 @@ def setPathCheck(key):
 }
 
 pai_config_schema = {
-    'paiConfig':{
+    'paiConfig': Or({
         'userName': setType('userName', str),
         'passWord': setType('passWord', str),
         'host': setType('host', str)
-    }
+    }, {
+        'userName': setType('userName', str),
+        'token': setType('token', str),
+        'host': setType('host', str)
+    })
 }
 
 kubeflow_trial_schema = {

From 3e0ef0052d40480691304cafa61cf71019c7b233 Mon Sep 17 00:00:00 2001
From: chicm-ms <38930155+chicm-ms@users.noreply.github.com>
Date: Fri, 8 Nov 2019 17:44:01 +0800
Subject: [PATCH 02/10] round-robin policy (#1702)

---
 .../remote_machine/gpuScheduler.ts            | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/nni_manager/training_service/remote_machine/gpuScheduler.ts b/src/nni_manager/training_service/remote_machine/gpuScheduler.ts
index 5e7f065971..4244eb8967 100644
--- a/src/nni_manager/training_service/remote_machine/gpuScheduler.ts
+++ b/src/nni_manager/training_service/remote_machine/gpuScheduler.ts
@@ -28,6 +28,8 @@ import {
     parseGpuIndices, RemoteMachineMeta, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail, ScheduleResultType, SSHClientManager
 } from './remoteMachineData';
 
+type SCHEDULE_POLICY_NAME = 'random' | 'round-robin';
+
 /**
  * A simple GPU scheduler implementation
  */
@@ -35,13 +37,18 @@ export class GPUScheduler {
 
     private readonly machineSSHClientMap : Map<RemoteMachineMeta, SSHClientManager>;
     private readonly log: Logger = getLogger();
+    private readonly policyName: SCHEDULE_POLICY_NAME = 'round-robin';
+    private roundRobinIndex: number = 0;
+    private configuredRMs: RemoteMachineMeta[] = [];
 
     /**
      * Constructor
      * @param machineSSHClientMap map from remote machine to sshClient
      */
     constructor(machineSSHClientMap : Map<RemoteMachineMeta, SSHClientManager>) {
+        assert(machineSSHClientMap.size > 0);
         this.machineSSHClientMap = machineSSHClientMap;
+        this.configuredRMs = Array.from(machineSSHClientMap.keys());
     }
 
     /**
@@ -189,7 +196,21 @@ export class GPUScheduler {
     private selectMachine(rmMetas: RemoteMachineMeta[]): RemoteMachineMeta {
         assert(rmMetas !== undefined && rmMetas.length > 0);
 
-        return randomSelect(rmMetas);
+        if (this.policyName === 'random') {
+            return randomSelect(rmMetas);
+        } else if (this.policyName === 'round-robin') {
+            return this.roundRobinSelect(rmMetas);
+        } else {
+            throw new Error(`Unsupported schedule policy: ${this.policyName}`);
+        }
+    }
+
+    private roundRobinSelect(rmMetas: RemoteMachineMeta[]): RemoteMachineMeta {
+        while (!rmMetas.includes(this.configuredRMs[this.roundRobinIndex % this.configuredRMs.length])) {
+            this.roundRobinIndex++;
+        }
+
+        return this.configuredRMs[this.roundRobinIndex++ % this.configuredRMs.length];
     }
 
     private selectGPUsForTrial(gpuInfos: GPUInfo[], requiredGPUNum: number): GPUInfo[] {

From fb18f0d4d1da46f23b501db176c3434b876dac4d Mon Sep 17 00:00:00 2001
From: Yuge Zhang <v-yugzh@microsoft.com>
Date: Fri, 8 Nov 2019 20:53:34 +0800
Subject: [PATCH 03/10] Fix pipeline node version for unittests (#1721)

* try to fix pipeline

* locate npm and node version

* override with node in local dir

* refactor path set

* fix install SMAC on macOS
---
 Makefile            |  6 ++++--
 azure-pipelines.yml | 38 +++++++++++++++++++-------------------
 2 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/Makefile b/Makefile
index 8877e5c2ae..4759d56c34 100644
--- a/Makefile
+++ b/Makefile
@@ -52,6 +52,7 @@ $(shell mkdir -p $(NNI_DEPENDENCY_FOLDER))
 NNI_NODE_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64.tar.xz
 NNI_NODE_FOLDER = $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64
 NNI_NODE ?= $(BIN_FOLDER)/node
+NNI_NPM ?= $(BIN_FOLDER)/npm
 NNI_YARN_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn.tar.gz
 NNI_YARN_FOLDER ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn
 NNI_YARN ?= PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn
@@ -149,8 +150,9 @@ install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL)
 	mkdir $(NNI_NODE_FOLDER)
 	tar -xf $(NNI_NODE_TARBALL) -C $(NNI_NODE_FOLDER) --strip-components 1
 	mkdir -p $(BIN_FOLDER)
-	rm -f $(NNI_NODE)
-	cp $(NNI_NODE_FOLDER)/bin/node $(NNI_NODE)
+	rm -f $(NNI_NODE) $(NNI_NPM)
+	ln -s $(NNI_NODE_FOLDER)/bin/node $(NNI_NODE)
+	ln -s $(NNI_NODE_FOLDER)/bin/npm $(NNI_NPM)
 	
 	#$(_INFO) Extracting Yarn $(_END)
 	rm -rf $(NNI_YARN_FOLDER)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 7b8e6f626c..14ef0197ca 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -12,6 +12,7 @@ jobs:
       python3 -m pip install --upgrade pip setuptools --user
       python3 -m pip install pylint==2.3.1 astroid==2.2.5 --user
       python3 -m pip install coverage --user
+      echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
     displayName: 'Install python tools'
   - script: |
       source install.sh
@@ -23,8 +24,8 @@ jobs:
       python3 -m pip install keras==2.1.6 --user
       python3 -m pip install gym onnx --user
       sudo apt-get install swig -y
-      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
-      PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
+      nnictl package install --name=SMAC
+      nnictl package install --name=BOHB
     displayName: 'Install dependencies'
   - script: |
       set -e
@@ -42,25 +43,23 @@ jobs:
     displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
-      sudo apt install -y swig
-      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
       source unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 naive_test.py
+      python3 naive_test.py
     displayName: 'Naive test'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 tuner_test.py
+      python3 tuner_test.py
     displayName: 'Built-in tuners / assessors tests'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
+      python3 metrics_test.py
     displayName: 'Trial job metrics test'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
+      python3 cli_test.py
     displayName: 'nnicli test'
 
 - job: 'basic_test_pr_macOS'
@@ -74,33 +73,34 @@ jobs:
   steps:
   - script: python3 -m pip install --upgrade pip setuptools
     displayName: 'Install python tools'
-  - script: |
-      python3 -m pip install torch==0.4.1 --user
-      python3 -m pip install torchvision==0.2.1 --user
-      python3 -m pip install tensorflow==1.13.1 --user
-    displayName: 'Install dependencies'
   - script: |
       source install.sh
+      echo "##vso[task.setvariable variable=PATH]${HOME}/Library/Python/3.7/bin:${PATH}"
     displayName: 'Install nni toolkit via source code'
   - script: |
-      cd test
+      python3 -m pip install torch==0.4.1 --user
+      python3 -m pip install torchvision==0.2.1 --user
+      python3 -m pip install tensorflow==1.13.1 --user
       ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" < /dev/null 2> /dev/null
       brew install swig@3
       ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig
-      PATH=$HOME/Library/Python/3.7/bin:$PATH nnictl package install --name=SMAC
-      PATH=$HOME/Library/Python/3.7/bin:$PATH source unittest.sh
+      nnictl package install --name=SMAC
+    displayName: 'Install dependencies'
+  - script: |
+      cd test
+      source unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 naive_test.py
+      python3 naive_test.py
     displayName: 'Naive test'
   - script: |
       cd test
-      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 tuner_test.py
+      python3 tuner_test.py
     displayName: 'Built-in tuners / assessors tests'
   - script: |
       cd test
-      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 cli_test.py
+      python3 cli_test.py
     displayName: 'nnicli test'
 
 - job: 'basic_test_pr_Windows'

From 2d375f435770b256657a4eb76b7a058b0eb16bd9 Mon Sep 17 00:00:00 2001
From: Tang Lang <lanny@mail.hfut.edu.cn>
Date: Sat, 9 Nov 2019 11:25:28 +0800
Subject: [PATCH 04/10] add export doc (#1705)

* add export doc
---
 docs/en_US/Compressor/Overview.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/en_US/Compressor/Overview.md b/docs/en_US/Compressor/Overview.md
index 7ee603e3e3..5fc8e45c5d 100644
--- a/docs/en_US/Compressor/Overview.md
+++ b/docs/en_US/Compressor/Overview.md
@@ -95,7 +95,17 @@ pruner.update_epoch(epoch)
 
 The other is `step`, it can be called with `pruner.step()` after each minibatch. Note that not all algorithms need these two APIs, for those that do not need them, calling them is allowed but has no effect.
 
-__[TODO]__ The last API is for users to export the compressed model. You will get a compressed model when you finish the training using this API. It also exports another file storing the values of masks.
+You can easily export the compressed model using the following API if you are pruning your model, ```state_dict``` of the sparse model weights will be stored in ```model.pth```, which can be loaded by ```torch.load('model.pth')```
+
+```
+pruner.export_model(model_path='model.pth')
+```
+
+```mask_dict ``` and pruned model in ```onnx``` format(```input_shape``` need to be specified) can also be exported like this:
+
+```python
+pruner.export_model(model_path='model.pth', mask_path='mask.pth', onnx_path='model.onnx', input_shape=[1, 1, 28, 28])
+```
 
 ## Customize new compression algorithms
 

From 901012eb90bf7c0dbb3d2566e5d8c0eda1b5b249 Mon Sep 17 00:00:00 2001
From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com>
Date: Sat, 9 Nov 2019 12:43:02 +0800
Subject: [PATCH 05/10] docstring fix (#1691)

---
 src/sdk/pynni/nni/assessor.py | 14 +++++++-------
 src/sdk/pynni/nni/tuner.py    | 20 ++++++++++----------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/sdk/pynni/nni/assessor.py b/src/sdk/pynni/nni/assessor.py
index 0f1dc95619..01a2abcbe9 100644
--- a/src/sdk/pynni/nni/assessor.py
+++ b/src/sdk/pynni/nni/assessor.py
@@ -53,14 +53,14 @@ class Assessor(Recoverable):
     to tell whether this trial can be early stopped or not.
 
     This is the abstract base class for all assessors.
-    Early stopping algorithms should derive this class and override :meth:`assess_trial` method,
+    Early stopping algorithms should inherit this class and override :meth:`assess_trial` method,
     which receives intermediate results from trials and give an assessing result.
 
     If :meth:`assess_trial` returns :obj:`AssessResult.Bad` for a trial,
     it hints NNI framework that the trial is likely to result in a poor final accuracy,
     and therefore should be killed to save resource.
 
-    If an accessor want's to get notified when a trial ends, it can also override :meth:`trial_end`.
+    If an accessor want's to be notified when a trial ends, it can also override :meth:`trial_end`.
 
     To write a new assessor, you can reference :class:`~nni.medianstop_assessor.MedianstopAssessor`'s code as an example.
 
@@ -77,7 +77,7 @@ def assess_trial(self, trial_job_id, trial_history):
 
         The NNI framework has little guarantee on ``trial_history``.
         This method is not guaranteed to be invoked for each time ``trial_history`` get updated.
-        It is also possible that a trial's history keeps updateing after receiving a bad result.
+        It is also possible that a trial's history keeps updating after receiving a bad result.
         And if the trial failed and retried, ``trial_history`` may be inconsistent with its previous value.
 
         The only guarantee is that ``trial_history`` is always growing.
@@ -96,9 +96,9 @@ def assess_trial(self, trial_job_id, trial_history):
 
         Parameters
         ----------
-        trial_job_id: str
+        trial_job_id : str
             Unique identifier of the trial.
-        trial_history: list
+        trial_history : list
             Intermediate results of this trial. The element type is decided by trial code.
 
         Returns
@@ -114,9 +114,9 @@ def trial_end(self, trial_job_id, success):
 
         Parameters
         ----------
-        trial_job_id: str
+        trial_job_id : str
             Unique identifier of the trial.
-        success: bool
+        success : bool
             True if the trial successfully completed; False if failed or terminated.
         """
 
diff --git a/src/sdk/pynni/nni/tuner.py b/src/sdk/pynni/nni/tuner.py
index f011022151..177232b7ed 100644
--- a/src/sdk/pynni/nni/tuner.py
+++ b/src/sdk/pynni/nni/tuner.py
@@ -42,7 +42,7 @@ class Tuner(Recoverable):
     A new trial will run with this configuration.
 
     This is the abstract base class for all tuners.
-    Tuning algorithms should derive this class and override :meth:`update_search_space`, :meth:`receive_trial_result`,
+    Tuning algorithms should inherit this class and override :meth:`update_search_space`, :meth:`receive_trial_result`,
     as well as :meth:`generate_parameters` or :meth:`generate_multiple_parameters`.
 
     After initializing, NNI will first call :meth:`update_search_space` to tell tuner the feasible region,
@@ -96,9 +96,9 @@ def generate_parameters(self, parameter_id, **kwargs):
 
         Parameters
         ----------
-        parameter_id: int
+        parameter_id : int
             Unique identifier for requested hyper-parameters. This will later be used in :meth:`receive_trial_result`.
-        **kwargs:
+        **kwargs
             Unstable parameters which should be ignored by normal users.
 
         Returns
@@ -129,10 +129,10 @@ def generate_multiple_parameters(self, parameter_id_list, **kwargs):
 
         Parameters
         ----------
-        parameter_id_list: list of int
+        parameter_id_list : list of int
             Unique identifiers for each set of requested hyper-parameters.
             These will later be used in :meth:`receive_trial_result`.
-        **kwargs:
+        **kwargs
             Unstable parameters which should be ignored by normal users.
 
         Returns
@@ -159,13 +159,13 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
 
         Parameters
         ----------
-        parameter_id: int
+        parameter_id : int
             Unique identifier of used hyper-parameters, same with :meth:`generate_parameters`.
         parameters
             Hyper-parameters generated by :meth:`generate_parameters`.
         value
             Result from trial (the return value of :func:`nni.report_final_result`).
-        **kwargs:
+        **kwargs
             Unstable parameters which should be ignored by normal users.
         """
         raise NotImplementedError('Tuner: receive_trial_result not implemented')
@@ -186,11 +186,11 @@ def trial_end(self, parameter_id, success, **kwargs):
 
         Parameters
         ----------
-        parameter_id: int
+        parameter_id : int
             Unique identifier for hyper-parameters used by this trial.
-        success: bool
+        success : bool
             True if the trial successfully completed; False if failed or terminated.
-        **kwargs:
+        **kwargs
             Unstable parameters which should be ignored by normal users.
         """
 

From bf23601287561b51b66b8b8a317827572ccecccf Mon Sep 17 00:00:00 2001
From: gxiaotian <gaoxiaotian@outlook.com>
Date: Mon, 11 Nov 2019 10:09:18 +0800
Subject: [PATCH 06/10] Add Docs for Tuning Systems with NNI (#1715)

* Add Docs for Tuning Systems with NNI

* Updates based on comments

* updates

* updates

* updates
---
 README_zh_CN.md                               |  2 +-
 docs/en_US/CommunitySharings/TuningSystems.md | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 docs/en_US/CommunitySharings/TuningSystems.md

diff --git a/README_zh_CN.md b/README_zh_CN.md
index 9af25db8ea..a0cafb5a72 100644
--- a/README_zh_CN.md
+++ b/README_zh_CN.md
@@ -359,4 +359,4 @@ You can use these commands to get more information about the experiment
 
 ## **许可协议**
 
-代码库遵循 [MIT 许可协议](LICENSE)
\ No newline at end of file
+代码库遵循 [MIT 许可协议](LICENSE)
diff --git a/docs/en_US/CommunitySharings/TuningSystems.md b/docs/en_US/CommunitySharings/TuningSystems.md
new file mode 100644
index 0000000000..30c44c7a8a
--- /dev/null
+++ b/docs/en_US/CommunitySharings/TuningSystems.md
@@ -0,0 +1,10 @@
+# Automatically tune systems with NNI
+
+As computer systems and networking get increasingly complicated, optimizing them manually with explicit rules and heuristics becomes harder than ever before, sometimes impossible. Below are two examples of tuning systems with NNI. Anyone can easily tune their own systems by following them.
+
+* [Tuning RocksDB with NNI](../TrialExample/RocksdbExamples.md)
+* [Tuning parameters of SPTAG (Space Partition Tree And Graph) with NNI](SptagAutoTune.md)
+
+Please see [this paper](https://dl.acm.org/citation.cfm?id=3352031) for more details:
+
+Mike Liang, Chieh-Jan, et al. "The Case for Learning-and-System Co-design." ACM SIGOPS Operating Systems Review 53.1 (2019): 68-74.

From 803f056a307366dbef613defae9000131c5391c7 Mon Sep 17 00:00:00 2001
From: Lijiao <35484733+lvybriage@users.noreply.github.com>
Date: Mon, 11 Nov 2019 10:40:58 +0800
Subject: [PATCH 07/10] maintain selected status (#1710)

---
 src/webui/src/App.tsx                 | 10 +++++++++-
 src/webui/src/components/Overview.tsx | 18 ++++++++++--------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/webui/src/App.tsx b/src/webui/src/App.tsx
index 3015a7e4ce..21ddfbd385 100644
--- a/src/webui/src/App.tsx
+++ b/src/webui/src/App.tsx
@@ -10,6 +10,7 @@ interface AppState {
     columnList: Array<string>;
     experimentUpdateBroadcast: number;
     trialsUpdateBroadcast: number;
+    metricGraphMode: 'max' | 'min'; // tuner's optimize_mode filed
 }
 
 class App extends React.Component<{}, AppState> {
@@ -22,6 +23,7 @@ class App extends React.Component<{}, AppState> {
             columnList: COLUMN,
             experimentUpdateBroadcast: 0,
             trialsUpdateBroadcast: 0,
+            metricGraphMode: 'max'
         };
     }
 
@@ -30,6 +32,7 @@ class App extends React.Component<{}, AppState> {
         this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 }));
         this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 }));
         this.timerId = window.setTimeout(this.refresh, this.state.interval * 1000);
+        this.setState({ metricGraphMode: (EXPERIMENT.optimizeMode === 'minimize' ? 'min' : 'max') });
     }
 
     changeInterval = (interval: number) => {
@@ -46,8 +49,12 @@ class App extends React.Component<{}, AppState> {
         this.setState({ columnList: columnList });
     }
 
+    changeMetricGraphMode = (val: 'max' | 'min') => {
+        this.setState({ metricGraphMode: val });
+    }
+
     render() {
-        const { interval, columnList, experimentUpdateBroadcast, trialsUpdateBroadcast } = this.state;
+        const { interval, columnList, experimentUpdateBroadcast, trialsUpdateBroadcast, metricGraphMode } = this.state;
         if (experimentUpdateBroadcast === 0 || trialsUpdateBroadcast === 0) {
             return null;  // TODO: render a loading page
         }
@@ -59,6 +66,7 @@ class App extends React.Component<{}, AppState> {
                     columnList, changeColumn: this.changeColumn,
                     experimentUpdateBroadcast,
                     trialsUpdateBroadcast,
+                    metricGraphMode, changeMetricGraphMode: this.changeMetricGraphMode
                 })
         );
         return (
diff --git a/src/webui/src/components/Overview.tsx b/src/webui/src/components/Overview.tsx
index 22d52e5458..c6a3af449c 100644
--- a/src/webui/src/components/Overview.tsx
+++ b/src/webui/src/components/Overview.tsx
@@ -19,31 +19,33 @@ require('../static/style/overviewTitle.scss');
 interface OverviewProps {
     experimentUpdateBroadcast: number;
     trialsUpdateBroadcast: number;
+    metricGraphMode: 'max' | 'min';
+    changeMetricGraphMode: (val: 'max' | 'min') => void;
 }
 
 interface OverviewState {
     trialConcurrency: number;
-    metricGraphMode: 'max' | 'min';
 }
 
 class Overview extends React.Component<OverviewProps, OverviewState> {
     constructor(props: OverviewProps) {
         super(props);
         this.state = {
-            trialConcurrency: EXPERIMENT.trialConcurrency,
-            metricGraphMode: (EXPERIMENT.optimizeMode === 'minimize' ? 'min' : 'max'),
+            trialConcurrency: EXPERIMENT.trialConcurrency
         };
     }
 
     clickMaxTop = (event: React.SyntheticEvent<EventTarget>) => {
         event.stopPropagation();
         // #999 panel active bgcolor; #b3b3b3 as usual
-        this.setState({ metricGraphMode: 'max' });
+        const { changeMetricGraphMode } = this.props;
+        changeMetricGraphMode('max');
     }
 
     clickMinTop = (event: React.SyntheticEvent<EventTarget>) => {
         event.stopPropagation();
-        this.setState({ metricGraphMode: 'min' });
+        const { changeMetricGraphMode } = this.props;
+        changeMetricGraphMode('min');
     }
 
     changeConcurrency = (val: number) => {
@@ -51,8 +53,8 @@ class Overview extends React.Component<OverviewProps, OverviewState> {
     }
 
     render() {
-        const { trialConcurrency, metricGraphMode } = this.state;
-        const { experimentUpdateBroadcast } = this.props;
+        const { trialConcurrency } = this.state;
+        const { experimentUpdateBroadcast, metricGraphMode } = this.props;
 
         const searchSpace = this.convertSearchSpace();
 
@@ -160,7 +162,7 @@ class Overview extends React.Component<OverviewProps, OverviewState> {
 
     private findBestTrials(): Trial[] {
         let bestTrials = TRIALS.sort();
-        if (this.state.metricGraphMode === 'max') {
+        if (this.props.metricGraphMode === 'max') {
             bestTrials.reverse().splice(10);
         } else {
             bestTrials.splice(10);

From 0168ff1c66026995b938eea93a24f94a0012c9be Mon Sep 17 00:00:00 2001
From: xuehui <xuehui@microsoft.com>
Date: Mon, 11 Nov 2019 10:45:27 +0800
Subject: [PATCH 08/10] update docstring and pylint (#1662)

* update docstring of batchtuner

* update docstring of batch tuner

* update docstring of evolution tuner

* update docstring and pylint of metis_tuner

* fix pylint related to logger in metis_tuner

* fix pylint

* update

* fix pylint in metis_tuner

* update in networkmorphsim_tuner

* update

* update

* update docstring in hyperopt_tuner

* update batch_tuner

* delete unused space

* update in metis

* update sdk_reference.rst

* update netowrkmorhism

* update networkmorphsim

* update batch_tuner

* update batch_tuner

* update

* update metis

* roll back to print

* update Returns

* update

* delete white space
---
 docs/en_US/sdk_reference.rst                  |   3 +
 src/sdk/pynni/nni/batch_tuner/batch_tuner.py  |  63 ++--
 .../nni/evolution_tuner/evolution_tuner.py    |  52 ++-
 .../nni/hyperopt_tuner/hyperopt_tuner.py      |   6 +-
 .../metis_tuner/Regression_GMM/CreateModel.py |  16 +-
 .../metis_tuner/Regression_GMM/Selection.py   |  21 +-
 .../Regression_GP/OutlierDetection.py         |  45 +--
 .../metis_tuner/lib_acquisition_function.py   |  50 ++-
 .../metis_tuner/lib_constraint_summation.py   |  37 +-
 src/sdk/pynni/nni/metis_tuner/lib_data.py     |   5 +-
 src/sdk/pynni/nni/metis_tuner/metis_tuner.py  | 337 ++++++++++++------
 .../nni/networkmorphism_tuner/bayesian.py     |  26 +-
 .../pynni/nni/networkmorphism_tuner/graph.py  |  65 ++--
 .../graph_transformer.py                      |  20 +-
 .../layer_transformer.py                      |  24 +-
 .../pynni/nni/networkmorphism_tuner/layers.py | 326 +++++++++++------
 .../networkmorphism_tuner.py                  | 117 +++---
 src/sdk/pynni/nni/networkmorphism_tuner/nn.py |  33 +-
 .../test_networkmorphism_tuner.py             |  17 +-
 19 files changed, 842 insertions(+), 421 deletions(-)

diff --git a/docs/en_US/sdk_reference.rst b/docs/en_US/sdk_reference.rst
index 0eccbc1b5c..7bf274996d 100644
--- a/docs/en_US/sdk_reference.rst
+++ b/docs/en_US/sdk_reference.rst
@@ -36,6 +36,9 @@ Tuner
 ..  autoclass:: nni.metis_tuner.metis_tuner.MetisTuner
     :members:
 
+..  autoclass:: nni.batch_tuner.batch_tuner.BatchTuner
+    :members:
+
 Assessor
 ------------------------
 ..  autoclass:: nni.assessor.Assessor
diff --git a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py
index 64012444ac..c223d93552 100644
--- a/src/sdk/pynni/nni/batch_tuner/batch_tuner.py
+++ b/src/sdk/pynni/nni/batch_tuner/batch_tuner.py
@@ -31,22 +31,27 @@ class BatchTuner
 CHOICE = 'choice'
 VALUE = '_value'
 
-logger = logging.getLogger('batch_tuner_AutoML')
+LOGGER = logging.getLogger('batch_tuner_AutoML')
 
 class BatchTuner(Tuner):
     """
     BatchTuner is tuner will running all the configure that user want to run batchly.
+
+    Examples
+    --------
     The search space only be accepted like:
+    ```
     {
         'combine_params': { '_type': 'choice',
                              '_value': '[{...}, {...}, {...}]',
                           }
     }
+    ```
     """
 
     def __init__(self):
-        self.count = -1
-        self.values = []
+        self._count = -1
+        self._values = []
 
     def is_valid(self, search_space):
         """
@@ -55,6 +60,11 @@ def is_valid(self, search_space):
         Parameters
         ----------
         search_space : dict
+
+        Returns
+        -------
+        None or list
+            If valid, return candidate values; else return None.
         """
         if not len(search_space) == 1:
             raise RuntimeError('BatchTuner only supprt one combined-paramreters key.')
@@ -62,11 +72,14 @@ def is_valid(self, search_space):
         for param in search_space:
             param_type = search_space[param][TYPE]
             if not param_type == CHOICE:
-                raise RuntimeError('BatchTuner only supprt one combined-paramreters type is choice.')
-            else:
-                if isinstance(search_space[param][VALUE], list):
-                    return search_space[param][VALUE]
-                raise RuntimeError('The combined-paramreters value in BatchTuner is not a list.')
+                raise RuntimeError('BatchTuner only supprt \
+                                    one combined-paramreters type is choice.')
+
+            if isinstance(search_space[param][VALUE], list):
+                return search_space[param][VALUE]
+
+            raise RuntimeError('The combined-paramreters \
+                                value in BatchTuner is not a list.')
         return None
 
     def update_search_space(self, search_space):
@@ -76,7 +89,7 @@ def update_search_space(self, search_space):
         ----------
         search_space : dict
         """
-        self.values = self.is_valid(search_space)
+        self._values = self.is_valid(search_space)
 
     def generate_parameters(self, parameter_id, **kwargs):
         """Returns a dict of trial (hyper-)parameters, as a serializable object.
@@ -84,41 +97,49 @@ def generate_parameters(self, parameter_id, **kwargs):
         Parameters
         ----------
         parameter_id : int
+
+        Returns
+        -------
+        dict
+            A candidate parameter group.
         """
-        self.count += 1
-        if self.count > len(self.values) - 1:
+        self._count += 1
+        if self._count > len(self._values) - 1:
             raise nni.NoMoreTrialError('no more parameters now.')
-        return self.values[self.count]
+        return self._values[self._count]
 
     def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
         pass
 
     def import_data(self, data):
         """Import additional data for tuning
+
         Parameters
         ----------
         data:
             a list of dictionarys, each of which has at least two keys, 'parameter' and 'value'
         """
-        if not self.values:
-            logger.info("Search space has not been initialized, skip this data import")
+        if not self._values:
+            LOGGER.info("Search space has not been initialized, skip this data import")
             return
 
-        self.values = self.values[(self.count+1):]
-        self.count = -1
+        self._values = self._values[(self._count+1):]
+        self._count = -1
 
         _completed_num = 0
         for trial_info in data:
-            logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data))
+            LOGGER .info("Importing data, current processing \
+                            progress %s / %s", _completed_num, len(data))
             # simply validate data format
             assert "parameter" in trial_info
             _params = trial_info["parameter"]
             assert "value" in trial_info
             _value = trial_info['value']
             if not _value:
-                logger.info("Useless trial data, value is %s, skip this trial data.", _value)
+                LOGGER.info("Useless trial data, value is %s, skip this trial data.", _value)
                 continue
             _completed_num += 1
-            if _params in self.values:
-                self.values.remove(_params)
-        logger.info("Successfully import data to batch tuner, total data: %d, imported data: %d.", len(data), _completed_num)
+            if _params in self._values:
+                self._values.remove(_params)
+        LOGGER .info("Successfully import data to batch tuner, \
+                        total data: %d, imported data: %d.", len(data), _completed_num)
diff --git a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py b/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py
index 8cec6df1ce..3b12ab7505 100644
--- a/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py
+++ b/src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py
@@ -32,7 +32,9 @@
 
 
 def json2space(x, oldy=None, name=NodeType.ROOT):
-    """Change search space from json format to hyperopt format
+    """
+    Change search space from json format to hyperopt format
+
     """
     y = list()
     if isinstance(x, dict):
@@ -59,7 +61,9 @@ def json2space(x, oldy=None, name=NodeType.ROOT):
     return y
 
 def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.ROOT):
-    """Json to pramaters.
+    """
+    Json to pramaters.
+
     """
     if isinstance(x, dict):
         if NodeType.TYPE in x.keys():
@@ -117,6 +121,17 @@ def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeTyp
 class Individual:
     """
     Indicidual class to store the indv info.
+
+    Attributes
+    ----------
+    config : str
+        Search space.
+    info : str
+        The str to save information of individual.
+    result : float
+        The final metric of a individual.
+    store_dir : str
+    save_dir : str
     """
 
     def __init__(self, config=None, info=None, result=None, save_dir=None):
@@ -124,6 +139,7 @@ def __init__(self, config=None, info=None, result=None, save_dir=None):
         Parameters
         ----------
         config : str
+            A config to represent a group of parameters.
         info : str
         result : float
         save_dir : str
@@ -140,6 +156,8 @@ def __str__(self):
 
     def mutation(self, config=None, info=None, save_dir=None):
         """
+        Mutation by reset state information.
+
         Parameters
         ----------
         config : str
@@ -177,8 +195,11 @@ def __init__(self, optimize_mode="maximize", population_size=32):
         self.population = None
         self.space = None
 
+
     def update_search_space(self, search_space):
-        """Update search space.
+        """
+        Update search space.
+
         Search_space contains the information that user pre-defined.
 
         Parameters
@@ -191,15 +212,19 @@ def update_search_space(self, search_space):
         self.random_state = np.random.RandomState()
         self.population = []
         is_rand = dict()
+
         for item in self.space:
             is_rand[item] = True
+
         for _ in range(self.population_size):
             config = json2parameter(
                 self.searchspace_json, is_rand, self.random_state)
             self.population.append(Individual(config=config))
 
+
     def generate_parameters(self, parameter_id, **kwargs):
-        """Returns a dict of trial (hyper-)parameters, as a serializable object.
+        """
+        This function will returns a dict of trial (hyper-)parameters, as a serializable object.
 
         Parameters
         ----------
@@ -207,15 +232,19 @@ def generate_parameters(self, parameter_id, **kwargs):
 
         Returns
         -------
-        config : dict
+        dict
+            A group of candaidte parameters that evolution tuner generated.
         """
         if not self.population:
             raise RuntimeError('The population is empty')
+
         pos = -1
+
         for i in range(len(self.population)):
             if self.population[i].result is None:
                 pos = i
                 break
+
         if pos != -1:
             indiv = copy.deepcopy(self.population[pos])
             self.population.pop(pos)
@@ -230,6 +259,7 @@ def generate_parameters(self, parameter_id, **kwargs):
                                self.population[0].config)
             is_rand = dict()
             mutation_pos = space[random.randint(0, len(space)-1)]
+
             for i in range(len(self.space)):
                 is_rand[self.space[i]] = (self.space[i] == mutation_pos)
             config = json2parameter(
@@ -238,21 +268,27 @@ def generate_parameters(self, parameter_id, **kwargs):
             # remove "_index" from config and save params-id
 
             total_config = config
+
         self.total_data[parameter_id] = total_config
         config = split_index(total_config)
+
         return config
 
+
     def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
-        '''Record the result from a trial
+        """
+        Record the result from a trial
 
         Parameters
         ----------
-        parameters: dict
+        parameter_id : int
+        parameters : dict
         value : dict/float
             if value is dict, it should have "default" key.
             value is final metrics of the trial.
-        '''
+        """
         reward = extract_scalar_reward(value)
+
         if parameter_id not in self.total_data:
             raise RuntimeError('Received parameter_id not in total_data.')
         # restore the paramsters contains "_index"
diff --git a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py
index 89c8d662c3..0e250fa8dd 100644
--- a/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py
+++ b/src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py
@@ -422,7 +422,8 @@ def miscs_update_idxs_vals(self,
                     misc_by_id[tid]['vals'][key] = [val]
 
     def get_suggestion(self, random_search=False):
-        """get suggestion from hyperopt
+        """
+        get suggestion from hyperopt
 
         Parameters
         ----------
@@ -473,7 +474,8 @@ def get_suggestion(self, random_search=False):
         return total_params
 
     def import_data(self, data):
-        """Import additional data for tuning
+        """
+        Import additional data for tuning
 
         Parameters
         ----------
diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py
index 3ed39e0cf8..7bc9e070fb 100644
--- a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py
+++ b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/CreateModel.py
@@ -16,7 +16,8 @@
 # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
 
 import os
 import sys
@@ -31,7 +32,8 @@ def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34):
     '''
     Create the Gaussian Mixture Model
     '''
-    samples = [samples_x[i] + [samples_y_aggregation[i]] for i in range(0, len(samples_x))]
+    samples = [samples_x[i] + [samples_y_aggregation[i]]
+               for i in range(0, len(samples_x))]
 
     # Sorts so that we can get the top samples
     samples = sorted(samples, key=itemgetter(-1))
@@ -39,13 +41,16 @@ def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34):
     samples_goodbatch = samples[0:samples_goodbatch_size]
     samples_badbatch = samples[samples_goodbatch_size:]
 
-    samples_x_goodbatch = [sample_goodbatch[0:-1] for sample_goodbatch in samples_goodbatch]
+    samples_x_goodbatch = [sample_goodbatch[0:-1]
+                           for sample_goodbatch in samples_goodbatch]
     #samples_y_goodbatch = [sample_goodbatch[-1] for sample_goodbatch in samples_goodbatch]
-    samples_x_badbatch = [sample_badbatch[0:-1] for sample_badbatch in samples_badbatch]
+    samples_x_badbatch = [sample_badbatch[0:-1]
+                          for sample_badbatch in samples_badbatch]
 
     # === Trains GMM clustering models === #
     #sys.stderr.write("[%s] Train GMM's GMM model\n" % (os.path.basename(__file__)))
-    bgmm_goodbatch = mm.BayesianGaussianMixture(n_components=max(1, samples_goodbatch_size - 1))
+    bgmm_goodbatch = mm.BayesianGaussianMixture(
+        n_components=max(1, samples_goodbatch_size - 1))
     bad_n_components = max(1, len(samples_x) - samples_goodbatch_size - 1)
     bgmm_badbatch = mm.BayesianGaussianMixture(n_components=bad_n_components)
     bgmm_goodbatch.fit(samples_x_goodbatch)
@@ -55,4 +60,3 @@ def create_model(samples_x, samples_y_aggregation, percentage_goodbatch=0.34):
     model['clusteringmodel_good'] = bgmm_goodbatch
     model['clusteringmodel_bad'] = bgmm_badbatch
     return model
-    
\ No newline at end of file
diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py
index eba35ae09d..758383f92f 100644
--- a/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py
+++ b/src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py
@@ -16,7 +16,8 @@
 # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
 
 import os
 import random
@@ -33,14 +34,17 @@
 CONSTRAINT_PARAMS_IDX = []
 
 
-def _ratio_scores(parameters_value, clusteringmodel_gmm_good, clusteringmodel_gmm_bad):
+def _ratio_scores(parameters_value, clusteringmodel_gmm_good,
+                  clusteringmodel_gmm_bad):
     '''
     The ratio is smaller the better
     '''
-    ratio = clusteringmodel_gmm_good.score([parameters_value]) / clusteringmodel_gmm_bad.score([parameters_value])
+    ratio = clusteringmodel_gmm_good.score(
+        [parameters_value]) / clusteringmodel_gmm_bad.score([parameters_value])
     sigma = 0
     return ratio, sigma
 
+
 def selection_r(x_bounds,
                 x_types,
                 clusteringmodel_gmm_good,
@@ -60,6 +64,7 @@ def selection_r(x_bounds,
 
     return outputs
 
+
 def selection(x_bounds,
               x_types,
               clusteringmodel_gmm_good,
@@ -69,13 +74,14 @@ def selection(x_bounds,
     '''
     Select the lowest mu value
     '''
-    results = lib_acquisition_function.next_hyperparameter_lowest_mu(\
-                    _ratio_scores, [clusteringmodel_gmm_good, clusteringmodel_gmm_bad],\
-                    x_bounds, x_types, minimize_starting_points, \
-                    minimize_constraints_fun=minimize_constraints_fun)
+    results = lib_acquisition_function.next_hyperparameter_lowest_mu(
+        _ratio_scores, [clusteringmodel_gmm_good, clusteringmodel_gmm_bad],
+        x_bounds, x_types, minimize_starting_points,
+        minimize_constraints_fun=minimize_constraints_fun)
 
     return results
 
+
 def _rand_with_constraints(x_bounds, x_types):
     '''
     Random generate the variable with constraints
@@ -96,6 +102,7 @@ def _rand_with_constraints(x_bounds, x_types):
                 outputs[i] = random.randint(x_bounds[i][0], x_bounds[i][1])
     return outputs
 
+
 def _minimize_constraints_fun_summation(x):
     '''
     Minimize constraints fun summation
diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py
index 7010815b23..24b2e03027 100644
--- a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py
+++ b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py
@@ -17,7 +17,9 @@
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
+"""
+OutlierDectection.py
+"""
 
 import os
 import sys
@@ -30,19 +32,21 @@
 
 
 def _outlierDetection_threaded(inputs):
-    '''
+    """
     Detect the outlier
-    '''
+    """
     [samples_idx, samples_x, samples_y_aggregation] = inputs
-    sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n"\
-                        % (os.path.basename(__file__), samples_idx + 1, len(samples_x)))
+    sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n"
+                     % (os.path.basename(__file__), samples_idx + 1, len(samples_x)))
     outlier = None
 
-    # Create a diagnostic regression model which removes the sample that we want to evaluate
-    diagnostic_regressor_gp = gp_create_model.create_model(\
-                                    samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\
-                                    samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:])
-    mu, sigma = gp_prediction.predict(samples_x[samples_idx], diagnostic_regressor_gp['model'])
+    # Create a diagnostic regression model which removes the sample that we
+    # want to evaluate
+    diagnostic_regressor_gp = gp_create_model.create_model(
+        samples_x[0:samples_idx] + samples_x[samples_idx + 1:],
+        samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:])
+    mu, sigma = gp_prediction.predict(
+        samples_x[samples_idx], diagnostic_regressor_gp['model'])
 
     # 2.33 is the z-score for 98% confidence level
     if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma):
@@ -52,16 +56,18 @@ def _outlierDetection_threaded(inputs):
                    "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}
     return outlier
 
+
 def outlierDetection_threaded(samples_x, samples_y_aggregation):
-    '''
+    """
     Use Multi-thread to detect the outlier
-    '''
+    """
     outliers = []
 
-    threads_inputs = [[samples_idx, samples_x, samples_y_aggregation]\
-                            for samples_idx in range(0, len(samples_x))]
+    threads_inputs = [[samples_idx, samples_x, samples_y_aggregation]
+                      for samples_idx in range(0, len(samples_x))]
     threads_pool = ThreadPool(min(4, len(threads_inputs)))
-    threads_results = threads_pool.map(_outlierDetection_threaded, threads_inputs)
+    threads_results = threads_pool.map(
+        _outlierDetection_threaded, threads_inputs)
     threads_pool.close()
     threads_pool.join()
 
@@ -69,15 +75,13 @@ def outlierDetection_threaded(samples_x, samples_y_aggregation):
         if threads_result is not None:
             outliers.append(threads_result)
         else:
-            print("error here.")
+            print("Error: threads_result is None.")
 
     outliers = outliers if outliers else None
     return outliers
 
+
 def outlierDetection(samples_x, samples_y_aggregation):
-    '''
-    TODO
-    '''
     outliers = []
     for samples_idx, _ in enumerate(samples_x):
         #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n"
@@ -92,7 +96,8 @@ def outlierDetection(samples_x, samples_y_aggregation):
             outliers.append({"samples_idx": samples_idx,
                              "expected_mu": mu,
                              "expected_sigma": sigma,
-                             "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)})
+                             "difference": \
+                                abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)})
 
     outliers = outliers if outliers else None
     return outliers
diff --git a/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py b/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py
index 8beff1a6e6..476323c93f 100644
--- a/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py
+++ b/src/sdk/pynni/nni/metis_tuner/lib_acquisition_function.py
@@ -16,7 +16,11 @@
 # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+"""
+lib_acquisition_function.py
+"""
 
 import sys
 import numpy
@@ -33,9 +37,9 @@ def next_hyperparameter_expected_improvement(fun_prediction,
                                              samples_y_aggregation,
                                              minimize_starting_points,
                                              minimize_constraints_fun=None):
-    '''
+    """
     "Expected Improvement" acquisition function
-    '''
+    """
     best_x = None
     best_acquisition_value = None
     x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds]
@@ -70,6 +74,7 @@ def next_hyperparameter_expected_improvement(fun_prediction,
 
     return outputs
 
+
 def _expected_improvement(x, fun_prediction, fun_prediction_args,
                           x_bounds, x_types, samples_y_aggregation,
                           minimize_constraints_fun):
@@ -77,7 +82,8 @@ def _expected_improvement(x, fun_prediction, fun_prediction_args,
     x = lib_data.match_val_type(x, x_bounds, x_types)
 
     expected_improvement = sys.maxsize
-    if (minimize_constraints_fun is None) or (minimize_constraints_fun(x) is True):
+    if (minimize_constraints_fun is None) or (
+            minimize_constraints_fun(x) is True):
         mu, sigma = fun_prediction(x, *fun_prediction_args)
 
         loss_optimum = min(samples_y_aggregation)
@@ -87,7 +93,7 @@ def _expected_improvement(x, fun_prediction, fun_prediction_args,
         with numpy.errstate(divide="ignore"):
             Z = scaling_factor * (mu - loss_optimum) / sigma
             expected_improvement = scaling_factor * (mu - loss_optimum) * \
-                                        norm.cdf(Z) + sigma * norm.pdf(Z)
+                norm.cdf(Z) + sigma * norm.pdf(Z)
             expected_improvement = 0.0 if sigma == 0.0 else expected_improvement
 
         # We want expected_improvement to be as large as possible
@@ -101,9 +107,9 @@ def next_hyperparameter_lowest_confidence(fun_prediction,
                                           x_bounds, x_types,
                                           minimize_starting_points,
                                           minimize_constraints_fun=None):
-    '''
+    """
     "Lowest Confidence" acquisition function
-    '''
+    """
     best_x = None
     best_acquisition_value = None
     x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds]
@@ -120,10 +126,12 @@ def next_hyperparameter_lowest_confidence(fun_prediction,
                              x_types,
                              minimize_constraints_fun))
 
-        if (best_acquisition_value) is None or (res.fun < best_acquisition_value):
+        if (best_acquisition_value) is None or (
+                res.fun < best_acquisition_value):
             res.x = numpy.ndarray.tolist(res.x)
             res.x = lib_data.match_val_type(res.x, x_bounds, x_types)
-            if (minimize_constraints_fun is None) or (minimize_constraints_fun(res.x) is True):
+            if (minimize_constraints_fun is None) or (
+                    minimize_constraints_fun(res.x) is True):
                 best_acquisition_value = res.fun
                 best_x = res.x
 
@@ -134,13 +142,15 @@ def next_hyperparameter_lowest_confidence(fun_prediction,
                    'expected_sigma': sigma, 'acquisition_func': "lc"}
     return outputs
 
+
 def _lowest_confidence(x, fun_prediction, fun_prediction_args,
                        x_bounds, x_types, minimize_constraints_fun):
     # This is only for step-wise optimization
     x = lib_data.match_val_type(x, x_bounds, x_types)
 
     ci = sys.maxsize
-    if (minimize_constraints_fun is None) or (minimize_constraints_fun(x) is True):
+    if (minimize_constraints_fun is None) or (
+            minimize_constraints_fun(x) is True):
         mu, sigma = fun_prediction(x, *fun_prediction_args)
         ci = (sigma * 1.96 * 2) / mu
         # We want ci to be as large as possible
@@ -156,9 +166,9 @@ def next_hyperparameter_lowest_mu(fun_prediction,
                                   x_bounds, x_types,
                                   minimize_starting_points,
                                   minimize_constraints_fun=None):
-    '''
+    """
     "Lowest Mu" acquisition function
-    '''
+    """
     best_x = None
     best_acquisition_value = None
     x_bounds_minmax = [[i[0], i[-1]] for i in x_bounds]
@@ -169,13 +179,15 @@ def next_hyperparameter_lowest_mu(fun_prediction,
                        x0=starting_point.reshape(1, -1),
                        bounds=x_bounds_minmax,
                        method="L-BFGS-B",
-                       args=(fun_prediction, fun_prediction_args, \
+                       args=(fun_prediction, fun_prediction_args,
                              x_bounds, x_types, minimize_constraints_fun))
 
-        if (best_acquisition_value is None) or (res.fun < best_acquisition_value):
+        if (best_acquisition_value is None) or (
+                res.fun < best_acquisition_value):
             res.x = numpy.ndarray.tolist(res.x)
             res.x = lib_data.match_val_type(res.x, x_bounds, x_types)
-            if (minimize_constraints_fun is None) or (minimize_constraints_fun(res.x) is True):
+            if (minimize_constraints_fun is None) or (
+                    minimize_constraints_fun(res.x) is True):
                 best_acquisition_value = res.fun
                 best_x = res.x
 
@@ -189,14 +201,14 @@ def next_hyperparameter_lowest_mu(fun_prediction,
 
 def _lowest_mu(x, fun_prediction, fun_prediction_args,
                x_bounds, x_types, minimize_constraints_fun):
-    '''
+    """
     Calculate the lowest mu
-    '''
+    """
     # This is only for step-wise optimization
     x = lib_data.match_val_type(x, x_bounds, x_types)
 
     mu = sys.maxsize
-    if (minimize_constraints_fun is None) or (minimize_constraints_fun(x) is True):
+    if (minimize_constraints_fun is None) or (
+            minimize_constraints_fun(x) is True):
         mu, _ = fun_prediction(x, *fun_prediction_args)
     return mu
-    
\ No newline at end of file
diff --git a/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py b/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py
index 1e9daaee95..cc385e9afc 100644
--- a/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py
+++ b/src/sdk/pynni/nni/metis_tuner/lib_constraint_summation.py
@@ -16,7 +16,11 @@
 # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+"""
+lib_constraint_summation.py
+"""
 
 import math
 import random
@@ -39,6 +43,7 @@ def check_feasibility(x_bounds, lowerbound, upperbound):
     return (x_bounds_lowerbound <= lowerbound <= x_bounds_upperbound) or \
            (x_bounds_lowerbound <= upperbound <= x_bounds_upperbound)
 
+
 def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100):
     '''
     Key idea is that we try to move towards upperbound, by randomly choose one
@@ -55,7 +60,8 @@ def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100):
             if x_types[i] == "discrete_int":
                 x_idx_sorted.append([i, len(x_bounds[i])])
             elif (x_types[i] == "range_int") or (x_types[i] == "range_continuous"):
-                x_idx_sorted.append([i, math.floor(x_bounds[i][1] - x_bounds[i][0])])
+                x_idx_sorted.append(
+                    [i, math.floor(x_bounds[i][1] - x_bounds[i][0])])
         x_idx_sorted = sorted(x_idx_sorted, key=itemgetter(1))
 
         for _ in range(max_retries):
@@ -77,12 +83,13 @@ def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100):
                                     temp.append(j)
                             # Randomly pick a number from the integer array
                             if temp:
-                                outputs[x_idx] = temp[random.randint(0, len(temp) - 1)]
+                                outputs[x_idx] = temp[random.randint(
+                                    0, len(temp) - 1)]
 
                         elif (x_types[x_idx] == "range_int") or \
-                                    (x_types[x_idx] == "range_continuous"):
-                            outputs[x_idx] = random.randint(x_bounds[x_idx][0],
-                                                            min(x_bounds[x_idx][-1], budget_max))
+                                (x_types[x_idx] == "range_continuous"):
+                            outputs[x_idx] = random.randint(
+                                x_bounds[x_idx][0], min(x_bounds[x_idx][-1], budget_max))
 
                 else:
                     # The last x that we need to assign a random number
@@ -91,26 +98,28 @@ def rand(x_bounds, x_types, lowerbound, upperbound, max_retries=100):
 
                     # This check:
                     # is our smallest possible value going to overflow the available budget space,
-                    # and is our largest possible value going to underflow the lower bound
+                    # and is our largest possible value going to underflow the
+                    # lower bound
                     if (x_bounds[x_idx][0] <= budget_max) and \
                             (x_bounds[x_idx][-1] >= randint_lowerbound):
                         if x_types[x_idx] == "discrete_int":
                             temp = []
                             for j in x_bounds[x_idx]:
-                                # if (j <= budget_max) and (j >= randint_lowerbound):
+                                # if (j <= budget_max) and (j >=
+                                # randint_lowerbound):
                                 if randint_lowerbound <= j <= budget_max:
                                     temp.append(j)
                             if temp:
-                                outputs[x_idx] = temp[random.randint(0, len(temp) - 1)]
+                                outputs[x_idx] = temp[random.randint(
+                                    0, len(temp) - 1)]
                         elif (x_types[x_idx] == "range_int") or \
                                 (x_types[x_idx] == "range_continuous"):
-                            outputs[x_idx] = random.randint(randint_lowerbound,
-                                                            min(x_bounds[x_idx][1], budget_max))
+                            outputs[x_idx] = random.randint(
+                                randint_lowerbound, min(
+                                    x_bounds[x_idx][1], budget_max))
                 if outputs[x_idx] is None:
                     break
-                else:
-                    budget_allocated += outputs[x_idx]
+                budget_allocated += outputs[x_idx]
             if None not in outputs:
                 break
     return outputs
-    
\ No newline at end of file
diff --git a/src/sdk/pynni/nni/metis_tuner/lib_data.py b/src/sdk/pynni/nni/metis_tuner/lib_data.py
index 6256dfc69a..0ff6dbdc55 100644
--- a/src/sdk/pynni/nni/metis_tuner/lib_data.py
+++ b/src/sdk/pynni/nni/metis_tuner/lib_data.py
@@ -16,7 +16,8 @@
 # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
 
 import math
 import random
@@ -56,7 +57,7 @@ def rand(x_bounds, x_types):
             temp = x_bounds[i][random.randint(0, len(x_bounds[i]) - 1)]
             outputs.append(temp)
         elif x_types[i] == "range_int":
-            temp = random.randint(x_bounds[i][0], x_bounds[i][1] -1)
+            temp = random.randint(x_bounds[i][0], x_bounds[i][1] - 1)
             outputs.append(temp)
         elif x_types[i] == "range_continuous":
             temp = random.uniform(x_bounds[i][0], x_bounds[i][1])
diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py
index d9ac3415fb..600cca99b5 100644
--- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py
+++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py
@@ -16,7 +16,11 @@
 # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+"""
+metis_tuner.py
+"""
 
 import copy
 import logging
@@ -51,10 +55,45 @@ class MetisTuner(Tuner):
 
     More algorithm information you could reference here:
     https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/
+
+    Attributes
+    ----------
+        optimize_mode : str
+            optimize_mode is a string that including two mode "maximize" and "minimize"
+
+        no_resampling : bool
+            True or False.
+            Should Metis consider re-sampling as part of the search strategy?
+            If you are confident that the training dataset is noise-free,
+            then you do not need re-sampling.
+
+        no_candidates : bool
+            True or False.
+            Should Metis suggest parameters for the next benchmark?
+            If you do not plan to do more benchmarks,
+            Metis can skip this step.
+
+        selection_num_starting_points : int
+            How many times Metis should try to find the global optimal in the search space?
+            The higher the number, the longer it takes to output the solution.
+
+        cold_start_num : int
+            Metis need some trial result to get cold start.
+            when the number of trial result is less than
+            cold_start_num, Metis will randomly sample hyper-parameter for trial.
+
+        exploration_probability: float
+            The probability of Metis to select parameter from exploration instead of exploitation.
     """
 
-    def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=False,
-                 selection_num_starting_points=600, cold_start_num=10, exploration_probability=0.9):
+    def __init__(
+            self,
+            optimize_mode="maximize",
+            no_resampling=True,
+            no_candidates=False,
+            selection_num_starting_points=600,
+            cold_start_num=10,
+            exploration_probability=0.9):
         """
         Parameters
         ----------
@@ -62,23 +101,34 @@ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=F
             optimize_mode is a string that including two mode "maximize" and "minimize"
 
         no_resampling : bool
-            True or False. Should Metis consider re-sampling as part of the search strategy?
-        If you are confident that the training dataset is noise-free, then you do not need re-sampling.
-
-        no_candidates: bool
-            True or False. Should Metis suggest parameters for the next benchmark?
-        If you do not plan to do more benchmarks, Metis can skip this step.
-
-        selection_num_starting_points: int
-            how many times Metis should try to find the global optimal in the search space?
-        The higher the number, the longer it takes to output the solution.
+            True or False.
+            Should Metis consider re-sampling as part of the search strategy?
+            If you are confident that the training dataset is noise-free,
+            then you do not need re-sampling.
+
+        no_candidates : bool
+            True or False.
+            Should Metis suggest parameters for the next benchmark?
+            If you do not plan to do more benchmarks,
+            Metis can skip this step.
+
+        selection_num_starting_points : int
+            How many times Metis should try to find the global optimal in the search space?
+            The higher the number, the longer it takes to output the solution.
+
+        cold_start_num : int
+            Metis need some trial result to get cold start.
+            when the number of trial result is less than
+            cold_start_num, Metis will randomly sample hyper-parameter for trial.
+
+        exploration_probability : float
+            The probability of Metis to select parameter from exploration instead of exploitation.
 
-        cold_start_num: int
-            Metis need some trial result to get cold start. when the number of trial result is less than
-        cold_start_num, Metis will randomly sample hyper-parameter for trial.
+        x_bounds : list
+            The constration of parameters.
 
-        exploration_probability: float
-            The probability of Metis to select parameter from exploration instead of exploitation.
+        x_types : list
+            The type of parameters.
         """
 
         self.samples_x = []
@@ -101,7 +151,8 @@ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=F
 
 
     def update_search_space(self, search_space):
-        """Update the self.x_bounds and self.x_types by the search_space.json
+        """
+        Update the self.x_bounds and self.x_types by the search_space.json
 
         Parameters
         ----------
@@ -120,12 +171,20 @@ def update_search_space(self, search_space):
                 key_range = search_space[key]['_value']
                 idx = self.key_order.index(key)
                 if key_type == 'quniform':
-                    if key_range[2] == 1 and key_range[0].is_integer() and key_range[1].is_integer():
-                        self.x_bounds[idx] = [key_range[0], key_range[1]+1]
+                    if key_range[2] == 1 and key_range[0].is_integer(
+                    ) and key_range[1].is_integer():
+                        self.x_bounds[idx] = [key_range[0], key_range[1] + 1]
                         self.x_types[idx] = 'range_int'
                     else:
                         low, high, q = key_range
-                        bounds = np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high)
+                        bounds = np.clip(
+                            np.arange(
+                                np.round(
+                                    low / q),
+                                np.round(
+                                    high / q) + 1) * q,
+                            low,
+                            high)
                         self.x_bounds[idx] = bounds
                         self.x_types[idx] = 'discrete_int'
                 elif key_type == 'randint':
@@ -139,22 +198,28 @@ def update_search_space(self, search_space):
 
                     for key_value in key_range:
                         if not isinstance(key_value, (int, float)):
-                            raise RuntimeError("Metis Tuner only support numerical choice.")
+                            raise RuntimeError(
+                                "Metis Tuner only support numerical choice.")
 
                     self.x_types[idx] = 'discrete_int'
                 else:
-                    logger.info("Metis Tuner doesn't support this kind of variable: %s", key_type)
-                    raise RuntimeError("Metis Tuner doesn't support this kind of variable: " + str(key_type))
+                    logger.info(
+                        "Metis Tuner doesn't support this kind of variable: %s",
+                        str(key_type))
+                    raise RuntimeError(
+                        "Metis Tuner doesn't support this kind of variable: %s" %
+                        str(key_type))
         else:
             logger.info("The format of search space is not a dict.")
             raise RuntimeError("The format of search space is not a dict.")
 
-        self.minimize_starting_points = _rand_init(self.x_bounds, self.x_types, \
-                                                        self.selection_num_starting_points)
+        self.minimize_starting_points = _rand_init(
+            self.x_bounds, self.x_types, self.selection_num_starting_points)
 
 
     def _pack_output(self, init_parameter):
-        """Pack the output
+        """
+        Pack the output
 
         Parameters
         ----------
@@ -167,14 +232,18 @@ def _pack_output(self, init_parameter):
         output = {}
         for i, param in enumerate(init_parameter):
             output[self.key_order[i]] = param
+
         return output
 
 
     def generate_parameters(self, parameter_id, **kwargs):
-        """Generate next parameter for trial
+        """
+        Generate next parameter for trial
+
         If the number of trial result is lower than cold start number,
         metis will first random generate some parameters.
-        Otherwise, metis will choose the parameters by the Gussian Process Model and the Gussian Mixture Model.
+        Otherwise, metis will choose the parameters by
+        the Gussian Process Model and the Gussian Mixture Model.
 
         Parameters
         ----------
@@ -188,26 +257,34 @@ def generate_parameters(self, parameter_id, **kwargs):
             init_parameter = _rand_init(self.x_bounds, self.x_types, 1)[0]
             results = self._pack_output(init_parameter)
         else:
-            self.minimize_starting_points = _rand_init(self.x_bounds, self.x_types, \
-                                                       self.selection_num_starting_points)
-            results = self._selection(self.samples_x, self.samples_y_aggregation, self.samples_y,
-                                      self.x_bounds, self.x_types,
-                                      threshold_samplessize_resampling=(None if self.no_resampling is True else 50),
-                                      no_candidates=self.no_candidates,
-                                      minimize_starting_points=self.minimize_starting_points,
-                                      minimize_constraints_fun=self.minimize_constraints_fun)
-
-        logger.info("Generate paramageters:\n%s", results)
+            self.minimize_starting_points = _rand_init(
+                self.x_bounds, self.x_types, self.selection_num_starting_points)
+            results = self._selection(
+                self.samples_x,
+                self.samples_y_aggregation,
+                self.samples_y,
+                self.x_bounds,
+                self.x_types,
+                threshold_samplessize_resampling=(
+                    None if self.no_resampling is True else 50),
+                no_candidates=self.no_candidates,
+                minimize_starting_points=self.minimize_starting_points,
+                minimize_constraints_fun=self.minimize_constraints_fun)
+
+        logger.info("Generate paramageters: \n%s", str(results))
         return results
 
 
     def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
-        """Tuner receive result from trial.
+        """
+        Tuner receive result from trial.
 
         Parameters
         ----------
         parameter_id : int
+            The id of parameters, generated by nni manager.
         parameters : dict
+            A group of parameters that trial has tried.
         value : dict/float
             if value is dict, it should have "default" key.
         """
@@ -216,8 +293,8 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
             value = -value
 
         logger.info("Received trial result.")
-        logger.info("value is :%s", value)
-        logger.info("parameter is : %s", parameters)
+        logger.info("value is : %s", str(value))
+        logger.info("parameter is : %s", str(parameters))
 
         # parse parameter to sample_x
         sample_x = [0 for i in range(len(self.key_order))]
@@ -244,11 +321,19 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
             self.samples_y_aggregation.append([value])
 
 
-    def _selection(self, samples_x, samples_y_aggregation, samples_y,
-                   x_bounds, x_types, max_resampling_per_x=3,
-                   threshold_samplessize_exploitation=12,
-                   threshold_samplessize_resampling=50, no_candidates=False,
-                   minimize_starting_points=None, minimize_constraints_fun=None):
+    def _selection(
+            self,
+            samples_x,
+            samples_y_aggregation,
+            samples_y,
+            x_bounds,
+            x_types,
+            max_resampling_per_x=3,
+            threshold_samplessize_exploitation=12,
+            threshold_samplessize_resampling=50,
+            no_candidates=False,
+            minimize_starting_points=None,
+            minimize_constraints_fun=None):
 
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
@@ -259,7 +344,8 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
         samples_size_unique = len(samples_y)
 
         # ===== STEP 1: Compute the current optimum =====
-        gp_model = gp_create_model.create_model(samples_x, samples_y_aggregation)
+        gp_model = gp_create_model.create_model(
+            samples_x, samples_y_aggregation)
         lm_current = gp_selection.selection(
             "lm",
             samples_y_aggregation,
@@ -278,7 +364,7 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
         })
 
         if no_candidates is False:
-            # ===== STEP 2: Get recommended configurations for exploration =====
+            # ===== STEP 2: Get recommended configurations for exploration ====
             results_exploration = gp_selection.selection(
                 "lc",
                 samples_y_aggregation,
@@ -303,25 +389,31 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
             else:
                 logger.info("DEBUG: No suitable exploration candidates were")
 
-            # ===== STEP 3: Get recommended configurations for exploitation =====
+            # ===== STEP 3: Get recommended configurations for exploitation ===
             if samples_size_all >= threshold_samplessize_exploitation:
                 logger.info("Getting candidates for exploitation...\n")
                 try:
-                    gmm = gmm_create_model.create_model(samples_x, samples_y_aggregation)
+                    gmm = gmm_create_model.create_model(
+                        samples_x, samples_y_aggregation)
 
                     if ("discrete_int" in x_types) or ("range_int" in x_types):
-                        results_exploitation = gmm_selection.selection(x_bounds, x_types,
-                                                                       gmm['clusteringmodel_good'],
-                                                                       gmm['clusteringmodel_bad'],
-                                                                       minimize_starting_points,
-                                                                       minimize_constraints_fun=minimize_constraints_fun)
+                        results_exploitation = gmm_selection.selection(
+                            x_bounds,
+                            x_types,
+                            gmm['clusteringmodel_good'],
+                            gmm['clusteringmodel_bad'],
+                            minimize_starting_points,
+                            minimize_constraints_fun=minimize_constraints_fun)
                     else:
-                        # If all parameters are of "range_continuous", let's use GMM to generate random starting points
-                        results_exploitation = gmm_selection.selection_r(x_bounds, x_types,
-                                                                         gmm['clusteringmodel_good'],
-                                                                         gmm['clusteringmodel_bad'],
-                                                                         num_starting_points=self.selection_num_starting_points,
-                                                                         minimize_constraints_fun=minimize_constraints_fun)
+                        # If all parameters are of "range_continuous",
+                        # let's use GMM to generate random starting points
+                        results_exploitation = gmm_selection.selection_r(
+                            x_bounds,
+                            x_types,
+                            gmm['clusteringmodel_good'],
+                            gmm['clusteringmodel_bad'],
+                            num_starting_points=self.selection_num_starting_points,
+                            minimize_constraints_fun=minimize_constraints_fun)
 
                     if results_exploitation is not None:
                         if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0:
@@ -335,24 +427,30 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
                             }
                             candidates.append(temp_candidate)
 
-                            logger.info("DEBUG: 1 exploitation_gmm candidate selected\n")
+                            logger.info(
+                                "DEBUG: 1 exploitation_gmm candidate selected\n")
                             logger.info(temp_candidate)
                     else:
-                        logger.info("DEBUG: No suitable exploitation_gmm candidates were found\n")
+                        logger.info(
+                            "DEBUG: No suitable exploitation_gmm candidates were found\n")
 
                 except ValueError as exception:
                     # The exception: ValueError: Fitting the mixture model failed
                     # because some components have ill-defined empirical covariance
                     # (for instance caused by singleton or collapsed samples).
-                    # Try to decrease the number of components, or increase reg_covar.
-                    logger.info("DEBUG: No suitable exploitation_gmm candidates were found due to exception.")
+                    # Try to decrease the number of components, or increase
+                    # reg_covar.
+                    logger.info(
+                        "DEBUG: No suitable exploitation_gmm \
+                        candidates were found due to exception.")
                     logger.info(exception)
 
             # ===== STEP 4: Get a list of outliers =====
             if (threshold_samplessize_resampling is not None) and \
-                        (samples_size_unique >= threshold_samplessize_resampling):
+                    (samples_size_unique >= threshold_samplessize_resampling):
                 logger.info("Getting candidates for re-sampling...\n")
-                results_outliers = gp_outlier_detection.outlierDetection_threaded(samples_x, samples_y_aggregation)
+                results_outliers = gp_outlier_detection.outlierDetection_threaded(
+                    samples_x, samples_y_aggregation)
 
                 if results_outliers is not None:
                     for results_outlier in results_outliers:  # pylint: disable=not-an-iterable
@@ -365,11 +463,13 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
                     logger.info("DEBUG: %d re-sampling candidates selected\n")
                     logger.info(temp_candidate)
                 else:
-                    logger.info("DEBUG: No suitable resampling candidates were found\n")
+                    logger.info(
+                        "DEBUG: No suitable resampling candidates were found\n")
 
             if candidates:
-                # ===== STEP 5: Compute the information gain of each candidate towards the optimum =====
-                logger.info("Evaluating information gain of %d candidates...\n")
+                # ===== STEP 5: Compute the information gain of each candidate
+                logger.info(
+                    "Evaluating information gain of %d candidates...\n")
                 next_improvement = 0
 
                 threads_inputs = [[
@@ -377,36 +477,45 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
                     minimize_constraints_fun, minimize_starting_points
                 ] for candidate in candidates]
                 threads_pool = ThreadPool(4)
-                # Evaluate what would happen if we actually sample each candidate
-                threads_results = threads_pool.map(_calculate_lowest_mu_threaded, threads_inputs)
+                # Evaluate what would happen if we actually sample each
+                # candidate
+                threads_results = threads_pool.map(
+                    _calculate_lowest_mu_threaded, threads_inputs)
                 threads_pool.close()
                 threads_pool.join()
 
                 for threads_result in threads_results:
                     if threads_result['expected_lowest_mu'] < lm_current['expected_mu']:
                         # Information gain
-                        temp_improvement = threads_result['expected_lowest_mu'] - lm_current['expected_mu']
+                        temp_improvement = threads_result['expected_lowest_mu'] - \
+                            lm_current['expected_mu']
 
                         if next_improvement > temp_improvement:
                             next_improvement = temp_improvement
                             next_candidate = threads_result['candidate']
             else:
-                # ===== STEP 6: If we have no candidates, randomly pick one =====
+                # ===== STEP 6: If we have no candidates, randomly pick one ===
                 logger.info(
                     "DEBUG: No candidates from exploration, exploitation,\
                                  and resampling. We will random a candidate for next_candidate\n"
                 )
 
-                next_candidate = _rand_with_constraints(x_bounds, x_types) \
-                                    if minimize_starting_points is None else minimize_starting_points[0]
-                next_candidate = lib_data.match_val_type(next_candidate, x_bounds, x_types)
-                expected_mu, expected_sigma = gp_prediction.predict(next_candidate, gp_model['model'])
-                next_candidate = {'hyperparameter': next_candidate, 'reason': "random",
-                                  'expected_mu': expected_mu, 'expected_sigma': expected_sigma}
-
-        # ===== STEP 7 =====
-        # If current optimal hyperparameter occurs in the history or exploration probability is less than the threshold,
-        # take next config as exploration step
+                next_candidate = _rand_with_constraints(
+                    x_bounds,
+                    x_types) if minimize_starting_points is None else minimize_starting_points[0]
+                next_candidate = lib_data.match_val_type(
+                    next_candidate, x_bounds, x_types)
+                expected_mu, expected_sigma = gp_prediction.predict(
+                    next_candidate, gp_model['model'])
+                next_candidate = {
+                    'hyperparameter': next_candidate,
+                    'reason': "random",
+                    'expected_mu': expected_mu,
+                    'expected_sigma': expected_sigma}
+
+        # STEP 7: If current optimal hyperparameter occurs in the history
+        # or exploration probability is less than the threshold, take next
+        # config as exploration step
         outputs = self._pack_output(lm_current['hyperparameter'])
         ap = random.uniform(0, 1)
         if outputs in self.total_data or ap <= self.exploration_probability:
@@ -419,11 +528,13 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
         return outputs
 
     def import_data(self, data):
-        """Import additional data for tuning
+        """
+        Import additional data for tuning
+
         Parameters
         ----------
-        data:
-            a list of dictionarys, each of which has at least two keys, 'parameter' and 'value'
+        data : a list of dict
+               each of which has at least two keys: 'parameter' and 'value'.
         """
         _completed_num = 0
         for trial_info in data:
@@ -437,18 +548,26 @@ def import_data(self, data):
                 logger.info("Useless trial data, value is %s, skip this trial data.", _value)
                 continue
             self.supplement_data_num += 1
-            _parameter_id = '_'.join(["ImportData", str(self.supplement_data_num)])
+            _parameter_id = '_'.join(
+                ["ImportData", str(self.supplement_data_num)])
             self.total_data.append(_params)
-            self.receive_trial_result(parameter_id=_parameter_id, parameters=_params, value=_value)
+            self.receive_trial_result(
+                parameter_id=_parameter_id,
+                parameters=_params,
+                value=_value)
         logger.info("Successfully import data to metis tuner.")
 
+
 def _rand_with_constraints(x_bounds, x_types):
     outputs = None
     x_bounds_withconstraints = [x_bounds[i] for i in CONSTRAINT_PARAMS_IDX]
     x_types_withconstraints = [x_types[i] for i in CONSTRAINT_PARAMS_IDX]
 
-    x_val_withconstraints = lib_constraint_summation.rand(x_bounds_withconstraints,\
-                                x_types_withconstraints, CONSTRAINT_LOWERBOUND, CONSTRAINT_UPPERBOUND)
+    x_val_withconstraints = lib_constraint_summation.rand(
+        x_bounds_withconstraints,
+        x_types_withconstraints,
+        CONSTRAINT_LOWERBOUND,
+        CONSTRAINT_UPPERBOUND)
     if not x_val_withconstraints:
         outputs = [None] * len(x_bounds)
 
@@ -462,12 +581,18 @@ def _rand_with_constraints(x_bounds, x_types):
 
 
 def _calculate_lowest_mu_threaded(inputs):
-    [candidate, samples_x, samples_y, x_bounds, x_types, minimize_constraints_fun, minimize_starting_points] = inputs
+    [candidate, samples_x, samples_y, x_bounds, x_types,
+     minimize_constraints_fun, minimize_starting_points] = inputs
 
     outputs = {"candidate": candidate, "expected_lowest_mu": None}
 
-    for expected_mu in [candidate['expected_mu'] + 1.96 * candidate['expected_sigma'],
-                        candidate['expected_mu'] - 1.96 * candidate['expected_sigma']]:
+    for expected_mu in [
+            candidate['expected_mu'] +
+            1.96 *
+            candidate['expected_sigma'],
+            candidate['expected_mu'] -
+            1.96 *
+            candidate['expected_sigma']]:
         temp_samples_x = copy.deepcopy(samples_x)
         temp_samples_y = copy.deepcopy(samples_y)
 
@@ -480,8 +605,10 @@ def _calculate_lowest_mu_threaded(inputs):
             temp_samples_y.append([expected_mu])
 
         # Aggregates multiple observation of the sample sampling points
-        temp_y_aggregation = [statistics.median(temp_sample_y) for temp_sample_y in temp_samples_y]
-        temp_gp = gp_create_model.create_model(temp_samples_x, temp_y_aggregation)
+        temp_y_aggregation = [statistics.median(
+            temp_sample_y) for temp_sample_y in temp_samples_y]
+        temp_gp = gp_create_model.create_model(
+            temp_samples_x, temp_y_aggregation)
         temp_results = gp_selection.selection(
             "lm",
             temp_y_aggregation,
@@ -491,7 +618,8 @@ def _calculate_lowest_mu_threaded(inputs):
             minimize_starting_points,
             minimize_constraints_fun=minimize_constraints_fun)
 
-        if outputs["expected_lowest_mu"] is None or outputs["expected_lowest_mu"] > temp_results['expected_mu']:
+        if outputs["expected_lowest_mu"] is None \
+            or outputs["expected_lowest_mu"] > temp_results['expected_mu']:
             outputs["expected_lowest_mu"] = temp_results['expected_mu']
 
     return outputs
@@ -510,18 +638,19 @@ def _rand_init(x_bounds, x_types, selection_num_starting_points):
     '''
     Random sample some init seed within bounds.
     '''
-    return [lib_data.rand(x_bounds, x_types) for i \
-                    in range(0, selection_num_starting_points)]
+    return [lib_data.rand(x_bounds, x_types) for i
+            in range(0, selection_num_starting_points)]
 
 
 def get_median(temp_list):
-    """Return median
+    """
+    Return median
     """
     num = len(temp_list)
     temp_list.sort()
     print(temp_list)
     if num % 2 == 0:
-        median = (temp_list[int(num/2)] + temp_list[int(num/2) - 1]) / 2
+        median = (temp_list[int(num / 2)] + temp_list[int(num / 2) - 1]) / 2
     else:
-        median = temp_list[int(num/2)]
+        median = temp_list[int(num / 2)]
     return median
diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py b/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py
index 360771139a..15c5e83cdd 100644
--- a/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py
+++ b/src/sdk/pynni/nni/networkmorphism_tuner/bayesian.py
@@ -38,7 +38,7 @@
 def layer_distance(a, b):
     """The distance between two layers."""
     # pylint: disable=unidiomatic-typecheck
-    if type(a) != type(b):
+    if not isinstance(a, type(b)):
         return 1.0
     if is_layer(a, "Conv"):
         att_diff = [
@@ -96,7 +96,8 @@ def skip_connection_distance(a, b):
         return 1.0
     len_a = abs(a[1] - a[0])
     len_b = abs(b[1] - b[0])
-    return (abs(a[0] - b[0]) + abs(len_a - len_b)) / (max(a[0], b[0]) + max(len_a, len_b))
+    return (abs(a[0] - b[0]) + abs(len_a - len_b)) / \
+        (max(a[0], b[0]) + max(len_a, len_b))
 
 
 def skip_connections_distance(list_a, list_b):
@@ -161,7 +162,8 @@ def fit(self, train_x, train_y):
     def incremental_fit(self, train_x, train_y):
         """ Incrementally fit the regressor. """
         if not self._first_fitted:
-            raise ValueError("The first_fit function needs to be called first.")
+            raise ValueError(
+                "The first_fit function needs to be called first.")
 
         train_x, train_y = np.array(train_x), np.array(train_y)
 
@@ -174,7 +176,7 @@ def incremental_fit(self, train_x, train_y):
         temp_distance_matrix = np.concatenate((up_k, down_k), axis=0)
         k_matrix = bourgain_embedding_matrix(temp_distance_matrix)
         diagonal = np.diag_indices_from(k_matrix)
-        diagonal = (diagonal[0][-len(train_x) :], diagonal[1][-len(train_x) :])
+        diagonal = (diagonal[0][-len(train_x):], diagonal[1][-len(train_x):])
         k_matrix[diagonal] += self.alpha
 
         try:
@@ -186,7 +188,8 @@ def incremental_fit(self, train_x, train_y):
         self._y = np.concatenate((self._y, train_y), axis=0)
         self._distance_matrix = temp_distance_matrix
 
-        self._alpha_vector = cho_solve((self._l_matrix, True), self._y)  # Line 3
+        self._alpha_vector = cho_solve(
+            (self._l_matrix, True), self._y)  # Line 3
 
         return self
 
@@ -209,7 +212,8 @@ def first_fit(self, train_x, train_y):
 
         self._l_matrix = cholesky(k_matrix, lower=True)  # Line 2
 
-        self._alpha_vector = cho_solve((self._l_matrix, True), self._y)  # Line 3
+        self._alpha_vector = cho_solve(
+            (self._l_matrix, True), self._y)  # Line 3
 
         self._first_fitted = True
         return self
@@ -227,7 +231,9 @@ def predict(self, train_x):
 
         # compute inverse K_inv of K based on its Cholesky
         # decomposition L and its inverse L_inv
-        l_inv = solve_triangular(self._l_matrix.T, np.eye(self._l_matrix.shape[0]))
+        l_inv = solve_triangular(
+            self._l_matrix.T, np.eye(
+                self._l_matrix.shape[0]))
         k_inv = l_inv.dot(l_inv.T)
         # Compute variance of predictive distribution
         y_var = np.ones(len(train_x), dtype=np.float)
@@ -378,7 +384,11 @@ def generate(self, descriptors):
                         continue
 
                     temp_acq_value = self.acq(temp_graph)
-                    pq.put(elem_class(temp_acq_value, elem.father_id, temp_graph))
+                    pq.put(
+                        elem_class(
+                            temp_acq_value,
+                            elem.father_id,
+                            temp_graph))
                     descriptors.append(temp_graph.extract_descriptor())
                     if self._accept_new_acq_value(opt_acq, temp_acq_value):
                         opt_acq = temp_acq_value
diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/graph.py b/src/sdk/pynni/nni/networkmorphism_tuner/graph.py
index 3d951a1965..abf825c48a 100644
--- a/src/sdk/pynni/nni/networkmorphism_tuner/graph.py
+++ b/src/sdk/pynni/nni/networkmorphism_tuner/graph.py
@@ -249,7 +249,8 @@ def _redirect_edge(self, u_id, v_id, new_v_id):
                 self.reverse_adj_list[v_id].remove(edge_tuple)
                 break
         self.reverse_adj_list[new_v_id].append((u_id, layer_id))
-        for index, value in enumerate(self.layer_id_to_output_node_ids[layer_id]):
+        for index, value in enumerate(
+                self.layer_id_to_output_node_ids[layer_id]):
             if value == v_id:
                 self.layer_id_to_output_node_ids[layer_id][index] = new_v_id
                 break
@@ -350,7 +351,8 @@ def _search(self, u, start_dim, total_dim, n_add):
                 self._replace_layer(layer_id, new_layer)
 
             elif is_layer(layer, "BatchNormalization"):
-                new_layer = wider_bn(layer, start_dim, total_dim, n_add, self.weighted)
+                new_layer = wider_bn(
+                    layer, start_dim, total_dim, n_add, self.weighted)
                 self._replace_layer(layer_id, new_layer)
                 self._search(v, start_dim, total_dim, n_add)
 
@@ -405,7 +407,8 @@ def to_deeper_model(self, target_id, new_layer):
             target_id: A convolutional layer ID. The new block should be inserted after the block.
             new_layer: An instance of StubLayer subclasses.
         """
-        self.operation_history.append(("to_deeper_model", target_id, new_layer))
+        self.operation_history.append(
+            ("to_deeper_model", target_id, new_layer))
         input_id = self.layer_id_to_input_node_ids[target_id][0]
         output_id = self.layer_id_to_output_node_ids[target_id][0]
         if self.weighted:
@@ -478,14 +481,20 @@ def to_add_skip_model(self, start_id, end_id):
         pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0]
         end_node_id = self.layer_id_to_output_node_ids[end_id][0]
 
-        skip_output_id = self._insert_pooling_layer_chain(start_node_id, end_node_id)
+        skip_output_id = self._insert_pooling_layer_chain(
+            start_node_id, end_node_id)
 
         # Add the conv layer
-        new_conv_layer = get_conv_class(self.n_dim)(filters_start, filters_end, 1)
+        new_conv_layer = get_conv_class(
+            self.n_dim)(
+                filters_start,
+                filters_end,
+                1)
         skip_output_id = self.add_layer(new_conv_layer, skip_output_id)
 
         # Add the add layer.
-        add_input_node_id = self._add_node(deepcopy(self.node_list[end_node_id]))
+        add_input_node_id = self._add_node(
+            deepcopy(self.node_list[end_node_id]))
         add_layer = StubAdd()
 
         self._redirect_edge(pre_end_node_id, end_node_id, add_input_node_id)
@@ -504,7 +513,8 @@ def to_add_skip_model(self, start_id, end_id):
             weights = np.zeros((filters_end, filters_start) + filter_shape)
             bias = np.zeros(filters_end)
             new_conv_layer.set_weights(
-                (add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
+                (add_noise(weights, np.array([0, 1])), add_noise(
+                    bias, np.array([0, 1])))
             )
 
     def to_concat_skip_model(self, start_id, end_id):
@@ -513,7 +523,8 @@ def to_concat_skip_model(self, start_id, end_id):
             start_id: The convolutional layer ID, after which to start the skip-connection.
             end_id: The convolutional layer ID, after which to end the skip-connection.
         """
-        self.operation_history.append(("to_concat_skip_model", start_id, end_id))
+        self.operation_history.append(
+            ("to_concat_skip_model", start_id, end_id))
         filters_end = self.layer_list[end_id].output.shape[-1]
         filters_start = self.layer_list[start_id].output.shape[-1]
         start_node_id = self.layer_id_to_output_node_ids[start_id][0]
@@ -521,9 +532,11 @@ def to_concat_skip_model(self, start_id, end_id):
         pre_end_node_id = self.layer_id_to_input_node_ids[end_id][0]
         end_node_id = self.layer_id_to_output_node_ids[end_id][0]
 
-        skip_output_id = self._insert_pooling_layer_chain(start_node_id, end_node_id)
+        skip_output_id = self._insert_pooling_layer_chain(
+            start_node_id, end_node_id)
 
-        concat_input_node_id = self._add_node(deepcopy(self.node_list[end_node_id]))
+        concat_input_node_id = self._add_node(
+            deepcopy(self.node_list[end_node_id]))
         self._redirect_edge(pre_end_node_id, end_node_id, concat_input_node_id)
 
         concat_layer = StubConcatenate()
@@ -532,7 +545,10 @@ def to_concat_skip_model(self, start_id, end_id):
             self.node_list[skip_output_id],
         ]
         concat_output_node_id = self._add_node(Node(concat_layer.output_shape))
-        self._add_edge(concat_layer, concat_input_node_id, concat_output_node_id)
+        self._add_edge(
+            concat_layer,
+            concat_input_node_id,
+            concat_output_node_id)
         self._add_edge(concat_layer, skip_output_id, concat_output_node_id)
         concat_layer.output = self.node_list[concat_output_node_id]
         self.node_list[concat_output_node_id].shape = concat_layer.output_shape
@@ -559,7 +575,8 @@ def to_concat_skip_model(self, start_id, end_id):
             )
             bias = np.zeros(filters_end)
             new_conv_layer.set_weights(
-                (add_noise(weights, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
+                (add_noise(weights, np.array([0, 1])), add_noise(
+                    bias, np.array([0, 1])))
             )
 
     def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
@@ -568,7 +585,8 @@ def _insert_pooling_layer_chain(self, start_node_id, end_node_id):
             new_layer = deepcopy(layer)
             if is_layer(new_layer, "Conv"):
                 filters = self.node_list[start_node_id].shape[-1]
-                new_layer = get_conv_class(self.n_dim)(filters, filters, 1, layer.stride)
+                new_layer = get_conv_class(self.n_dim)(
+                    filters, filters, 1, layer.stride)
                 if self.weighted:
                     init_conv_weight(new_layer)
             else:
@@ -601,8 +619,10 @@ def extract_descriptor(self):
                     temp_v = v
                     temp_layer_id = layer_id
                     skip_type = None
-                    while not (temp_v in index_in_main_chain and temp_u in index_in_main_chain):
-                        if is_layer(self.layer_list[temp_layer_id], "Concatenate"):
+                    while not (
+                            temp_v in index_in_main_chain and temp_u in index_in_main_chain):
+                        if is_layer(
+                                self.layer_list[temp_layer_id], "Concatenate"):
                             skip_type = NetworkDescriptor.CONCAT_CONNECT
                         if is_layer(self.layer_list[temp_layer_id], "Add"):
                             skip_type = NetworkDescriptor.ADD_CONNECT
@@ -711,7 +731,8 @@ def deep_layer_ids(self):
 
     def wide_layer_ids(self):
         return (
-            self._conv_layer_ids_in_order()[:-1] + self._dense_layer_ids_in_order()[:-1]
+            self._conv_layer_ids_in_order(
+            )[:-1] + self._dense_layer_ids_in_order()[:-1]
         )
 
     def skip_connection_layer_ids(self):
@@ -810,7 +831,8 @@ def __init__(self, graph):
         topo_node_list = self.graph.topological_order
         output_id = topo_node_list[-1]
         input_id = topo_node_list[0]
-        input_tensor = keras.layers.Input(shape=graph.node_list[input_id].shape)
+        input_tensor = keras.layers.Input(
+            shape=graph.node_list[input_id].shape)
 
         node_list = deepcopy(self.graph.node_list)
         node_list[input_id] = input_tensor
@@ -838,7 +860,8 @@ def __init__(self, graph):
         output_tensor = keras.layers.Activation("softmax", name="activation_add")(
             output_tensor
         )
-        self.model = keras.models.Model(inputs=input_tensor, outputs=output_tensor)
+        self.model = keras.models.Model(
+            inputs=input_tensor, outputs=output_tensor)
 
         if graph.weighted:
             for index, layer in enumerate(self.layers):
@@ -892,7 +915,8 @@ def __init__(self, graph):
 
         for layer_id, item in enumerate(graph.layer_list):
             layer = graph.layer_list[layer_id]
-            layer_information = layer_description_extractor(layer, graph.node_to_id)
+            layer_information = layer_description_extractor(
+                layer, graph.node_to_id)
             layer_list.append((layer_id, layer_information))
 
         data["node_list"] = node_list
@@ -938,7 +962,8 @@ def json_to_graph(json_model: str):
 
     graph.input_shape = input_shape
     vis = json_model["vis"]
-    graph.vis = {tuple(item): True for item in vis} if vis is not None else None
+    graph.vis = {
+        tuple(item): True for item in vis} if vis is not None else None
     graph.weighted = json_model["weighted"]
     layer_id_to_input_node_ids = json_model["layer_id_to_input_node_ids"]
     graph.layer_id_to_input_node_ids = {
diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py b/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py
index a318188f3e..6b36e8ed97 100644
--- a/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py
+++ b/src/sdk/pynni/nni/networkmorphism_tuner/graph_transformer.py
@@ -40,7 +40,8 @@ def to_wider_graph(graph):
     '''
     weighted_layer_ids = graph.wide_layer_ids()
     weighted_layer_ids = list(
-        filter(lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids)
+        filter(
+            lambda x: graph.layer_list[x].output.shape[-1], weighted_layer_ids)
     )
     wider_layers = sample(weighted_layer_ids, 1)
 
@@ -58,12 +59,14 @@ def to_wider_graph(graph):
 def to_skip_connection_graph(graph):
     ''' skip connection graph
     '''
-    # The last conv layer cannot be widen since wider operator cannot be done over the two sides of flatten.
+    # The last conv layer cannot be widen since wider operator cannot be done
+    # over the two sides of flatten.
     weighted_layer_ids = graph.skip_connection_layer_ids()
     valid_connection = []
-    for skip_type in sorted([NetworkDescriptor.ADD_CONNECT, NetworkDescriptor.CONCAT_CONNECT]):
+    for skip_type in sorted(
+            [NetworkDescriptor.ADD_CONNECT, NetworkDescriptor.CONCAT_CONNECT]):
         for index_a in range(len(weighted_layer_ids)):
-            for index_b in range(len(weighted_layer_ids))[index_a + 1 :]:
+            for index_b in range(len(weighted_layer_ids))[index_a + 1:]:
                 valid_connection.append((index_a, index_b, skip_type))
 
     if not valid_connection:
@@ -84,9 +87,14 @@ def create_new_layer(layer, n_dim):
 
     input_shape = layer.output.shape
     dense_deeper_classes = [StubDense, get_dropout_class(n_dim), StubReLU]
-    conv_deeper_classes = [get_conv_class(n_dim), get_batch_norm_class(n_dim), StubReLU]
+    conv_deeper_classes = [
+        get_conv_class(n_dim),
+        get_batch_norm_class(n_dim),
+        StubReLU]
     if is_layer(layer, "ReLU"):
-        conv_deeper_classes = [get_conv_class(n_dim), get_batch_norm_class(n_dim)]
+        conv_deeper_classes = [
+            get_conv_class(n_dim),
+            get_batch_norm_class(n_dim)]
         dense_deeper_classes = [StubDense, get_dropout_class(n_dim)]
     elif is_layer(layer, "Dropout"):
         dense_deeper_classes = [StubDense, StubReLU]
diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py b/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py
index bd580a1cab..ab6d275fbe 100644
--- a/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py
+++ b/src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py
@@ -52,7 +52,8 @@ def deeper_conv_block(conv_layer, kernel_size, weighted=True):
 
     if weighted:
         new_conv_layer.set_weights(
-            (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
+            (add_noise(weight, np.array([0, 1])),
+             add_noise(bias, np.array([0, 1])))
         )
         new_weights = [
             add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])),
@@ -74,7 +75,8 @@ def dense_to_deeper_block(dense_layer, weighted=True):
     new_dense_layer = StubDense(units, units)
     if weighted:
         new_dense_layer.set_weights(
-            (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
+            (add_noise(weight, np.array([0, 1])),
+             add_noise(bias, np.array([0, 1])))
         )
     return [StubReLU(), new_dense_layer]
 
@@ -97,8 +99,11 @@ def wider_pre_dense(layer, n_add, weighted=True):
         teacher_index = rand[i]
         new_weight = teacher_w[teacher_index, :]
         new_weight = new_weight[np.newaxis, :]
-        student_w = np.concatenate((student_w, add_noise(new_weight, student_w)), axis=0)
-        student_b = np.append(student_b, add_noise(teacher_b[teacher_index], student_b))
+        student_w = np.concatenate(
+            (student_w, add_noise(new_weight, student_w)), axis=0)
+        student_b = np.append(
+            student_b, add_noise(
+                teacher_b[teacher_index], student_b))
 
     new_pre_layer = StubDense(layer.input_units, n_units2 + n_add)
     new_pre_layer.set_weights((student_w, student_b))
@@ -209,7 +214,7 @@ def wider_next_dense(layer, start_dim, total_dim, n_add, weighted=True):
             student_w[:, : start_dim * n_units_each_channel],
             add_noise(new_weight, student_w),
             student_w[
-                :, start_dim * n_units_each_channel : total_dim * n_units_each_channel
+                :, start_dim * n_units_each_channel: total_dim * n_units_each_channel
             ],
         ),
         axis=1,
@@ -225,7 +230,8 @@ def add_noise(weights, other_weights):
     '''
     w_range = np.ptp(other_weights.flatten())
     noise_range = NOISE_RATIO * w_range
-    noise = np.random.uniform(-noise_range / 2.0, noise_range / 2.0, weights.shape)
+    noise = np.random.uniform(-noise_range / 2.0,
+                              noise_range / 2.0, weights.shape)
     return np.add(noise, weights)
 
 
@@ -236,7 +242,8 @@ def init_dense_weight(layer):
     weight = np.eye(units)
     bias = np.zeros(units)
     layer.set_weights(
-        (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
+        (add_noise(weight, np.array([0, 1])),
+         add_noise(bias, np.array([0, 1])))
     )
 
 
@@ -256,7 +263,8 @@ def init_conv_weight(layer):
     bias = np.zeros(n_filters)
 
     layer.set_weights(
-        (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
+        (add_noise(weight, np.array([0, 1])),
+         add_noise(bias, np.array([0, 1])))
     )
 
 
diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py b/src/sdk/pynni/nni/networkmorphism_tuner/layers.py
index e2f0ac484c..d9c2e5d99e 100644
--- a/src/sdk/pynni/nni/networkmorphism_tuner/layers.py
+++ b/src/sdk/pynni/nni/networkmorphism_tuner/layers.py
@@ -28,8 +28,10 @@
 
 
 class AvgPool(nn.Module):
-    '''AvgPool Module.
-    '''
+    """
+    AvgPool Module.
+    """
+
     def __init__(self):
         super().__init__()
 
@@ -39,8 +41,10 @@ def forward(self, input_tensor):
 
 
 class GlobalAvgPool1d(AvgPool):
-    '''GlobalAvgPool1d Module.
-    '''
+    """
+    GlobalAvgPool1d Module.
+    """
+
     def forward(self, input_tensor):
         return functional.avg_pool1d(input_tensor, input_tensor.size()[2:]).view(
             input_tensor.size()[:2]
@@ -48,8 +52,10 @@ def forward(self, input_tensor):
 
 
 class GlobalAvgPool2d(AvgPool):
-    '''GlobalAvgPool2d Module.
-    '''
+    """
+    GlobalAvgPool2d Module.
+    """
+
     def forward(self, input_tensor):
         return functional.avg_pool2d(input_tensor, input_tensor.size()[2:]).view(
             input_tensor.size()[:2]
@@ -57,8 +63,10 @@ def forward(self, input_tensor):
 
 
 class GlobalAvgPool3d(AvgPool):
-    '''GlobalAvgPool3d Module.
-    '''
+    """
+    GlobalAvgPool3d Module.
+    """
+
     def forward(self, input_tensor):
         return functional.avg_pool3d(input_tensor, input_tensor.size()[2:]).view(
             input_tensor.size()[:2]
@@ -66,70 +74,86 @@ def forward(self, input_tensor):
 
 
 class StubLayer:
-    '''StubLayer Module. Base Module.
-    '''
+    """
+    StubLayer Module. Base Module.
+    """
+
     def __init__(self, input_node=None, output_node=None):
         self.input = input_node
         self.output = output_node
         self.weights = None
 
     def build(self, shape):
-        '''build shape.
-        '''
+        """
+        build shape.
+        """
 
     def set_weights(self, weights):
-        '''set weights.
-        '''
+        """
+        set weights.
+        """
         self.weights = weights
 
     def import_weights(self, torch_layer):
-        '''import weights.
-        '''
+        """
+        import weights.
+        """
 
     def import_weights_keras(self, keras_layer):
-        '''import weights from keras layer.
-        '''
+        """
+        import weights from keras layer.
+        """
 
     def export_weights(self, torch_layer):
-        '''export weights.
-        '''
+        """
+        export weights.
+        """
 
     def export_weights_keras(self, keras_layer):
-        '''export weights to keras layer.
-        '''
+        """
+        export weights to keras layer.
+        """
 
     def get_weights(self):
-        '''get weights.
-        '''
+        """
+        get weights.
+        """
         return self.weights
 
     def size(self):
-        '''size().
-        '''
+        """
+        size().
+        """
         return 0
 
     @property
     def output_shape(self):
-        '''output shape.
-        '''
+        """
+        output shape.
+        """
         return self.input.shape
 
     def to_real_layer(self):
-        '''to real layer.
-        '''
+        """
+        to real layer.
+        """
 
     def __str__(self):
-        '''str() function to print.
-        '''
+        """
+        str() function to print.
+        """
         return type(self).__name__[4:]
 
 
 class StubWeightBiasLayer(StubLayer):
-    '''StubWeightBiasLayer Module to set the bias.
-    '''
+    """
+    StubWeightBiasLayer Module to set the bias.
+    """
+
     def import_weights(self, torch_layer):
         self.set_weights(
-            (torch_layer.weight.data.cpu().numpy(), torch_layer.bias.data.cpu().numpy())
+            (torch_layer.weight.data.cpu().numpy(),
+             torch_layer.bias.data.cpu().numpy())
         )
 
     def import_weights_keras(self, keras_layer):
@@ -144,8 +168,10 @@ def export_weights_keras(self, keras_layer):
 
 
 class StubBatchNormalization(StubWeightBiasLayer):
-    '''StubBatchNormalization Module. Batch Norm.
-    '''
+    """
+    StubBatchNormalization Module. Batch Norm.
+    """
+
     def __init__(self, num_features, input_node=None, output_node=None):
         super().__init__(input_node, output_node)
         self.num_features = num_features
@@ -175,29 +201,37 @@ def to_real_layer(self):
 
 
 class StubBatchNormalization1d(StubBatchNormalization):
-    '''StubBatchNormalization1d Module.
-    '''
+    """
+    StubBatchNormalization1d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.BatchNorm1d(self.num_features)
 
 
 class StubBatchNormalization2d(StubBatchNormalization):
-    '''StubBatchNormalization2d Module.
-    '''
+    """
+    StubBatchNormalization2d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.BatchNorm2d(self.num_features)
 
 
 class StubBatchNormalization3d(StubBatchNormalization):
-    '''StubBatchNormalization3d Module.
-    '''
+    """
+    StubBatchNormalization3d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.BatchNorm3d(self.num_features)
 
 
 class StubDense(StubWeightBiasLayer):
-    '''StubDense Module. Linear.
-    '''
+    """
+    StubDense Module. Linear.
+    """
+
     def __init__(self, input_units, units, input_node=None, output_node=None):
         super().__init__(input_node, output_node)
         self.input_units = input_units
@@ -208,7 +242,9 @@ def output_shape(self):
         return (self.units,)
 
     def import_weights_keras(self, keras_layer):
-        self.set_weights((keras_layer.get_weights()[0].T, keras_layer.get_weights()[1]))
+        self.set_weights(
+            (keras_layer.get_weights()[0].T,
+             keras_layer.get_weights()[1]))
 
     def export_weights_keras(self, keras_layer):
         keras_layer.set_weights((self.weights[0].T, self.weights[1]))
@@ -221,9 +257,12 @@ def to_real_layer(self):
 
 
 class StubConv(StubWeightBiasLayer):
-    '''StubConv Module. Conv.
-    '''
-    def __init__(self, input_channel, filters, kernel_size, stride=1, input_node=None, output_node=None):
+    """
+    StubConv Module. Conv.
+    """
+
+    def __init__(self, input_channel, filters, kernel_size,
+                 stride=1, input_node=None, output_node=None):
         super().__init__(input_node, output_node)
         self.input_channel = input_channel
         self.filters = filters
@@ -242,13 +281,16 @@ def output_shape(self):
         return tuple(ret)
 
     def import_weights_keras(self, keras_layer):
-        self.set_weights((keras_layer.get_weights()[0].T, keras_layer.get_weights()[1]))
+        self.set_weights(
+            (keras_layer.get_weights()[0].T,
+             keras_layer.get_weights()[1]))
 
     def export_weights_keras(self, keras_layer):
         keras_layer.set_weights((self.weights[0].T, self.weights[1]))
 
     def size(self):
-        return (self.input_channel * self.kernel_size * self.kernel_size + 1) * self.filters
+        return (self.input_channel * self.kernel_size *
+                self.kernel_size + 1) * self.filters
 
     @abstractmethod
     def to_real_layer(self):
@@ -272,8 +314,10 @@ def __str__(self):
 
 
 class StubConv1d(StubConv):
-    '''StubConv1d Module.
-    '''
+    """
+    StubConv1d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.Conv1d(
             self.input_channel,
@@ -285,8 +329,10 @@ def to_real_layer(self):
 
 
 class StubConv2d(StubConv):
-    '''StubConv2d Module.
-    '''
+    """
+    StubConv2d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.Conv2d(
             self.input_channel,
@@ -298,8 +344,10 @@ def to_real_layer(self):
 
 
 class StubConv3d(StubConv):
-    '''StubConv3d Module.
-    '''
+    """
+    StubConv3d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.Conv3d(
             self.input_channel,
@@ -311,8 +359,10 @@ def to_real_layer(self):
 
 
 class StubAggregateLayer(StubLayer):
-    '''StubAggregateLayer Module.
-    '''
+    """
+    StubAggregateLayer Module.
+    """
+
     def __init__(self, input_nodes=None, output_node=None):
         if input_nodes is None:
             input_nodes = []
@@ -320,8 +370,8 @@ def __init__(self, input_nodes=None, output_node=None):
 
 
 class StubConcatenate(StubAggregateLayer):
-    '''StubConcatenate Module.
-    '''
+    """StubConcatenate Module.
+    """
     @property
     def output_shape(self):
         ret = 0
@@ -335,8 +385,9 @@ def to_real_layer(self):
 
 
 class StubAdd(StubAggregateLayer):
-    '''StubAdd Module.
-    '''
+    """
+    StubAdd Module.
+    """
     @property
     def output_shape(self):
         return self.input[0].shape
@@ -346,8 +397,9 @@ def to_real_layer(self):
 
 
 class StubFlatten(StubLayer):
-    '''StubFlatten Module.
-    '''
+    """
+    StubFlatten Module.
+    """
     @property
     def output_shape(self):
         ret = 1
@@ -360,22 +412,28 @@ def to_real_layer(self):
 
 
 class StubReLU(StubLayer):
-    '''StubReLU Module.
-    '''
+    """
+    StubReLU Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.ReLU()
 
 
 class StubSoftmax(StubLayer):
-    '''StubSoftmax Module.
-    '''
+    """
+    StubSoftmax Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.LogSoftmax(dim=1)
 
 
 class StubDropout(StubLayer):
-    '''StubDropout Module.
-    '''
+    """
+    StubDropout Module.
+    """
+
     def __init__(self, rate, input_node=None, output_node=None):
         super().__init__(input_node, output_node)
         self.rate = rate
@@ -386,36 +444,45 @@ def to_real_layer(self):
 
 
 class StubDropout1d(StubDropout):
-    '''StubDropout1d Module.
-    '''
+    """
+    StubDropout1d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.Dropout(self.rate)
 
 
 class StubDropout2d(StubDropout):
-    '''StubDropout2d Module.
-    '''
+    """
+    StubDropout2d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.Dropout2d(self.rate)
 
 
 class StubDropout3d(StubDropout):
-    '''StubDropout3d Module.
-    '''
+    """
+    StubDropout3d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.Dropout3d(self.rate)
 
 
 class StubInput(StubLayer):
-    '''StubInput Module.
-    '''
+    """
+    StubInput Module.
+    """
+
     def __init__(self, input_node=None, output_node=None):
         super().__init__(input_node, output_node)
 
 
 class StubPooling(StubLayer):
-    '''StubPooling Module.
-    '''
+    """
+    StubPooling Module.
+    """
 
     def __init__(self,
                  kernel_size=None,
@@ -444,30 +511,37 @@ def to_real_layer(self):
 
 
 class StubPooling1d(StubPooling):
-    '''StubPooling1d Module.
-    '''
+    """
+    StubPooling1d Module.
+    """
 
     def to_real_layer(self):
         return torch.nn.MaxPool1d(self.kernel_size, stride=self.stride)
 
 
 class StubPooling2d(StubPooling):
-    '''StubPooling2d Module.
-    '''
+    """
+    StubPooling2d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.MaxPool2d(self.kernel_size, stride=self.stride)
 
 
 class StubPooling3d(StubPooling):
-    '''StubPooling3d Module.
-    '''
+    """
+    StubPooling3d Module.
+    """
+
     def to_real_layer(self):
         return torch.nn.MaxPool3d(self.kernel_size, stride=self.stride)
 
 
 class StubGlobalPooling(StubLayer):
-    '''StubGlobalPooling Module.
-    '''
+    """
+    StubGlobalPooling Module.
+    """
+
     def __init__(self, input_node=None, output_node=None):
         super().__init__(input_node, output_node)
 
@@ -481,49 +555,63 @@ def to_real_layer(self):
 
 
 class StubGlobalPooling1d(StubGlobalPooling):
-    '''StubGlobalPooling1d Module.
-    '''
+    """
+    StubGlobalPooling1d Module.
+    """
+
     def to_real_layer(self):
         return GlobalAvgPool1d()
 
 
 class StubGlobalPooling2d(StubGlobalPooling):
-    '''StubGlobalPooling2d Module.
-    '''
+    """
+    StubGlobalPooling2d Module.
+    """
+
     def to_real_layer(self):
         return GlobalAvgPool2d()
 
 
 class StubGlobalPooling3d(StubGlobalPooling):
-    '''StubGlobalPooling3d Module.
-    '''
+    """
+    StubGlobalPooling3d Module.
+    """
+
     def to_real_layer(self):
         return GlobalAvgPool3d()
 
 
 class TorchConcatenate(nn.Module):
-    '''TorchConcatenate Module.
-    '''
+    """
+    TorchConcatenate Module.
+    """
+
     def forward(self, input_list):
         return torch.cat(input_list, dim=1)
 
 
 class TorchAdd(nn.Module):
-    '''TorchAdd Module.
-    '''
+    """
+    TorchAdd Module.
+    """
+
     def forward(self, input_list):
         return input_list[0] + input_list[1]
 
 
 class TorchFlatten(nn.Module):
-    '''TorchFlatten Module.
-    '''
+    """
+    TorchFlatten Module.
+    """
+
     def forward(self, input_tensor):
         return input_tensor.view(input_tensor.size(0), -1)
 
+
 def keras_dropout(layer, rate):
-    '''keras dropout layer.
-    '''
+    """
+    Keras dropout layer.
+    """
 
     from keras import layers
 
@@ -539,8 +627,9 @@ def keras_dropout(layer, rate):
 
 
 def to_real_keras_layer(layer):
-    ''' real keras layer.
-    '''
+    """
+    Real keras layer.
+    """
     from keras import layers
 
     if is_layer(layer, "Dense"):
@@ -574,10 +663,14 @@ def to_real_keras_layer(layer):
 
 
 def is_layer(layer, layer_type):
-    '''judge the layer type.
-    Returns:
+    """
+    Judge the layer type.
+
+    Returns
+    -------
+    bool
         boolean -- True or False
-    '''
+    """
 
     if layer_type == "Input":
         return isinstance(layer, StubInput)
@@ -607,8 +700,9 @@ def is_layer(layer, layer_type):
 
 
 def layer_description_extractor(layer, node_to_id):
-    '''get layer description.
-    '''
+    """
+    Get layer description.
+    """
 
     layer_input = layer.input
     layer_output = layer.output
@@ -641,7 +735,8 @@ def layer_description_extractor(layer, node_to_id):
             layer.units,
         ]
     elif isinstance(layer, (StubBatchNormalization,)):
-        return (type(layer).__name__, layer_input, layer_output, layer.num_features)
+        return (type(layer).__name__, layer_input,
+                layer_output, layer.num_features)
     elif isinstance(layer, (StubDropout,)):
         return (type(layer).__name__, layer_input, layer_output, layer.rate)
     elif isinstance(layer, StubPooling):
@@ -658,8 +753,8 @@ def layer_description_extractor(layer, node_to_id):
 
 
 def layer_description_builder(layer_information, id_to_node):
-    '''build layer from description.
-    '''
+    """build layer from description.
+    """
     layer_type = layer_information[0]
 
     layer_input_ids = layer_information[1]
@@ -696,8 +791,9 @@ def layer_description_builder(layer_information, id_to_node):
 
 
 def layer_width(layer):
-    '''get layer width.
-    '''
+    """
+    Get layer width.
+    """
 
     if is_layer(layer, "Dense"):
         return layer.units
diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py
index 893e718041..a5bdec98cb 100644
--- a/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py
+++ b/src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py
@@ -17,11 +17,13 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
 # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 # ==================================================================================================
+"""
+networkmorphsim_tuner.py
+"""
 
 import logging
 import os
 
-
 from nni.tuner import Tuner
 from nni.utils import OptimizeMode, extract_scalar_reward
 from nni.networkmorphism_tuner.bayesian import BayesianOptimizer
@@ -34,7 +36,35 @@
 
 
 class NetworkMorphismTuner(Tuner):
-    """NetworkMorphismTuner is a tuner which using network morphism techniques."""
+    """
+    NetworkMorphismTuner is a tuner which using network morphism techniques.
+
+    Attributes
+    ----------
+    n_classes : int
+        The class number or output node number (default: ``10``)
+    input_shape : tuple
+        A tuple including: (input_width, input_width, input_channel)
+    t_min : float
+        The minimum temperature for simulated annealing. (default: ``Constant.T_MIN``)
+    beta : float
+        The beta in acquisition function. (default: ``Constant.BETA``)
+    algorithm_name : str
+        algorithm name used in the network morphism (default: ``"Bayesian"``)
+    optimize_mode : str
+        optimize mode "minimize" or "maximize" (default: ``"minimize"``)
+    verbose : bool
+        verbose to print the log (default: ``True``)
+    bo : BayesianOptimizer
+        The optimizer used in networkmorphsim tuner.
+    max_model_size : int
+        max model size to the graph (default: ``Constant.MAX_MODEL_SIZE``)
+    default_model_len : int
+        default model length (default: ``Constant.MODEL_LEN``)
+    default_model_width : int
+        default model width (default: ``Constant.MODEL_WIDTH``)
+    search_space : dict
+    """
 
     def __init__(
             self,
@@ -52,36 +82,8 @@ def __init__(
             default_model_len=Constant.MODEL_LEN,
             default_model_width=Constant.MODEL_WIDTH,
     ):
-        """ initilizer of the NetworkMorphismTuner.
-
-        Parameters
-        ----------
-        task : str
-            task mode, such as "cv","common" etc. (default: {"cv"})
-        input_width : int
-            input sample shape (default: {32})
-        input_channel : int
-            input sample shape (default: {3})
-        n_output_node : int
-            output node number (default: {10})
-        algorithm_name : str
-            algorithm name used in the network morphism (default: {"Bayesian"})
-        optimize_mode : str
-            optimize mode "minimize" or "maximize" (default: {"minimize"})
-        path : str
-            default mode path to save the model file (default: {"model_path"})
-        verbose : bool
-            verbose to print the log (default: {True})
-        beta : float
-            The beta in acquisition function. (default: {Constant.BETA})
-        t_min : float
-            The minimum temperature for simulated annealing. (default: {Constant.T_MIN})
-        max_model_size : int
-            max model size to the graph (default: {Constant.MAX_MODEL_SIZE})
-        default_model_len : int
-            default model length (default: {Constant.MODEL_LEN})
-        default_model_width : int
-            default model width (default: {Constant.MODEL_WIDTH})
+        """
+        initilizer of the NetworkMorphismTuner.
         """
 
         if not os.path.exists(path):
@@ -92,7 +94,8 @@ def __init__(
         elif task == "common":
             self.generators = [MlpGenerator]
         else:
-            raise NotImplementedError('{} task not supported in List ["cv","common"]')
+            raise NotImplementedError(
+                '{} task not supported in List ["cv","common"]')
 
         self.n_classes = n_output_node
         self.input_shape = (input_width, input_width, input_channel)
@@ -106,7 +109,8 @@ def __init__(
         self.verbose = verbose
         self.model_count = 0
 
-        self.bo = BayesianOptimizer(self, self.t_min, self.optimize_mode, self.beta)
+        self.bo = BayesianOptimizer(
+            self, self.t_min, self.optimize_mode, self.beta)
         self.training_queue = []
         self.descriptors = []
         self.history = []
@@ -117,6 +121,7 @@ def __init__(
 
         self.search_space = dict()
 
+
     def update_search_space(self, search_space):
         """
         Update search space definition in tuner by search_space in neural architecture.
@@ -140,7 +145,8 @@ def generate_parameters(self, parameter_id, **kwargs):
             new_father_id, generated_graph = self.generate()
             new_model_id = self.model_count
             self.model_count += 1
-            self.training_queue.append((generated_graph, new_father_id, new_model_id))
+            self.training_queue.append(
+                (generated_graph, new_father_id, new_model_id))
             self.descriptors.append(generated_graph.extract_descriptor())
 
         graph, father_id, model_id = self.training_queue.pop(0)
@@ -153,12 +159,15 @@ def generate_parameters(self, parameter_id, **kwargs):
         return json_out
 
     def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
-        """ Record an observation of the objective function.
+        """
+        Record an observation of the objective function.
 
         Parameters
         ----------
         parameter_id : int
+            the id of a group of paramters that generated by nni manager.
         parameters : dict
+            A group of parameters.
         value : dict/float
             if value is dict, it should have "default" key.
         """
@@ -175,8 +184,11 @@ def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
         self.add_model(reward, model_id)
         self.update(father_id, graph, reward, model_id)
 
+
     def init_search(self):
-        """Call the generators to generate the initial architectures for the search."""
+        """
+        Call the generators to generate the initial architectures for the search.
+        """
         if self.verbose:
             logger.info("Initializing search.")
         for generator in self.generators:
@@ -191,14 +203,16 @@ def init_search(self):
         if self.verbose:
             logger.info("Initialization finished.")
 
+
     def generate(self):
-        """Generate the next neural architecture.
+        """
+        Generate the next neural architecture.
 
         Returns
         -------
-        other_info: any object
+        other_info : any object
             Anything to be saved in the training queue together with the architecture.
-        generated_graph: Graph
+        generated_graph : Graph
             An instance of Graph.
         """
         generated_graph, new_father_id = self.bo.generate(self.descriptors)
@@ -211,7 +225,8 @@ def generate(self):
         return new_father_id, generated_graph
 
     def update(self, other_info, graph, metric_value, model_id):
-        """ Update the controller with evaluation result of a neural architecture.
+        """
+        Update the controller with evaluation result of a neural architecture.
 
         Parameters
         ----------
@@ -228,7 +243,8 @@ def update(self, other_info, graph, metric_value, model_id):
         self.bo.add_child(father_id, model_id)
 
     def add_model(self, metric_value, model_id):
-        """ Add model to the history, x_queue and y_queue
+        """
+        Add model to the history, x_queue and y_queue
 
         Parameters
         ----------
@@ -252,16 +268,21 @@ def add_model(self, metric_value, model_id):
             file.close()
         return ret
 
+
     def get_best_model_id(self):
-        """ Get the best model_id from history using the metric value
+        """
+        Get the best model_id from history using the metric value
         """
 
         if self.optimize_mode is OptimizeMode.Maximize:
-            return max(self.history, key=lambda x: x["metric_value"])["model_id"]
+            return max(self.history, key=lambda x: x["metric_value"])[
+                "model_id"]
         return min(self.history, key=lambda x: x["metric_value"])["model_id"]
 
+
     def load_model_by_id(self, model_id):
-        """Get the model by model_id
+        """
+        Get the model by model_id
 
         Parameters
         ----------
@@ -281,7 +302,8 @@ def load_model_by_id(self, model_id):
         return load_model
 
     def load_best_model(self):
-        """ Get the best model by model id
+        """
+        Get the best model by model id
 
         Returns
         -------
@@ -291,7 +313,8 @@ def load_best_model(self):
         return self.load_model_by_id(self.get_best_model_id())
 
     def get_metric_value_by_id(self, model_id):
-        """ Get the model metric valud by its model_id
+        """
+        Get the model metric valud by its model_id
 
         Parameters
         ----------
diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/nn.py b/src/sdk/pynni/nni/networkmorphism_tuner/nn.py
index 363c06be5a..2e820ab2c2 100644
--- a/src/sdk/pynni/nni/networkmorphism_tuner/nn.py
+++ b/src/sdk/pynni/nni/networkmorphism_tuner/nn.py
@@ -92,17 +92,25 @@ def generate(self, model_len=None, model_width=None):
         for i in range(model_len):
             output_node_id = graph.add_layer(StubReLU(), output_node_id)
             output_node_id = graph.add_layer(
-                self.batch_norm(graph.node_list[output_node_id].shape[-1]), output_node_id
+                self.batch_norm(
+                    graph.node_list[output_node_id].shape[-1]), output_node_id
             )
             output_node_id = graph.add_layer(
-                self.conv(temp_input_channel, model_width, kernel_size=3, stride=stride),
+                self.conv(
+                    temp_input_channel,
+                    model_width,
+                    kernel_size=3,
+                    stride=stride),
                 output_node_id,
             )
             temp_input_channel = model_width
-            if pooling_len == 0 or ((i + 1) % pooling_len == 0 and i != model_len - 1):
-                output_node_id = graph.add_layer(self.pooling(), output_node_id)
+            if pooling_len == 0 or (
+                    (i + 1) % pooling_len == 0 and i != model_len - 1):
+                output_node_id = graph.add_layer(
+                    self.pooling(), output_node_id)
 
-        output_node_id = graph.add_layer(self.global_avg_pooling(), output_node_id)
+        output_node_id = graph.add_layer(
+            self.global_avg_pooling(), output_node_id)
         output_node_id = graph.add_layer(
             self.dropout(Constant.CONV_DROPOUT_RATE), output_node_id
         )
@@ -111,7 +119,11 @@ def generate(self, model_len=None, model_width=None):
             output_node_id,
         )
         output_node_id = graph.add_layer(StubReLU(), output_node_id)
-        graph.add_layer(StubDense(model_width, self.n_output_node), output_node_id)
+        graph.add_layer(
+            StubDense(
+                model_width,
+                self.n_output_node),
+            output_node_id)
         return graph
 
 
@@ -145,7 +157,8 @@ def generate(self, model_len=None, model_width=None):
         if model_width is None:
             model_width = Constant.MODEL_WIDTH
         if isinstance(model_width, list) and not len(model_width) == model_len:
-            raise ValueError("The length of 'model_width' does not match 'model_len'")
+            raise ValueError(
+                "The length of 'model_width' does not match 'model_len'")
         elif isinstance(model_width, int):
             model_width = [model_width] * model_len
 
@@ -162,5 +175,9 @@ def generate(self, model_len=None, model_width=None):
             output_node_id = graph.add_layer(StubReLU(), output_node_id)
             n_nodes_prev_layer = width
 
-        graph.add_layer(StubDense(n_nodes_prev_layer, self.n_output_node), output_node_id)
+        graph.add_layer(
+            StubDense(
+                n_nodes_prev_layer,
+                self.n_output_node),
+            output_node_id)
         return graph
diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/test_networkmorphism_tuner.py b/src/sdk/pynni/nni/networkmorphism_tuner/test_networkmorphism_tuner.py
index 09bbe820a9..5da56c487f 100644
--- a/src/sdk/pynni/nni/networkmorphism_tuner/test_networkmorphism_tuner.py
+++ b/src/sdk/pynni/nni/networkmorphism_tuner/test_networkmorphism_tuner.py
@@ -59,9 +59,12 @@ def test_graph_json_transform(self):
             graph_recover.layer_id_to_input_node_ids,
         )
         self.assertEqual(graph_init.adj_list, graph_recover.adj_list)
-        self.assertEqual(graph_init.reverse_adj_list, graph_recover.reverse_adj_list)
         self.assertEqual(
-            len(graph_init.operation_history), len(graph_recover.operation_history)
+            graph_init.reverse_adj_list,
+            graph_recover.reverse_adj_list)
+        self.assertEqual(
+            len(graph_init.operation_history), len(
+                graph_recover.operation_history)
         )
         self.assertEqual(graph_init.n_dim, graph_recover.n_dim)
         self.assertEqual(graph_init.conv, graph_recover.conv)
@@ -71,7 +74,8 @@ def test_graph_json_transform(self):
         node_list_init = [node.shape for node in graph_init.node_list]
         node_list_recover = [node.shape for node in graph_recover.node_list]
         self.assertEqual(node_list_init, node_list_recover)
-        self.assertEqual(len(graph_init.node_to_id), len(graph_recover.node_to_id))
+        self.assertEqual(len(graph_init.node_to_id),
+                         len(graph_recover.node_to_id))
         layer_list_init = [
             layer_description_extractor(item, graph_init.node_to_id)
             for item in graph_init.layer_list
@@ -82,7 +86,8 @@ def test_graph_json_transform(self):
         ]
         self.assertEqual(layer_list_init, layer_list_recover)
 
-        node_to_id_init = [graph_init.node_to_id[node] for node in graph_init.node_list]
+        node_to_id_init = [graph_init.node_to_id[node]
+                           for node in graph_init.node_list]
         node_to_id_recover = [
             graph_recover.node_to_id[node] for node in graph_recover.node_list
         ]
@@ -192,8 +197,8 @@ def test_get_best_model_id(self):
         """
 
         tuner = NetworkMorphismTuner()
-        tuner.add_model(0.8,  0)
-        tuner.add_model(0.9,  1)
+        tuner.add_model(0.8, 0)
+        tuner.add_model(0.9, 1)
         self.assertEqual(tuner.get_best_model_id(), 1)
 
 

From 9dec51e2ba04030d0f092ecc2ad61fab9bc331a6 Mon Sep 17 00:00:00 2001
From: SparkSnail <shinyang@microsoft.com>
Date: Mon, 11 Nov 2019 11:19:25 +0800
Subject: [PATCH 09/10] Support space in logDir (#1694)

---
 .../common/experimentStartupInfo.ts           |  2 +-
 .../training_service/common/util.ts           | 28 +++++++++----------
 .../local/localTrainingService.ts             | 12 ++++----
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/nni_manager/common/experimentStartupInfo.ts b/src/nni_manager/common/experimentStartupInfo.ts
index 5675facdde..ba8bd8789b 100644
--- a/src/nni_manager/common/experimentStartupInfo.ts
+++ b/src/nni_manager/common/experimentStartupInfo.ts
@@ -43,7 +43,7 @@ class ExperimentStartupInfo {
         this.initialized = true;
 
         if (logDir !== undefined && logDir.length > 0) {
-            this.logDir = path.join(logDir, getExperimentId());
+            this.logDir = path.join(path.normalize(logDir), getExperimentId());
         } else {
             this.logDir = path.join(os.homedir(), 'nni', 'experiments', getExperimentId());
         }
diff --git a/src/nni_manager/training_service/common/util.ts b/src/nni_manager/training_service/common/util.ts
index ef05ac57b3..294728ee6d 100644
--- a/src/nni_manager/training_service/common/util.ts
+++ b/src/nni_manager/training_service/common/util.ts
@@ -70,11 +70,11 @@ export async function validateCodeDir(codeDir: string) : Promise<number> {
  */
 export async function execMkdir(directory: string, share: boolean = false): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe New-Item -Path ${directory} -ItemType "directory" -Force`);
+        await cpp.exec(`powershell.exe New-Item -Path "${directory}" -ItemType "directory" -Force`);
     } else if (share) {
-        await cpp.exec(`(umask 0; mkdir -p ${directory})`);
+        await cpp.exec(`(umask 0; mkdir -p '${directory}')`);
     } else {
-        await cpp.exec(`mkdir -p ${directory}`);
+        await cpp.exec(`mkdir -p '${directory}'`);
     }
 
     return Promise.resolve();
@@ -87,9 +87,9 @@ export async function execMkdir(directory: string, share: boolean = false): Prom
  */
 export async function execCopydir(source: string, destination: string): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe Copy-Item ${source} -Destination ${destination} -Recurse`);
+        await cpp.exec(`powershell.exe Copy-Item "${source}" -Destination "${destination}" -Recurse`);
     } else {
-        await cpp.exec(`cp -r ${source} ${destination}`);
+        await cpp.exec(`cp -r '${source}' '${destination}'`);
     }
 
     return Promise.resolve();
@@ -101,9 +101,9 @@ export async function execCopydir(source: string, destination: string): Promise<
  */
 export async function execNewFile(filename: string): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe New-Item -Path ${filename} -ItemType "file" -Force`);
+        await cpp.exec(`powershell.exe New-Item -Path "${filename}" -ItemType "file" -Force`);
     } else {
-        await cpp.exec(`touch ${filename}`);
+        await cpp.exec(`touch '${filename}'`);
     }
 
     return Promise.resolve();
@@ -115,9 +115,9 @@ export async function execNewFile(filename: string): Promise<void> {
  */
 export function runScript(filePath: string): cp.ChildProcess {
     if (process.platform === 'win32') {
-        return cp.exec(`powershell.exe -ExecutionPolicy Bypass -file ${filePath}`);
+        return cp.exec(`powershell.exe -ExecutionPolicy Bypass -file "${filePath}"`);
     } else {
-        return cp.exec(`bash ${filePath}`);
+        return cp.exec(`bash '${filePath}'`);
     }
 }
 
@@ -128,9 +128,9 @@ export function runScript(filePath: string): cp.ChildProcess {
 export async function execTail(filePath: string): Promise<cpp.childProcessPromise.Result> {
     let cmdresult: cpp.childProcessPromise.Result;
     if (process.platform === 'win32') {
-        cmdresult = await cpp.exec(`powershell.exe Get-Content ${filePath} -Tail 1`);
+        cmdresult = await cpp.exec(`powershell.exe Get-Content "${filePath}" -Tail 1`);
     } else {
-        cmdresult = await cpp.exec(`tail -n 1 ${filePath}`);
+        cmdresult = await cpp.exec(`tail -n 1 '${filePath}'`);
     }
 
     return Promise.resolve(cmdresult);
@@ -142,9 +142,9 @@ export async function execTail(filePath: string): Promise<cpp.childProcessPromis
  */
 export async function execRemove(directory: string): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe Remove-Item ${directory} -Recurse -Force`);
+        await cpp.exec(`powershell.exe Remove-Item "${directory}" -Recurse -Force`);
     } else {
-        await cpp.exec(`rm -rf ${directory}`);
+        await cpp.exec(`rm -rf '${directory}'`);
     }
 
     return Promise.resolve();
@@ -173,7 +173,7 @@ export function setEnvironmentVariable(variable: { key: string; value: string })
     if (process.platform === 'win32') {
         return `$env:${variable.key}="${variable.value}"`;
     } else {
-        return `export ${variable.key}=${variable.value}`;
+        return `export ${variable.key}='${variable.value}'`;
     }
 }
 
diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts
index 2d4d1a1745..17cfc1fab9 100644
--- a/src/nni_manager/training_service/local/localTrainingService.ts
+++ b/src/nni_manager/training_service/local/localTrainingService.ts
@@ -490,18 +490,18 @@ class LocalTrainingService implements TrainingService {
         const script: string[] = [];
         if (process.platform === 'win32') {
             script.push(
-                `cmd.exe /c ${localTrialConfig.command} 2>${path.join(workingDirectory, 'stderr')}`,
+                `cmd.exe /c ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`,
                 `$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`,
                 `$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`,
-                `Write $LASTEXITCODE " " $NOW_DATE  | Out-File ${path.join(workingDirectory, '.nni', 'state')} -NoNewline -encoding utf8`);
+                `Write $LASTEXITCODE " " $NOW_DATE  | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`);
         } else {
-            script.push(`eval ${localTrialConfig.command} 2>${path.join(workingDirectory, 'stderr')}`);
+            script.push(`eval ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`);
             if (process.platform === 'darwin') {
                 // https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
                 // Considering the worst case, write 999 to avoid negative duration
-                script.push(`echo $? \`date +%s999\` >${path.join(workingDirectory, '.nni', 'state')}`);
+                script.push(`echo $? \`date +%s999\` >'${path.join(workingDirectory, '.nni', 'state')}'`);
             } else {
-                script.push(`echo $? \`date +%s%3N\` >${path.join(workingDirectory, '.nni', 'state')}`);
+                script.push(`echo $? \`date +%s%3N\` >'${path.join(workingDirectory, '.nni', 'state')}'`);
             }
         }
 
@@ -522,7 +522,7 @@ class LocalTrainingService implements TrainingService {
         if (process.platform !== 'win32') {
             runScriptContent.push('#!/bin/bash');
         }
-        runScriptContent.push(`cd ${this.localTrialConfig.codeDir}`);
+        runScriptContent.push(`cd '${this.localTrialConfig.codeDir}'`);
         for (const variable of variables) {
             runScriptContent.push(setEnvironmentVariable(variable));
         }

From 0e3906aaf5d34684294b64effffecd6afedf4831 Mon Sep 17 00:00:00 2001
From: xuehui <xuehui@microsoft.com>
Date: Mon, 11 Nov 2019 11:20:07 +0800
Subject: [PATCH 10/10] update gbdt docs (#1720)

---
 docs/en_US/TrialExample/GbdtExample.md | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/docs/en_US/TrialExample/GbdtExample.md b/docs/en_US/TrialExample/GbdtExample.md
index 22a94f48bb..95ab71d06f 100644
--- a/docs/en_US/TrialExample/GbdtExample.md
+++ b/docs/en_US/TrialExample/GbdtExample.md
@@ -44,7 +44,15 @@ Given the features and label in train data, we train a GBDT regression model and
 
 ## 3. How to run in nni
 
-### 3.1 Prepare your trial code
+
+### 3.1 Install all the requirments
+
+```
+pip install lightgbm
+pip install pandas
+```
+
+### 3.2 Prepare your trial code
 
 You need to prepare a basic code as following:
 
@@ -86,7 +94,7 @@ if __name__ == '__main__':
     run(lgb_train, lgb_eval, PARAMS, X_test, y_test)
 ```
 
-### 3.2 Prepare your search space.
+### 3.3 Prepare your search space.
 If you like to tune `num_leaves`, `learning_rate`, `bagging_fraction` and `bagging_freq`, you could write a [search_space.json](https://github.com/Microsoft/nni/blob/master/examples/trials/auto-gbdt/search_space.json) as follow:
 
 ```json
@@ -100,7 +108,7 @@ If you like to tune `num_leaves`, `learning_rate`, `bagging_fraction` and `baggi
 
 More support variable type you could reference [here](../Tutorial/SearchSpaceSpec.md).
 
-### 3.3 Add SDK of nni into your code.
+### 3.4 Add SDK of nni into your code.
 
 ```diff
 +import nni
@@ -146,7 +154,7 @@ if __name__ == '__main__':
     run(lgb_train, lgb_eval, PARAMS, X_test, y_test)
 ```
 
-### 3.4 Write a config file and run it.
+### 3.5 Write a config file and run it.
 
 In the config file, you could set some settings including: