pull code

Makefile

@@ -52,6 +52,7 @@ $(shell mkdir -p $(NNI_DEPENDENCY_FOLDER))
 NNI_NODE_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64.tar.xz
 NNI_NODE_FOLDER = $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64
 NNI_NODE ?= $(BIN_FOLDER)/node
+NNI_NPM ?= $(BIN_FOLDER)/npm
 NNI_YARN_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn.tar.gz
 NNI_YARN_FOLDER ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn
 NNI_YARN ?= PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn
@@ -149,8 +150,9 @@ install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL)
 	mkdir $(NNI_NODE_FOLDER)
 	tar -xf $(NNI_NODE_TARBALL) -C $(NNI_NODE_FOLDER) --strip-components 1
 	mkdir -p $(BIN_FOLDER)
-	rm -f $(NNI_NODE)
-	cp $(NNI_NODE_FOLDER)/bin/node $(NNI_NODE)
+	rm -f $(NNI_NODE) $(NNI_NPM)
+	ln -s $(NNI_NODE_FOLDER)/bin/node $(NNI_NODE)
+	ln -s $(NNI_NODE_FOLDER)/bin/npm $(NNI_NPM)
 
 	#$(_INFO) Extracting Yarn $(_END)
 	rm -rf $(NNI_YARN_FOLDER)

README_zh_CN.md

@@ -359,4 +359,4 @@ You can use these commands to get more information about the experiment
 
 ## **许可协议**
 
-代码库遵循 [MIT 许可协议](LICENSE)
+代码库遵循 [MIT 许可协议](LICENSE)

azure-pipelines.yml

@@ -12,6 +12,7 @@ jobs:
       python3 -m pip install --upgrade pip setuptools --user
       python3 -m pip install pylint==2.3.1 astroid==2.2.5 --user
       python3 -m pip install coverage --user
+      echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
     displayName: 'Install python tools'
   - script: |
       source install.sh
@@ -23,8 +24,8 @@ jobs:
       python3 -m pip install keras==2.1.6 --user
       python3 -m pip install gym onnx --user
       sudo apt-get install swig -y
-      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
-      PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
+      nnictl package install --name=SMAC
+      nnictl package install --name=BOHB
     displayName: 'Install dependencies'
   - script: |
       set -e
@@ -42,25 +43,23 @@ jobs:
     displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
-      sudo apt install -y swig
-      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
       source unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 naive_test.py
+      python3 naive_test.py
     displayName: 'Naive test'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 tuner_test.py
+      python3 tuner_test.py
     displayName: 'Built-in tuners / assessors tests'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
+      python3 metrics_test.py
     displayName: 'Trial job metrics test'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
+      python3 cli_test.py
     displayName: 'nnicli test'
 
 - job: 'basic_test_pr_macOS'
@@ -74,33 +73,34 @@ jobs:
   steps:
   - script: python3 -m pip install --upgrade pip setuptools
     displayName: 'Install python tools'
-  - script: |
-      python3 -m pip install torch==0.4.1 --user
-      python3 -m pip install torchvision==0.2.1 --user
-      python3 -m pip install tensorflow==1.13.1 --user
-    displayName: 'Install dependencies'
   - script: |
       source install.sh
+      echo "##vso[task.setvariable variable=PATH]${HOME}/Library/Python/3.7/bin:${PATH}"
     displayName: 'Install nni toolkit via source code'
   - script: |
-      cd test
+      python3 -m pip install torch==0.4.1 --user
+      python3 -m pip install torchvision==0.2.1 --user
+      python3 -m pip install tensorflow==1.13.1 --user
       ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" < /dev/null 2> /dev/null
       brew install swig@3
       ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig
-      PATH=$HOME/Library/Python/3.7/bin:$PATH nnictl package install --name=SMAC
-      PATH=$HOME/Library/Python/3.7/bin:$PATH source unittest.sh
+      nnictl package install --name=SMAC
+    displayName: 'Install dependencies'
+  - script: |
+      cd test
+      source unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 naive_test.py
+      python3 naive_test.py
     displayName: 'Naive test'
   - script: |
       cd test
-      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 tuner_test.py
+      python3 tuner_test.py
     displayName: 'Built-in tuners / assessors tests'
   - script: |
       cd test
-      PATH=$HOME/Library/Python/3.7/bin:$PATH python3 cli_test.py
+      python3 cli_test.py
     displayName: 'nnicli test'
 
 - job: 'basic_test_pr_Windows'

docs/en_US/CommunitySharings/TuningSystems.md

@@ -0,0 +1,10 @@
+# Automatically tune systems with NNI
+
+As computer systems and networking get increasingly complicated, optimizing them manually with explicit rules and heuristics becomes harder than ever before, sometimes impossible. Below are two examples of tuning systems with NNI. Anyone can easily tune their own systems by following them.
+
+* [Tuning RocksDB with NNI](../TrialExample/RocksdbExamples.md)
+* [Tuning parameters of SPTAG (Space Partition Tree And Graph) with NNI](SptagAutoTune.md)
+
+Please see [this paper](https://dl.acm.org/citation.cfm?id=3352031) for more details:
+
+Mike Liang, Chieh-Jan, et al. "The Case for Learning-and-System Co-design." ACM SIGOPS Operating Systems Review 53.1 (2019): 68-74.

docs/en_US/Compressor/Overview.md

@@ -95,7 +95,17 @@ pruner.update_epoch(epoch)
 
 The other is `step`, it can be called with `pruner.step()` after each minibatch. Note that not all algorithms need these two APIs, for those that do not need them, calling them is allowed but has no effect.
 
-__[TODO]__ The last API is for users to export the compressed model. You will get a compressed model when you finish the training using this API. It also exports another file storing the values of masks.
+You can easily export the compressed model using the following API if you are pruning your model, ```state_dict``` of the sparse model weights will be stored in ```model.pth```, which can be loaded by ```torch.load('model.pth')```
+
+```
+pruner.export_model(model_path='model.pth')
+```
+
+```mask_dict ``` and pruned model in ```onnx``` format(```input_shape``` need to be specified) can also be exported like this:
+
+```python
+pruner.export_model(model_path='model.pth', mask_path='mask.pth', onnx_path='model.onnx', input_shape=[1, 1, 28, 28])
+```
 
 ## Customize new compression algorithms
 

docs/en_US/TrainingService/PaiMode.md

@@ -82,6 +82,22 @@ Compared with [LocalMode](LocalMode.md) and [RemoteMachineMode](RemoteMachineMod
         portNumber: 1
     ```
 
+NNI support two kind of authorization method in PAI, including password and PAI token, [refer](https://github.com/microsoft/pai/blob/b6bd2ab1c8890f91b7ac5859743274d2aa923c22/docs/rest-server/API.md#2-authentication). The authorization is configured in `paiConfig` field.  
+For password authorization, the `paiConfig` schema is:
+```
+paiConfig:
+  userName: your_pai_nni_user
+  passWord: your_pai_password
+  host: 10.1.1.1
+```  
+For pai token authorization, the `paiConfig` schema is:
+```
+paiConfig:
+  userName: your_pai_nni_user
+  token: your_pai_token
+  host: 10.1.1.1
+```
+
 Once complete to fill NNI experiment config file and save (for example, save as exp_pai.yml), then run the following command
 ```
 nnictl create --config exp_pai.yml

docs/en_US/TrialExample/GbdtExample.md

@@ -44,7 +44,15 @@ Given the features and label in train data, we train a GBDT regression model and
 
 ## 3. How to run in nni
 
-### 3.1 Prepare your trial code
+
+### 3.1 Install all the requirments
+
+```
+pip install lightgbm
+pip install pandas
+```
+
+### 3.2 Prepare your trial code
 
 You need to prepare a basic code as following:
 
@@ -86,7 +94,7 @@ if __name__ == '__main__':
     run(lgb_train, lgb_eval, PARAMS, X_test, y_test)
 ```
 
-### 3.2 Prepare your search space.
+### 3.3 Prepare your search space.
 If you like to tune `num_leaves`, `learning_rate`, `bagging_fraction` and `bagging_freq`, you could write a [search_space.json](https://github.com/Microsoft/nni/blob/master/examples/trials/auto-gbdt/search_space.json) as follow:
 
 ```json
@@ -100,7 +108,7 @@ If you like to tune `num_leaves`, `learning_rate`, `bagging_fraction` and `baggi
 
 More support variable type you could reference [here](../Tutorial/SearchSpaceSpec.md).
 
-### 3.3 Add SDK of nni into your code.
+### 3.4 Add SDK of nni into your code.
 
 ```diff
 +import nni
@@ -146,7 +154,7 @@ if __name__ == '__main__':
     run(lgb_train, lgb_eval, PARAMS, X_test, y_test)
 ```
 
-### 3.4 Write a config file and run it.
+### 3.5 Write a config file and run it.
 
 In the config file, you could set some settings including:
 

docs/en_US/sdk_reference.rst

@@ -36,6 +36,9 @@ Tuner
 ..  autoclass:: nni.metis_tuner.metis_tuner.MetisTuner
     :members:
 
+..  autoclass:: nni.batch_tuner.batch_tuner.BatchTuner
+    :members:
+
 Assessor
 ------------------------
 ..  autoclass:: nni.assessor.Assessor

src/nni_manager/common/experimentStartupInfo.ts

@@ -43,7 +43,7 @@ class ExperimentStartupInfo {
         this.initialized = true;
 
         if (logDir !== undefined && logDir.length > 0) {
-            this.logDir = path.join(logDir, getExperimentId());
+            this.logDir = path.join(path.normalize(logDir), getExperimentId());
         } else {
             this.logDir = path.join(os.homedir(), 'nni', 'experiments', getExperimentId());
         }

src/nni_manager/rest_server/restValidationSchemas.ts

@@ -107,7 +107,8 @@ export namespace ValidationSchemas {
             }),
             pai_config: joi.object({
                 userName: joi.string().min(1).required(),
-                passWord: joi.string().min(1).required(),
+                passWord: joi.string().min(1),
+                token: joi.string().min(1),
                 host: joi.string().min(1).required()
             }),
             kubeflow_config: joi.object({

src/nni_manager/training_service/common/util.ts

@@ -70,11 +70,11 @@ export async function validateCodeDir(codeDir: string) : Promise<number> {
  */
 export async function execMkdir(directory: string, share: boolean = false): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe New-Item -Path ${directory} -ItemType "directory" -Force`);
+        await cpp.exec(`powershell.exe New-Item -Path "${directory}" -ItemType "directory" -Force`);
     } else if (share) {
-        await cpp.exec(`(umask 0; mkdir -p ${directory})`);
+        await cpp.exec(`(umask 0; mkdir -p '${directory}')`);
     } else {
-        await cpp.exec(`mkdir -p ${directory}`);
+        await cpp.exec(`mkdir -p '${directory}'`);
     }
 
     return Promise.resolve();
@@ -87,9 +87,9 @@ export async function execMkdir(directory: string, share: boolean = false): Prom
  */
 export async function execCopydir(source: string, destination: string): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe Copy-Item ${source} -Destination ${destination} -Recurse`);
+        await cpp.exec(`powershell.exe Copy-Item "${source}" -Destination "${destination}" -Recurse`);
     } else {
-        await cpp.exec(`cp -r ${source} ${destination}`);
+        await cpp.exec(`cp -r '${source}' '${destination}'`);
     }
 
     return Promise.resolve();
@@ -101,9 +101,9 @@ export async function execCopydir(source: string, destination: string): Promise<
  */
 export async function execNewFile(filename: string): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe New-Item -Path ${filename} -ItemType "file" -Force`);
+        await cpp.exec(`powershell.exe New-Item -Path "${filename}" -ItemType "file" -Force`);
     } else {
-        await cpp.exec(`touch ${filename}`);
+        await cpp.exec(`touch '${filename}'`);
     }
 
     return Promise.resolve();
@@ -115,9 +115,9 @@ export async function execNewFile(filename: string): Promise<void> {
  */
 export function runScript(filePath: string): cp.ChildProcess {
     if (process.platform === 'win32') {
-        return cp.exec(`powershell.exe -ExecutionPolicy Bypass -file ${filePath}`);
+        return cp.exec(`powershell.exe -ExecutionPolicy Bypass -file "${filePath}"`);
     } else {
-        return cp.exec(`bash ${filePath}`);
+        return cp.exec(`bash '${filePath}'`);
     }
 }
 
@@ -128,9 +128,9 @@ export function runScript(filePath: string): cp.ChildProcess {
 export async function execTail(filePath: string): Promise<cpp.childProcessPromise.Result> {
     let cmdresult: cpp.childProcessPromise.Result;
     if (process.platform === 'win32') {
-        cmdresult = await cpp.exec(`powershell.exe Get-Content ${filePath} -Tail 1`);
+        cmdresult = await cpp.exec(`powershell.exe Get-Content "${filePath}" -Tail 1`);
     } else {
-        cmdresult = await cpp.exec(`tail -n 1 ${filePath}`);
+        cmdresult = await cpp.exec(`tail -n 1 '${filePath}'`);
     }
 
     return Promise.resolve(cmdresult);
@@ -142,9 +142,9 @@ export async function execTail(filePath: string): Promise<cpp.childProcessPromis
  */
 export async function execRemove(directory: string): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe Remove-Item ${directory} -Recurse -Force`);
+        await cpp.exec(`powershell.exe Remove-Item "${directory}" -Recurse -Force`);
     } else {
-        await cpp.exec(`rm -rf ${directory}`);
+        await cpp.exec(`rm -rf '${directory}'`);
     }
 
     return Promise.resolve();
@@ -173,7 +173,7 @@ export function setEnvironmentVariable(variable: { key: string; value: string })
     if (process.platform === 'win32') {
         return `$env:${variable.key}="${variable.value}"`;
     } else {
-        return `export ${variable.key}=${variable.value}`;
+        return `export ${variable.key}='${variable.value}'`;
     }
 }
 

src/nni_manager/training_service/local/localTrainingService.ts

@@ -490,18 +490,18 @@ class LocalTrainingService implements TrainingService {
         const script: string[] = [];
         if (process.platform === 'win32') {
             script.push(
-                `cmd.exe /c ${localTrialConfig.command} 2>${path.join(workingDirectory, 'stderr')}`,
+                `cmd.exe /c ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`,
                 `$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`,
                 `$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`,
-                `Write $LASTEXITCODE " " $NOW_DATE  | Out-File ${path.join(workingDirectory, '.nni', 'state')} -NoNewline -encoding utf8`);
+                `Write $LASTEXITCODE " " $NOW_DATE  | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`);
         } else {
-            script.push(`eval ${localTrialConfig.command} 2>${path.join(workingDirectory, 'stderr')}`);
+            script.push(`eval ${localTrialConfig.command} 2>"${path.join(workingDirectory, 'stderr')}"`);
             if (process.platform === 'darwin') {
                 // https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
                 // Considering the worst case, write 999 to avoid negative duration
-                script.push(`echo $? \`date +%s999\` >${path.join(workingDirectory, '.nni', 'state')}`);
+                script.push(`echo $? \`date +%s999\` >'${path.join(workingDirectory, '.nni', 'state')}'`);
             } else {
-                script.push(`echo $? \`date +%s%3N\` >${path.join(workingDirectory, '.nni', 'state')}`);
+                script.push(`echo $? \`date +%s%3N\` >'${path.join(workingDirectory, '.nni', 'state')}'`);
             }
         }
 
@@ -522,7 +522,7 @@ class LocalTrainingService implements TrainingService {
         if (process.platform !== 'win32') {
             runScriptContent.push('#!/bin/bash');
         }
-        runScriptContent.push(`cd ${this.localTrialConfig.codeDir}`);
+        runScriptContent.push(`cd '${this.localTrialConfig.codeDir}'`);
         for (const variable of variables) {
             runScriptContent.push(setEnvironmentVariable(variable));
         }

src/nni_manager/training_service/pai/paiConfig.ts

@@ -107,19 +107,22 @@ export class PAIJobConfig {
  */
 export class PAIClusterConfig {
     public readonly userName: string;
-    public readonly passWord: string;
+    public readonly passWord?: string;
     public readonly host: string;
+    public readonly token?: string;
 
     /**
      * Constructor
      * @param userName User name of PAI Cluster
      * @param passWord password of PAI Cluster
      * @param host Host IP of PAI Cluster
+     * @param token PAI token of PAI Cluster
      */
-    constructor(userName: string, passWord : string, host : string) {
+    constructor(userName: string, host : string, passWord?: string, token?: string) {
         this.userName = userName;
         this.passWord = passWord;
         this.host = host;
+        this.token = token;
     }
 }