Merge pull request #44 from Microsoft/master

merge master

deployment/Dockerfile

@@ -1,7 +1,60 @@
-FROM nni.build.base:cuda9.0-cudnn7-devel-ubuntu16.04
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
 
 LABEL maintainer='Microsoft NNI Team<nni@microsoft.com>'
 
+RUN DEBIAN_FRONTEND=noninteractive && \
+    apt-get -y update && \
+    apt-get -y install sudo \
+    apt-utils \
+    git \
+    curl \
+    vim \
+    unzip \
+    wget \
+    build-essential \ 
+    cmake \
+    libopenblas-dev \
+    automake \
+    openssh-client \
+    openssh-server \
+    lsof \
+    python3.5 \
+    python3-dev \
+    python3-pip \
+    python3-tk \
+    libcupti-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# numpy 1.14.3  scipy 1.1.0 
+RUN pip3 --no-cache-dir install \
+    numpy==1.14.3 scipy==1.1.0 
+
+#
+#Install NNI
+#
+RUN pip3 install -v --user git+https://github.com/Microsoft/nni.git@$(curl --silent "https://api.github.com/repos/Microsoft/nni/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+
 #
 #Tensorflow 1.10.0
 #
@@ -12,4 +65,6 @@ RUN pip3 --no-cache-dir install tensorflow-gpu==1.10.0
 #
 RUN pip3 --no-cache-dir install Keras==2.1.6
 
-WORKDIR /root
+ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin:
+
+WORKDIR /root

deployment/README.md

@@ -2,7 +2,6 @@ Dockerfile
 ===
 ## 1.Description  
 This is the Dockerfile of nni project, including the most kinds of deeplearning frameworks and nni source code.  You can run your nni experiment in this docker container directly.
-Dockerfile.build.base could build the base Docker image, users can get a docker image with Ubuntu and NNI environment after building this file. 
 Dockerfile could build the customized docker image, users could build their customized docker image using this file.
 ## 2.Including Libraries  
 
@@ -17,6 +16,5 @@ NNI v0.1
 
 ## 3 How to run  
 
-    docker build -f Dockerfile.build.base -t nni.build.base:cuda9.0-cudnn7-devel-ubuntu16.04 .
     docker build -t nni/nni .
     nvidia-docker run -it nni/nni

docs/NNICTLDOC.md

@@ -282,6 +282,48 @@ nnictl webui
         
     	Options:
     	
+       | Name, shorthand | Required|Default | Description |
+       | ------ | ------ | ------ |------ |
+     | id|  False| |ID of the experiment you want to set|
+
+
+### Manage tensorboard
+* __nnictl tensorboard start__
+   * Description
+
+	     Start the tensorboard process. 
+
+   * Usage
+
+		    nnictl tensorboard start
+        
+    	Options:
+    	
+       | Name, shorthand | Required|Default | Description |
+       | ------ | ------ | ------ |------ |
+     | id|  False| |ID of the experiment you want to set|
+     | --trialid|  False| |ID of the trial|
+     | --port|  False| 6006|The port of the tensorboard process|
+
+   * Detail
+
+	     1. NNICTL support tensorboard function in local and remote platform for the moment, other platforms will be supported later.   
+         2. If you want to use tensorboard, you need to write your tensorboard log data to environment variable [NNI_OUTPUT_DIR] path.  
+         3. In local mode, nnictl will set --logdir=[NNI_OUTPUT_DIR] directly and start a tensorboard process.
+         4. In remote mode, nnictl will create a ssh client to copy log data from remote machine to local temp directory firstly, and then start a tensorboard process in your local machine. You need to notice that nnictl only copy the log data one time when you use the command, if you want to see the later result of tensorboard, you should execute nnictl tensorboard command again.
+         5. If there is only one trial job, you don't need to set trialid. If there are multiple trial jobs running, you should set the trialid, or you could use [nnictl tensorboard start --trialid all] to map --logdir to all trial log paths.
+
+* __nnictl tensorboard stop__
+   * Description
+
+	     Stop all of the tensorboard process. 
+
+   * Usage
+
+		    nnictl tensorboard stop
+        
+    	Options:
+    	
        | Name, shorthand | Required|Default | Description |
        | ------ | ------ | ------ |------ |
      | id|  False| |ID of the experiment you want to set|

setup.py

@@ -62,7 +62,8 @@ def run(self):
         'requests',
         'scipy',
         'schema',
-        'pyhdfs'
+        'pyhdfs',
+        'paramiko'
     ],
 
     cmdclass={

src/nni_manager/core/nnimanager.ts

@@ -48,7 +48,7 @@ import { createDispatcherInterface, IpcInterface } from './ipcInterface';
 class NNIManager implements Manager {
     private trainingService: TrainingService;
     private dispatcher: IpcInterface | undefined;
-    private currSubmittedTrialNum: number; // need to be recovered
+    private currSubmittedTrialNum: number;  // need to be recovered
     private trialConcurrencyChange: number; // >0: increase, <0: decrease
     private customizedTrials: string[]; // need to be recovered
     private log: Logger;
@@ -58,7 +58,6 @@ class NNIManager implements Manager {
     private status: NNIManagerStatus;
     private waitingTrials: string[];
     private trialJobs: Map<string, TrialJobDetail>;
-    private suspendDuration: number;
 
     constructor() {
         this.currSubmittedTrialNum = 0;
@@ -69,7 +68,6 @@ class NNIManager implements Manager {
         this.dispatcherPid = 0;
         this.waitingTrials = [];
         this.trialJobs = new Map<string, TrialJobDetail>();
-        this.suspendDuration = 0;
 
         this.log = getLogger();
         this.dataStore = component.get(DataStore);
@@ -336,12 +334,16 @@ class NNIManager implements Manager {
     }
 
     private async periodicallyUpdateExecDuration(): Promise<void> {
-        const startTime: number = Date.now();
-        const execDuration: number = this.experimentProfile.execDuration;
+        let count: number = 1;
         for (; ;) {
-            await delay(1000 * 60 * 10); // 10 minutes
-            this.experimentProfile.execDuration = execDuration + (Date.now() - startTime) / 1000 - this.suspendDuration;
-            await this.storeExperimentProfile();
+            await delay(1000 * 1); // 1 seconds
+            if (this.status.status === 'EXPERIMENT_RUNNING') {
+                this.experimentProfile.execDuration += 1;
+                if (count % 10 === 0) {
+                    await this.storeExperimentProfile();
+                }
+            }
+            count += 1;
         }
     }
 
@@ -351,7 +353,6 @@ class NNIManager implements Manager {
         for (const trialJobId of Array.from(this.trialJobs.keys())) {
             const trialJobDetail: TrialJobDetail = await this.trainingService.getTrialJob(trialJobId);
             const oldTrialJobDetail: TrialJobDetail | undefined = this.trialJobs.get(trialJobId);
-            //assert(oldTrialJobDetail);
             if (oldTrialJobDetail !== undefined && oldTrialJobDetail.status !== trialJobDetail.status) {
                 this.trialJobs.set(trialJobId, Object.assign({}, trialJobDetail));
                 await this.dataStore.storeTrialJobEvent(trialJobDetail.status, trialJobDetail.id, undefined, trialJobDetail.url);
@@ -388,8 +389,6 @@ class NNIManager implements Manager {
             throw new Error('Error: tuner has not been setup');
         }
         let allFinishedTrialJobNum: number = 0;
-        const startTime: number = Date.now();
-        let suspendStartTime: number = 0;
         for (; ;) {
             if (this.status.status === 'STOPPING') {
                 break;
@@ -426,18 +425,18 @@ class NNIManager implements Manager {
             }
 
             // check maxtrialnum and maxduration here
-            if ((Date.now() - startTime) / 1000 + this.experimentProfile.execDuration - this.suspendDuration 
-                > this.experimentProfile.params.maxExecDuration ||
+            if (this.experimentProfile.execDuration > this.experimentProfile.params.maxExecDuration ||
                 this.currSubmittedTrialNum >= this.experimentProfile.params.maxTrialNum) {
                 assert(this.status.status === 'EXPERIMENT_RUNNING' || this.status.status === 'DONE');
                 if (this.status.status === 'EXPERIMENT_RUNNING') {
-                    suspendStartTime = Date.now();
+                    this.experimentProfile.endTime = Date.now();
+                    await this.storeExperimentProfile();
                 }
                 this.status.status = 'DONE';
             } else {
                 if (this.status.status === 'DONE') {
-                    assert(suspendStartTime !== 0);
-                    this.suspendDuration += (Date.now() - suspendStartTime) / 1000;
+                    delete this.experimentProfile.endTime;
+                    await this.storeExperimentProfile();
                 }
                 this.status.status = 'EXPERIMENT_RUNNING';
                 for (let i: number = this.trialJobs.size; i < this.experimentProfile.params.trialConcurrency; i++) {

tools/nnicmd/launcher.py

@@ -28,11 +28,10 @@
 from nni_annotation import *
 from .launcher_utils import validate_all_content
 from .rest_utils import rest_put, rest_post, check_rest_server, check_rest_server_quick, check_response
-from .url_utils import cluster_metadata_url, experiment_url
+from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
 from .config_utils import Config, Experiments
 from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process, detect_port
 from .constants import *
-from .webui_utils import *
 import time
 import random
 import string
@@ -288,7 +287,8 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen
         except Exception:
             raise Exception(ERROR_INFO % 'Restful server stopped!')
         exit(1)
-    web_ui_url_list = get_web_ui_urls(args.port, config_file_name)
+    web_ui_url_list = get_local_urls(args.port)
+    nni_config.set_config('webuiUrl', web_ui_url_list)
 
     #save experiment information
     experiment_config = Experiments()

tools/nnicmd/nnictl.py

@@ -25,6 +25,7 @@
 from .nnictl_utils import *
 from .package_management import *
 from .constants import *
+from .tensorboard_utils import *
 
 def nni_help_info(*args):
     print('please run "nnictl {positional argument} --help" to see nnictl guidance')
@@ -148,6 +149,18 @@ def parse_args():
     parser_package_show = parser_package_subparsers.add_parser('show', help='show the information of packages')
     parser_package_show.set_defaults(func=package_show)
 
+    #parse tensorboard command
+    parser_tensorboard = subparsers.add_parser('tensorboard', help='manage tensorboard')
+    parser_tensorboard_subparsers = parser_tensorboard.add_subparsers()
+    parser_tensorboard_start = parser_tensorboard_subparsers.add_parser('start', help='start tensorboard')
+    parser_tensorboard_start.add_argument('id', nargs='?', help='the id of experiment')
+    parser_tensorboard_start.add_argument('--trialid', dest='trialid', help='the id of trial')
+    parser_tensorboard_start.add_argument('--port', dest='port', default=6006, help='the port to start tensorboard')
+    parser_tensorboard_start.set_defaults(func=start_tensorboard)
+    parser_tensorboard_start = parser_tensorboard_subparsers.add_parser('stop', help='stop tensorboard')
+    parser_tensorboard_start.add_argument('id', nargs='?', help='the id of experiment')
+    parser_tensorboard_start.set_defaults(func=stop_tensorboard)
+
     args = parser.parse_args()
     args.func(args)
 

tools/nnicmd/nnictl_utils.py

@@ -174,8 +174,17 @@ def stop_experiment(args):
             time.sleep(3)
             rest_pid = nni_config.get_config('restServerPid')
             if rest_pid:
-                cmds = ['pkill', '-P', str(rest_pid)]
-                call(cmds)
+                stop_rest_cmds = ['pkill', '-P', str(rest_pid)]
+                call(stop_rest_cmds)
+                tensorboard_pid_list = nni_config.get_config('tensorboardPidList')
+                if tensorboard_pid_list:
+                    for tensorboard_pid in tensorboard_pid_list:
+                        try:
+                            cmds = ['kill', '-9', str(tensorboard_pid)]
+                            call(cmds)
+                        except Exception as exception:
+                            print_error(exception)
+                    nni_config.set_config('tensorboardPidList', [])
             if stop_rest_result:
                 print_normal('Stop experiment success!')
             experiment_config.update_experiment(experiment_id, 'status', 'stopped')
@@ -343,3 +352,4 @@ def experiment_list(args):
         experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
         experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
     print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
+