Merge pull request #1 from chicm-ms/v0.5.2

Merge v0.5.2 back to master

deployment/deployment-pipelines.yml

@@ -1,6 +1,60 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 jobs:
+- job: 'version_number_validation'
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  strategy:
+    matrix:
+      Python36:
+        PYTHON_VERSION: '3.6'
+  steps:
+  - script: |
+      echo $(build_version)
+      if [[ $(build_version) =~ ^v[0-9](.[0-9]){1,3}$ ]]; then
+        echo 'valid build version $(build_version)'
+        echo `git describe --tags --abbrev=0`
+      else
+        echo 'invalid build version $(build_version)'
+        exit 1
+      fi
+    condition: eq( variables['build_type'], 'prerelease' )
+    displayName: 'Validate prerelease version number'
+
+  - script: |
+      export BRANCH_TAG=`git describe --tags --abbrev=0`
+      echo $BRANCH_TAG
+      if [[ $BRANCH_TAG = $(build_version) && $BRANCH_TAG =~ ^v[0-9](.[0-9]){1,3}$ ]]; then
+        echo 'Build version match branch tag'
+      else
+        echo 'Build version does not match branch tag'
+        exit 1
+      fi
+    condition: eq( variables['build_type'], 'release' )
+    displayName: 'Validate release version number and branch tag'
+
+
 - job: 'Build_upload_nni_ubuntu'
+  dependsOn: version_number_validation
+  condition: succeeded()
   pool:
     vmImage: 'Ubuntu 16.04'
   strategy:
@@ -13,21 +67,22 @@ jobs:
       python3 -m pip install --upgrade pip setuptools --user
       python3 -m pip install twine --user
     displayName: 'Install twine'
+
   - script: |
-      # NNI build scripts (Makefile) uses branch tag as package version number
-      # To test this pipeline without impacting nni testpypi/pypi packages, uncomment following git tag command
-      # git tag v0.0.1
       cd deployment/pypi
       if [ $(build_type) = 'prerelease' ]
       then
+        # NNI build scripts (Makefile) uses branch tag as package version number
+        git tag $(build_version)
         echo 'building prerelease package...'
         make version_ts=true build
       else
         echo 'building release package...'
         make build
       fi
-    condition: eq( variables['upload_package'], 'true' )
+    condition: eq( variables['upload_package'], 'true')
     displayName: 'build nni bdsit_wheel'
+
   - script: |
       cd deployment/pypi
       if [ $(build_type) = 'prerelease' ]
@@ -38,7 +93,7 @@ jobs:
         echo 'uploading release package to pypi...'
         python3 -m twine upload -u $(pypi_user) -p $(pypi_pwd) dist/*
       fi
-    condition: eq( variables['upload_package'], 'true' )
+    condition: eq( variables['upload_package'], 'true')
     displayName: 'upload nni package to pypi/testpypi'
 
   - script: |
@@ -48,7 +103,7 @@ jobs:
       then
         docker login -u $(docker_hub_dev_user) -p $(docker_hub_dev_pwd)
         export IMG_NAME=$(dev_docker_img)
-        export IMG_TAG=`git describe --tags --abbrev=0`.`date +%y%m%d%H%M`
+        export IMG_TAG=`git describe --tags --abbrev=0`.`date -u +%y%m%d%H%M`
         echo 'updating docker file for testpyi...'
         sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/RUN python3 -m pip install --user --no-cache-dir --index-url https:\/\/test.pypi.org\/simple --extra-index-url https:\/\/pypi.org\/simple nni/' Dockerfile
       else
@@ -67,10 +122,12 @@ jobs:
         docker push $IMG_NAME:latest
       fi
 
-    condition: eq( variables['build_docker_img'], 'true' )
+    condition: eq( variables['build_docker_img'], 'true')
     displayName: 'build and upload nni docker image'
 
 - job: 'Build_upload_nni_macos'
+  dependsOn: version_number_validation
+  condition: succeeded()
   pool:
     vmImage: 'macOS 10.13'
   strategy:
@@ -82,24 +139,26 @@ jobs:
       python3 -m pip install --upgrade pip setuptools --user
       python3 -m pip install twine --user
     displayName: 'Install twine'
+
   - script: |
       make install-dependencies
     displayName: 'Install nni dependencies'
+
   - script: |
-      # NNI build scripts (Makefile) uses branch tag as package version number
-      # To test this pipeline without impacting nni testpypi/pypi packages, uncomment following git tag command
-      # git tag v0.0.1
       cd deployment/pypi
       if [ $(build_type) = 'prerelease' ]
       then
+        # NNI build scripts (Makefile) uses branch tag as package version number
+        git tag $(build_version)
         echo 'building prerelease package...'
         PATH=$HOME/Library/Python/3.7/bin:$PATH make version_ts=true build
       else
         echo 'building release package...'
         PATH=$HOME/Library/Python/3.7/bin:$PATH make build
       fi
-    condition: eq( variables['upload_package'], 'true' )
+    condition: eq( variables['upload_package'], 'true')
     displayName: 'build nni bdsit_wheel'
+
   - script: |
       cd deployment/pypi
       if [ $(build_type) = 'prerelease' ]
@@ -110,5 +169,5 @@ jobs:
         echo 'uploading release package to pypi...'
         python3 -m twine upload -u $(pypi_user) -p $(pypi_pwd) dist/*
       fi
-    condition: eq( variables['upload_package'], 'true' )
+    condition: eq( variables['upload_package'], 'true')
     displayName: 'upload nni package to pypi/testpypi'

deployment/pypi/Makefile

@@ -11,7 +11,7 @@ else
 	$(error platform $(UNAME_S) not supported)
 endif
 
-TIME_STAMP = $(shell date "+%y%m%d%H%M")
+TIME_STAMP = $(shell date -u "+%y%m%d%H%M")
 NNI_VERSION_VALUE = $(shell git describe --tags --abbrev=0)
 
 # To include time stamp in version value, run:

deployment/pypi/setup.py

@@ -56,6 +56,7 @@
         'nni_gpu_tool': '../../tools/nni_gpu_tool',
         'nni': '../../src/sdk/pynni/nni'
     },
+    package_data = {'nni': ['**/requirements.txt']},
     python_requires = '>=3.5',
     install_requires = [
         'schema',
@@ -81,4 +82,4 @@
             'nnictl = nni_cmd.nnictl:parse_args'
         ]
     }
-)
+)

docs/en_US/RELEASE.md

@@ -1,5 +1,23 @@
 # ChangeLog
 
+## Release 0.5.2 - 3/4/2019
+### Improvements
+* Curve fitting assessor performance improvement.
+
+### Documentation
+* Chinese version document: https://nni.readthedocs.io/zh/latest/
+* Debuggability/serviceability document: https://nni.readthedocs.io/en/latest/HowToDebug.html
+* Tuner assessor reference: https://nni.readthedocs.io/en/latest/sdk_reference.html#tuner
+
+### Bug Fixes and Other Changes
+* Fix a race condition bug that does not store trial job cancel status correctly.
+* Fix search space parsing error when using SMAC tuner.
+* Fix cifar10 example broken pipe issue.
+* Add unit test cases for nnimanager and local training service.
+* Add integration test azure pipelines for remote machine, PAI and kubeflow training services.
+* Support Pylon in PAI webhdfs client.
+
+
 ## Release 0.5.1 - 1/31/2018
 ### Improvements
 * Making [log directory](https://github.com/Microsoft/nni/blob/v0.5.1/docs/en_US/ExperimentConfig.md) configurable

src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py

@@ -57,8 +57,8 @@ def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, thresho
         self.threshold = threshold
         # Record the number of gap
         self.gap = gap
-        # Record the number of times of judgments
-        self.judgment_num = 0
+        # Record the number of intermediate result in the lastest judgment
+        self.last_judgment_num = dict()
         # Record the best performance
         self.set_best_performance = False
         self.completed_best_performance = None
@@ -112,9 +112,10 @@ def assess_trial(self, trial_job_id, trial_history):
         curr_step = len(trial_history)
         if curr_step < self.start_step:
             return AssessResult.Good
-        if (curr_step - self.start_step) // self.gap <= self.judgment_num:
+
+        if trial_job_id in self.last_judgment_num.keys() and curr_step - self.last_judgment_num[trial_job_id] < self.gap:
             return AssessResult.Good
-        self.judgment_num = (curr_step - self.start_step) // self.gap
+        self.last_judgment_num[trial_job_id] = curr_step
 
         try:
             start_time = datetime.datetime.now()

src/sdk/pynni/nni/metis_tuner/metis_tuner.py

@@ -341,7 +341,6 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
 
                 if results_outliers is not None:
                     #temp = len(candidates)
-
                     for results_outlier in results_outliers:
                         if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x:
                             candidates.append({'hyperparameter': samples_x[results_outlier['samples_idx']],\
@@ -370,13 +369,6 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y,
                         temp_improvement = threads_result['expected_lowest_mu'] - lm_current['expected_mu']
 
                         if next_improvement > temp_improvement:
-                            # logger.info("DEBUG: \"next_candidate\" changed: \
-                            #                 lowest mu might reduce from %f (%s) to %f (%s), %s\n" %\
-                            #                 lm_current['expected_mu'], str(lm_current['hyperparameter']),\
-                            #                 threads_result['expected_lowest_mu'],\
-                            #                 str(threads_result['candidate']['hyperparameter']),\
-                            #                 threads_result['candidate']['reason'])
-
                             next_improvement = temp_improvement
                             next_candidate = threads_result['candidate']
             else:

src/webui/src/components/TrialsDetail.tsx

@@ -92,7 +92,7 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
                             formatter: function (data: TooltipForAccuracy) {
                                 const result = '<div class="tooldetailAccuracy">' +
                                     '<div>Trial No: ' + data.data[0] + '</div>' +
-                                    '<div>Default Metrc: ' + data.data[1] + '</div>' +
+                                    '<div>Default Metric: ' + data.data[1] + '</div>' +
                                     '<div>Parameters: ' +
                                     '<pre>' + JSON.stringify(data.data[2], null, 4) + '</pre>' +
                                     '</div>' +

test/pipelines-it-kubeflow.yml

@@ -1,10 +1,26 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 jobs:
 - job: 'integration_test_kubeflow'
   timeoutInMinutes: 0
-  pool: 'NNI CI KUBE CLI'
-
-  variables:
-    new_docker_img: msranni/nni.it.kb:latest
 
   steps:
   - script: python3 -m pip install --upgrade pip setuptools --user
@@ -18,28 +34,25 @@ jobs:
     condition: eq( variables['build_docker_img'], 'true' )
     displayName: 'build nni bdsit_wheel'
 
-  - script: |
-      cd deployment/pypi
-      docker login -u $(docker_hub_user) -p $(docker_hub_pwd)
-
-      echo 'updating docker file for installing nni from local...'
-      # update Dockerfile to install NNI in docker image from whl file built in last step
-      sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
-      cat ../docker/Dockerfile
-      echo $IMG_TAG
-      docker build -f ../docker/Dockerfile -t $(new_docker_img) .
-      docker push $(new_docker_img)
-    condition: eq( variables['build_docker_img'], 'true' )
-    displayName: 'build and upload nni docker image'
-
   - script: |
       source install.sh
     displayName: 'Install nni toolkit via source code'
 
   - script: |
       if [ $(build_docker_img) = 'true' ]
       then
-        export TEST_IMG=$(new_docker_img)
+        cd deployment/pypi
+        docker login -u $(docker_hub_user) -p $(docker_hub_pwd)
+
+        echo 'updating docker file for installing nni from local...'
+        # update Dockerfile to install NNI in docker image from whl file built in last step
+        sed -ie 's/RUN python3 -m pip --no-cache-dir install nni/COPY .\/dist\/* .\nRUN python3 -m pip install nni-*.whl/' ../docker/Dockerfile
+        cat ../docker/Dockerfile
+        export IMG_TAG=`date -u +%y%m%d%H%M`
+        docker build -f ../docker/Dockerfile -t $(test_docker_img_name):$IMG_TAG .
+        docker push $(test_docker_img_name):$IMG_TAG
+        export TEST_IMG=$(test_docker_img_name):$IMG_TAG
+        cd ../../
       else
         export TEST_IMG=$(existing_docker_img)
       fi