Refactor pipeline (#833)

* Refactoring test pipelines, move local integrate test from PR pipeline into a sepearate pipeline which is scheduled daily.

azure-pipelines.yml

@@ -1,26 +1,18 @@
-trigger:
-- master
-- dev-remote-ci
-
 jobs:
-- job: 'Ubuntu_16_04'
-  pool: 'NNI CI GPU'
+- job: 'basic_test_pr_ubuntu'
+  pool:
+    vmImage: 'Ubuntu 16.04'
+  strategy:
+    matrix:
+      Python36:
+        PYTHON_VERSION: '3.6'
 
   steps:
   - script: python3 -m pip install --upgrade pip setuptools --user
     displayName: 'Install python tools'
   - script: |
       source install.sh
     displayName: 'Install nni toolkit via source code'
-  - script: |
-      python3 -m pip install scikit-learn==0.20.0 --user
-      python3 -m pip install torch==0.4.1 --user
-      python3 -m pip install torchvision==0.2.1 --user
-      python3 -m pip install keras==2.1.6 --user
-      python3 -m pip install tensorflow-gpu==1.10.0 --user
-      sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-    displayName: 'Install dependencies for integration tests'
   - script: |
       cd test
       source unittest.sh
@@ -33,16 +25,12 @@ jobs:
       cd test
       PATH=$HOME/.local/bin:$PATH python3 tuner_test.py
     displayName: 'Built-in tuners / assessors tests'
-  - script: |
-      cd test
-      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --local_gpu
-    displayName: 'Examples and advanced features tests on local machine'
   - script: |
       cd test
       PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
     displayName: 'Trial job metrics test'
 
-- job: 'macOS_10_13'
+- job: 'basic_test_pr_macOS'
   pool:
     vmImage: 'macOS 10.13'
   strategy:

test/pipelines-it-local.yml

@@ -0,0 +1,38 @@
+jobs:
+- job: 'integration_test_local_ubuntu'
+
+  steps:
+  - script: python3 -m pip install --upgrade pip setuptools --user
+    displayName: 'Install python tools'
+  - script: |
+      source install.sh
+    displayName: 'Install nni toolkit via source code'
+  - script: |
+      python3 -m pip install scikit-learn==0.20.0 --user
+      python3 -m pip install torch==0.4.1 --user
+      python3 -m pip install torchvision==0.2.1 --user
+      python3 -m pip install keras==2.1.6 --user
+      python3 -m pip install tensorflow-gpu==1.10.0 --user
+      sudo apt-get install swig -y
+      nnictl package install --name=SMAC
+    displayName: 'Install dependencies for integration tests'
+  - script: |
+      cd test
+      source unittest.sh
+    displayName: 'Unit test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 naive_test.py
+    displayName: 'Naive test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 tuner_test.py
+    displayName: 'Built-in tuners / assessors tests'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --local_gpu
+    displayName: 'Examples and advanced features tests on local machine'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
+    displayName: 'Trial job metrics test'