From 127d59b1674c3d1e573ba8d5b4a849fe9743d3a3 Mon Sep 17 00:00:00 2001
From: Chengmin Chi <chec@microsoft.com>
Date: Thu, 18 Jul 2019 15:07:02 +0800
Subject: [PATCH 1/4] updates

---
 test/config_test.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/config_test.py b/test/config_test.py
index dece4cf490..6b7becb5b5 100644
--- a/test/config_test.py
+++ b/test/config_test.py
@@ -131,3 +131,5 @@ def run(args):
     setup_experiment(args.preinstall)
 
     run(args)
+
+#
\ No newline at end of file

From ee4c8aeacbfd6cf4a63c01f72894f196321f0e27 Mon Sep 17 00:00:00 2001
From: Chengmin Chi <chec@microsoft.com>
Date: Tue, 6 Aug 2019 11:29:44 +0800
Subject: [PATCH 2/4] Fix pipeline

---
 test/pipelines-it-pai.yml    | 2 +-
 test/pipelines-it-remote.yml | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/pipelines-it-pai.yml b/test/pipelines-it-pai.yml
index 66a06b7c3e..6e31c92396 100644
--- a/test/pipelines-it-pai.yml
+++ b/test/pipelines-it-pai.yml
@@ -76,5 +76,5 @@ jobs:
       --nni_docker_image $TEST_IMG --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
 
       PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai --exclude multi_phase_batch,multi_phase_grid
-      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
+      #TODO add metrics test
     displayName: 'integration test'
diff --git a/test/pipelines-it-remote.yml b/test/pipelines-it-remote.yml
index d07f037d5c..9dca783ddf 100644
--- a/test/pipelines-it-remote.yml
+++ b/test/pipelines-it-remote.yml
@@ -52,8 +52,14 @@ jobs:
       python3 generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
       --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
       cat training_service.yml
-      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10,multi_phase_batch,multi_phase_grid
       PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
+    displayName: 'metrics test'
+  - script: |
+      cd test
+      python3 generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
+      --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
+      cat training_service.yml
+      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10,multi_phase_batch,multi_phase_grid
     displayName: 'integration test'
   - task: SSH@0
     inputs:

From 6bf442cb3289b6ad35674c3cd5dfdff8e43ceec4 Mon Sep 17 00:00:00 2001
From: Chengmin Chi <chec@microsoft.com>
Date: Tue, 6 Aug 2019 11:42:33 +0800
Subject: [PATCH 3/4] updates

---
 test/pipelines-it-pai.yml    | 3 ++-
 test/pipelines-it-remote.yml | 9 ++-------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/test/pipelines-it-pai.yml b/test/pipelines-it-pai.yml
index 6e31c92396..c038c01cce 100644
--- a/test/pipelines-it-pai.yml
+++ b/test/pipelines-it-pai.yml
@@ -50,6 +50,7 @@ jobs:
     displayName: 'Install dependencies for integration tests in PAI mode'
 
   - script: |
+      set -e
       if [ $(build_docker_img) = 'true' ]
       then
         cd deployment/pypi
@@ -76,5 +77,5 @@ jobs:
       --nni_docker_image $TEST_IMG --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
 
       PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai --exclude multi_phase_batch,multi_phase_grid
-      #TODO add metrics test
+      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
     displayName: 'integration test'
diff --git a/test/pipelines-it-remote.yml b/test/pipelines-it-remote.yml
index 9dca783ddf..923e4215f3 100644
--- a/test/pipelines-it-remote.yml
+++ b/test/pipelines-it-remote.yml
@@ -48,18 +48,13 @@ jobs:
       cat test/port
     displayName: 'Get docker port'
   - script: |
-      cd test
-      python3 generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
-      --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
-      cat training_service.yml
-      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
-    displayName: 'metrics test'
-  - script: |
+      set -e
       cd test
       python3 generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
       --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
       cat training_service.yml
       PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10,multi_phase_batch,multi_phase_grid
+      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
     displayName: 'integration test'
   - task: SSH@0
     inputs:

From 926e73aedc10931df33ea5b288b02af8d2a12bab Mon Sep 17 00:00:00 2001
From: Chengmin Chi <chec@microsoft.com>
Date: Tue, 6 Aug 2019 11:45:54 +0800
Subject: [PATCH 4/4] updates

---
 test/config_test.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/config_test.py b/test/config_test.py
index 6b7becb5b5..dece4cf490 100644
--- a/test/config_test.py
+++ b/test/config_test.py
@@ -131,5 +131,3 @@ def run(args):
     setup_experiment(args.preinstall)
 
     run(args)
-
-#
\ No newline at end of file