Revert "[kvm] Use run_tests.sh script in sonic-mgmt to run PR tests (s…

…onic-net#139)" (sonic-net#142)

This reverts commit b52807c.

jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile

@@ -39,8 +39,8 @@ pipeline {
 
             post {
                 always {
-                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
-                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
+                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
                 }
             }
         }

jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile

@@ -36,8 +36,8 @@ pipeline {
 
             post {
                 always {
-                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
-                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
+                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
                 }
             }
         }

jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile

@@ -67,8 +67,8 @@ pipeline {
     post {
 
         always {
-            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
-            archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
+            archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
         }
 
         fixed {

jenkins/vs/buildimage-vs-image-pr/Jenkinsfile

@@ -87,8 +87,8 @@ sudo cp ../target/sonic-vs.bin /nfs/jenkins/sonic-vs-${JOB_NAME##*/}.${BUILD_NUM
 
             post {
                 always {
-                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
-                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
+                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
                 }
             }
         }

jenkins/vs/buildimage-vs-image-test/Jenkinsfile

@@ -67,8 +67,8 @@ pipeline {
     post {
 
         always {
-            archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
-            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
+            archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, sonic-mgmt/tests/results/**, kvmdump/**, ptfdump/**')
+            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
         }
 
         fixed {

jenkins/vs/buildimage-vs-image/Jenkinsfile

@@ -100,8 +100,8 @@ sudo cp ../target/sonic-vs-dbg.bin /nfs/jenkins/sonic-vs-dbg-${JOB_NAME##*/}.${B
 
             post {
                 always {
-                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
-                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
+                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
                 }
             }
 

scripts/vs/buildimage-vs-image/runtest.sh

@@ -1,7 +1,25 @@
 #!/bin/bash -xe
 
 tbname=$1
-dut=$2
+
+run_pytest()
+{
+    tgname=$1
+    shift
+    tests=$@
+
+    echo "run tests: $tests"
+
+    mkdir -p logs/$tgname
+    for tn in ${tests}; do
+        tdir=$(dirname $tn)
+        if [ $tdir != "." ]; then
+            mkdir -p logs/$tgname/$tdir
+            mkdir -p results/$tgname/$tdir
+        fi
+        py.test $PYTEST_COMMON_OPTS --log-file logs/$tgname/$tn.log --junitxml=results/$tgname/$tn.xml $tn.py
+    done
+}
 
 cd $HOME
 mkdir -p .ssh
@@ -22,59 +40,68 @@ export ANSIBLE_LIBRARY=/data/sonic-mgmt/ansible/library/
 # workaround for issue https://github.com/Azure/sonic-mgmt/issues/1659
 export export ANSIBLE_KEEP_REMOTE_FILES=1
 
-PYTEST_CLI_COMMON_OPTS="\
-    -i veos.vtb \
-    -d $dut \
-    -n $tbname \
-    -f vtestbed.csv \
-    -k debug \
-    -l warning \
-    -m group \
-    -e --disable_loganalyzer
-"
+PYTEST_COMMON_OPTS="--inventory veos.vtb \
+                    --host-pattern all \
+                    --user admin \
+                    -vvv \
+                    --show-capture stdout \
+                    --testbed $tbname \
+                    --testbed_file vtestbed.csv \
+                    --disable_loganalyzer \
+                    --log-file-level debug"
 
+# Check testbed health
 cd /data/sonic-mgmt/tests
-rm -rf logs
+rm -rf logs results
 mkdir -p logs
+mkdir -p results
+py.test $PYTEST_COMMON_OPTS --log-file logs/test_nbr_health.log --junitxml=results/tr.xml test_nbr_health.py
 
-# Run tests_1vlan on vlab-01 virtual switch
-# TODO: Use a marker to select these tests rather than providing a hard-coded list here.
+# Run anounce route test case in order to populate BGP route
+py.test $PYTEST_COMMON_OPTS --log-file logs/test_announce_routes.log --junitxml=results/tr.xml test_announce_routes.py
+
+# Tests to run using one vlan configuration
 tgname=1vlan
 tests="\
-test_interfaces.py \
-bgp/test_bgp_fact.py \
-bgp/test_bgp_gr_helper.py \
-bgp/test_bgp_speaker.py \
-cacl/test_cacl_application.py \
-cacl/test_cacl_function.py \
-dhcp_relay/test_dhcp_relay.py \
-lldp/test_lldp.py \
-ntp/test_ntp.py \
-pc/test_po_update.py \
-route/test_default_route.py \
-snmp/test_snmp_cpu.py \
-snmp/test_snmp_interfaces.py \
-snmp/test_snmp_lldp.py \
-snmp/test_snmp_pfc_counters.py \
-snmp/test_snmp_queue.py \
-syslog/test_syslog.py \
-tacacs/test_rw_user.py \
-tacacs/test_ro_user.py \
-telemetry/test_telemetry.py"
+    test_interfaces \
+    pc/test_po_update \
+    bgp/test_bgp_fact \
+    lldp/test_lldp \
+    route/test_default_route \
+    bgp/test_bgp_speaker \
+    bgp/test_bgp_gr_helper \
+    dhcp_relay/test_dhcp_relay \
+    syslog/test_syslog \
+    tacacs/test_rw_user \
+    tacacs/test_ro_user \
+    ntp/test_ntp \
+    cacl/test_cacl_application \
+    cacl/test_cacl_function \
+    telemetry/test_telemetry \
+    snmp/test_snmp_cpu \
+    snmp/test_snmp_interfaces \
+    snmp/test_snmp_lldp \
+    snmp/test_snmp_pfc_counters \
+    snmp/test_snmp_queue
+"
 
+# Run tests_1vlan on vlab-01 virtual switch
 pushd /data/sonic-mgmt/tests
-./run_tests.sh $PYTEST_CLI_COMMON_OPTS -c "$tests" -p logs/$tgname
+run_pytest $tgname $tests
 popd
 
 # Create and deploy two vlan configuration (two_vlan_a) to the virtual switch
 cd /data/sonic-mgmt/ansible
 ./testbed-cli.sh -m veos.vtb -t vtestbed.csv deploy-mg $tbname lab password.txt -e vlan_config=two_vlan_a
 sleep 180
 
-# Run tests_2vlans on vlab-01 virtual switch
+# Tests to run using two vlan configuration
 tgname=2vlans
-tests="dhcp_relay/test_dhcp_relay.py"
+tests="\
+    dhcp_relay/test_dhcp_relay \
+"
 
+# Run tests_2vlans on vlab-01 virtual switch
 pushd /data/sonic-mgmt/tests
-./run_tests.sh $PYTEST_CLI_COMMON_OPTS -c "$tests" -p logs/$tgname
+run_pytest $tgname $tests
 popd

scripts/vs/buildimage-vs-image/test.sh

@@ -28,7 +28,7 @@ cd sonic-mgmt/ansible
 sed -i s:use_own_value:johnar: veos.vtb
 echo abc > password.txt
 cd ../../
-docker run --rm=true -v $(pwd):/data -w /data -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt ./scripts/vs/buildimage-vs-image/runtest.sh $tbname $dut
+docker run --rm=true -v $(pwd):/data -w /data -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt ./scripts/vs/buildimage-vs-image/runtest.sh $tbname
 
 # save dut state if test fails
 if [ $? != 0 ]; then