From 1eaca24cfc937296a1345e356be398c4249fe438 Mon Sep 17 00:00:00 2001 From: lguohan Date: Wed, 1 Apr 2020 20:59:02 -0700 Subject: [PATCH] [kvmtest]: archive the log for each test (#91) --- .../Jenkinsfile | 3 +- jenkins/vs/buildimage-vs-image-pr/Jenkinsfile | 4 +- .../vs/buildimage-vs-image-test/Jenkinsfile | 3 +- jenkins/vs/buildimage-vs-image/Jenkinsfile | 4 +- scripts/vs/buildimage-vs-image/runtest.sh | 80 +++++++++++++------ 5 files changed, 63 insertions(+), 31 deletions(-) diff --git a/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile b/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile index 89674b794e90..31bf6341ff5a 100644 --- a/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile +++ b/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile @@ -62,7 +62,8 @@ pipeline { post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/tr*.xml') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/tr*.xml') + archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**') } fixed { diff --git a/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile b/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile index 46804cfd28fb..c5b73c5f9e6e 100644 --- a/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile +++ b/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile @@ -71,8 +71,8 @@ mv target ../ post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/tr*.xml') - archiveArtifacts(artifacts: 'target/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/tr*.xml') + archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**') } } } diff --git a/jenkins/vs/buildimage-vs-image-test/Jenkinsfile b/jenkins/vs/buildimage-vs-image-test/Jenkinsfile index 26f3671e6eef..3d32f11b4393 100644 --- a/jenkins/vs/buildimage-vs-image-test/Jenkinsfile +++ b/jenkins/vs/buildimage-vs-image-test/Jenkinsfile @@ -62,7 +62,8 @@ pipeline { post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/tr*.xml') + archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, sonic-mgmt/tests/results/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/tr*.xml') } fixed { diff --git a/jenkins/vs/buildimage-vs-image/Jenkinsfile b/jenkins/vs/buildimage-vs-image/Jenkinsfile index 219bbd2c44de..baacc5ae4e53 100644 --- a/jenkins/vs/buildimage-vs-image/Jenkinsfile +++ b/jenkins/vs/buildimage-vs-image/Jenkinsfile @@ -75,8 +75,8 @@ mv target ../ post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/tr*.xml') - archiveArtifacts(artifacts: 'target/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/tr*.xml') + archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**') } fixed { diff --git a/scripts/vs/buildimage-vs-image/runtest.sh b/scripts/vs/buildimage-vs-image/runtest.sh index 3b5390349ac4..daf181820638 100755 --- a/scripts/vs/buildimage-vs-image/runtest.sh +++ b/scripts/vs/buildimage-vs-image/runtest.sh @@ -1,5 +1,23 @@ #!/bin/bash -xe +run_pytest() +{ + tgname=$1 + shift + tests=$@ + + echo "run tests: $tests" + + mkdir -p logs/$tgname + for tn in ${tests}; do + tdir=$(dirname $tn) + if [ $tdir != "." ]; then + mkdir -p logs/$tgname/$tdir + fi + py.test $PYTEST_COMMON_OPTS --log-file logs/$tgname/$tn.log --junitxml=results/tr_$tn-$tgname.xml $tn.py + done +} + cd $HOME mkdir -p .ssh cp /data/pkey.txt .ssh/id_rsa @@ -16,35 +34,46 @@ sleep 180 export ANSIBLE_LIBRARY=/data/sonic-mgmt/ansible/library/ +PYTEST_COMMON_OPTS="--inventory veos.vtb \ + --host-pattern all \ + --user admin \ + -vvv \ + --show-capture stdout \ + --testbed vms-kvm-t0 \ + --testbed_file vtestbed.csv \ + --disable_loganalyzer" + # Check testbed health cd /data/sonic-mgmt/tests -py.test --inventory veos.vtb --host-pattern all --user admin -vvv --show-capture stdout --testbed vms-kvm-t0 \ - --testbed_file vtestbed.csv --disable_loganalyzer --junitxml=tr.xml test_nbr_health.py +mkdir -p logs +mkdir -p results +py.test $PYTEST_COMMON_OPTS --log-file logs/test_nbr_health.log --junitxml=results/tr.xml test_nbr_health.py # Run anounce route test case in order to populate BGP route -py.test --inventory veos.vtb --host-pattern all --user admin -vvv --show-capture stdout --testbed vms-kvm-t0 \ - --testbed_file vtestbed.csv --disable_loganalyzer --junitxml=tr.xml test_announce_routes.py +py.test $PYTEST_COMMON_OPTS --log-file logs/test_announce_routes.log --junitxml=results/tr.xml test_announce_routes.py # Tests to run using one vlan configuration -tests_1vlan="\ - test_interfaces.py \ - test_bgp_fact.py \ - test_lldp.py \ - test_bgp_speaker.py \ - test_dhcp_relay.py \ - tacacs/test_rw_user.py \ - tacacs/test_ro_user.py \ - ntp/test_ntp.py \ - snmp/test_snmp_cpu.py \ - snmp/test_snmp_interfaces.py \ - snmp/test_snmp_lldp.py \ - snmp/test_snmp_pfc_counters.py \ - snmp/test_snmp_queue.py +tgname=1vlan +tests="\ + test_interfaces \ + test_bgp_fact \ + test_lldp \ + test_bgp_speaker \ + test_dhcp_relay \ + tacacs/test_rw_user \ + tacacs/test_ro_user \ + ntp/test_ntp \ + snmp/test_snmp_cpu \ + snmp/test_snmp_interfaces \ + snmp/test_snmp_lldp \ + snmp/test_snmp_pfc_counters \ + snmp/test_snmp_queue " # Run tests_1vlan on vlab-01 virtual switch -py.test --inventory veos.vtb --host-pattern all --user admin -vvv --show-capture stdout --testbed vms-kvm-t0 \ - --testbed_file vtestbed.csv --disable_loganalyzer --junitxml=tr_1vlan.xml $tests_1vlan +pushd /data/sonic-mgmt/tests +run_pytest $tgname $tests +popd # Create and deploy two vlan configuration (two_vlan_a) to the virtual switch cd /data/sonic-mgmt/ansible @@ -52,11 +81,12 @@ cd /data/sonic-mgmt/ansible sleep 180 # Tests to run using two vlan configuration -tests_2vlans="\ - test_dhcp_relay.py \ +tgname=2vlans +tests="\ + test_dhcp_relay \ " -cd /data/sonic-mgmt/tests # Run tests_2vlans on vlab-01 virtual switch -py.test --inventory veos.vtb --host-pattern all --user admin -vvv --show-capture stdout --testbed vms-kvm-t0 \ - --testbed_file vtestbed.csv --disable_loganalyzer --junitxml=tr_2vlans.xml $tests_2vlans +pushd /data/sonic-mgmt/tests +run_pytest $tgname $tests +popd