[Build] Support to collect the test coverage in cobertura format (son…

…ic-net#3019)

* [Build] Support to collect the test coverage in cobertura format

.artifactignore

@@ -1,2 +1,5 @@
 **/*
 !*.deb
+!coverage.info
+!coverage.xml
+!build.info

.azure-pipelines/build-docker-sonic-vs-template.yml

@@ -141,12 +141,17 @@ jobs:
 
       find $(Build.ArtifactStagingDirectory)/download/sairedis -name '*.deb' -exec cp "{}" .azure-pipelines/docker-sonic-vs/debs \;
       cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
+      if [ -f $(Build.ArtifactStagingDirectory)/download/coverage.info ]; then
+        cp -v $(Build.ArtifactStagingDirectory)/download/coverage.info $(Build.ArtifactStagingDirectory)/
+        cp -v $(Build.ArtifactStagingDirectory)/download/coverage.xml $(Build.ArtifactStagingDirectory)/
+      fi
 
       pushd .azure-pipelines
 
-      build_args=""
+      build_dir=$(grep BUILD_DIR $(Build.ArtifactStagingDirectory)/download/build.info | cut -d= -f2)
+      build_args="--build-arg build_dir=$build_dir"
       if [ '${{ parameters.asan }}' == True ]; then
-        build_args="--build-arg need_dbg=y"
+        build_args="$build_args --build-arg need_dbg=y"
       fi
 
       docker build $build_args --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} docker-sonic-vs

.azure-pipelines/build-template.yml

@@ -105,6 +105,7 @@ jobs:
         swig4.0 \
         libdbus-1-dev \
         libteam-dev
+      sudo pip3 install lcov_cobertura
     displayName: "Install dependencies"
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -193,6 +194,7 @@ jobs:
       cp -r pytest.tgz $(Build.ArtifactStagingDirectory)/
       if [ '${{ parameters.archive_gcov }}' == True ]; then
          export ENABLE_GCOV=y
+         echo BUILD_DIR=$(pwd) > build.info
       fi
       if [ '${{ parameters.asan }}' == True ]; then
          export ENABLE_ASAN=y

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -2,6 +2,8 @@ FROM docker-sonic-vs
 
 ARG docker_container_name
 ARG need_dbg
+ARG build_dir
+ENV BUILD_DIR=$build_dir
 
 COPY ["debs", "/debs"]
 
@@ -24,4 +26,12 @@ RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/swss-dbg_1.0.0_amd64.deb ; fi
 
 RUN apt-get update
 
-RUN apt-get -y install lcov
+RUN apt-get -y install software-properties-common libdatetime-perl libcapture-tiny-perl build-essential libcpanel-json-xs-perl git
+
+RUN git clone -b v2.0 --single-branch --depth 1 https://github.com/linux-test-project/lcov && cd lcov && make install
+
+RUN lcov --version
+
+RUN pip3 install lcov_cobertura
+
+RUN if [ -n "$BUILD_DIR" ]; then mkdir -p $BUILD_DIR && tar -xf /tmp/gcov/gcov-source.tar -C $BUILD_DIR; fi

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -44,11 +44,16 @@ jobs:
 - job:
   displayName: vstest
   timeoutInMinutes: ${{ parameters.timeout }}
+  ${{ if parameters.archive_gcov }}:
+    variables:
+      DIFF_COVER_CHECK_THRESHOLD: 80
+      DIFF_COVER_ENABLE: 'true'
 
-  pool: sonic-common
+  pool: sonic-common-test
 
   steps:
   - script: |
+      ip a show dev eth0 || true
       ls -A1 | xargs -I{} sudo rm -rf {}
     displayName: "Clean workspace"
   - checkout: self
@@ -78,6 +83,16 @@ jobs:
       path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic buildimage ubuntu20.04 deb packages"
 
+  - script: |
+        set -ex
+        # Install .NET CORE
+        curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+        sudo apt-add-repository https://packages.microsoft.com/ubuntu/20.04/prod
+        sudo apt-get update
+        sudo apt-get install -y dotnet-sdk-7.0
+        sudo dotnet tool install dotnet-reportgenerator-globaltool  --tool-path /usr/bin
+    displayName: "Install .NET CORE"
+
   - script: |
       set -ex
       sudo .azure-pipelines/build_and_install_module.sh
@@ -92,6 +107,7 @@ jobs:
       sudo apt-get install -y net-tools bridge-utils vlan
       sudo apt-get install -y python3-pip
       sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0
+      sudo pip3 install lcov_cobertura
     displayName: "Install dependencies"
 
   - script: |
@@ -106,7 +122,9 @@ jobs:
 
       params=""
       if [ '${{ parameters.archive_gcov }}' == True ]; then
-        params=" ${params} --keeptb "
+        cp $(Build.ArtifactStagingDirectory)/download/coverage.info ./
+        cp $(Build.ArtifactStagingDirectory)/download/coverage.xml ./
+        params=" ${params} --enable-coverage --force-recreate-dvs "
       fi
       if [ '${{ parameters.asan }}' == True ]; then
         params=" ${params} --graceful-stop "
@@ -115,44 +133,37 @@ jobs:
         params=" ${params} --num-ports=${{ parameters.num_ports }} "
       fi
 
-      all_tests=$(ls test_*.py)
+      all_tests=$(ls test_*.py | xargs)
       all_tests="${all_tests} p4rt"
 
       if [ -n '${{ parameters.run_tests_pattern }}' ]; then
-        all_tests=" $(ls ${{ parameters.run_tests_pattern }}) "
-      fi
-
-      test_set=()
-      # Run 20 tests as a set.
-      for test in ${all_tests}; do
-        test_set+=("${test}")
-        if [ ${#test_set[@]} -ge 20 ]; then
-          test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-          echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
-          container_count=$(docker ps -q -a | wc -l)
-          if [ '${{ parameters.archive_gcov }}' == True ] && [ ${container_count} -gt 0 ]; then
-            ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
-            docker stop $(docker ps -q -a)
-            docker rm $(docker ps -q -a)
-          fi
-          test_set=()
-        fi
-      done
-      if [ ${#test_set[@]} -gt 0 ]; then
-        test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-        echo "${test_set[*]}" | xargs sudo py.test -v $params --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
-        container_count=$(docker ps -q -a | wc -l)
-        if [ '${{ parameters.archive_gcov }}' == True ] && [ ${container_count} -gt 0 ]; then
-          ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
-          docker stop $(docker ps -q -a)
-          docker rm $(docker ps -q -a)
-        fi
+        all_tests=" $(ls ${{ parameters.run_tests_pattern }} | xargs) "
       fi
+      # Run the tests in parallel and retry
+      retry=3
+      IMAGE_NAME=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
+      echo $all_tests | xargs -n 1 | xargs -P 8 -I TEST_MODULE sudo ./run-tests.py "$IMAGE_NAME" "$params" "TEST_MODULE" 3
 
       rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"
     continueOnError: ${{ parameters.asan }}
 
+  - script: |
+      set -ex
+      reportgenerator -reporttypes:Cobertura -reports:tests/*coverage.xml -targetdir:.
+      mkdir $(Build.ArtifactStagingDirectory)/gcov
+      cp Cobertura.xml tests/*coverage.xml $(Build.ArtifactStagingDirectory)/gcov/
+      cp tests/*coverage.info $(Build.ArtifactStagingDirectory)/gcov/
+    condition: ${{ parameters.archive_gcov }}
+    displayName: "Generate coverage.xml"
+
+  - task: PublishCodeCoverageResults@1
+    condition: ${{ parameters.archive_gcov }}
+    inputs:
+      codeCoverageTool: Cobertura
+      summaryFileLocation: '$(System.DefaultWorkingDirectory)/Cobertura.xml'
+    displayName: 'Publish test coverage'
+
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '**/*_tr.xml'
@@ -165,21 +176,9 @@ jobs:
       if [ '${{ parameters.asan }}' == True ]; then
         cp -vr tests/log/*/log/asan $(Build.ArtifactStagingDirectory)/
       fi
-
-      if [ '${{ parameters.archive_gcov }}' == True ]; then
-        sudo apt-get install -y lcov
-        cd $(Build.ArtifactStagingDirectory)/gcov_tmp/
-        tar -zcvf sonic-gcov.tar.gz sonic-gcov/
-        rm -rf sonic-gcov
-      fi
     displayName: "Collect logs"
     condition: always()
 
-  - publish: $(Build.ArtifactStagingDirectory)/gcov_tmp
-    artifact: ${{ parameters.gcov_artifact_name }}
-    displayName: "Publish gcov output"
-    condition: and(succeeded(), eq('${{ parameters.archive_gcov }}', true))
-
   - publish: $(Build.ArtifactStagingDirectory)/
     artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)
     displayName: "Publish logs"

azure-pipelines.yml

@@ -41,6 +41,7 @@ stages:
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: amd64
+      pool: sonicbld
       sonic_slave: sonic-slave-bullseye
       common_lib_artifact_name: common-lib
       swss_common_artifact_name: sonic-swss-common
@@ -55,6 +56,7 @@ stages:
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: amd64
+      pool: sonicbld
       sonic_slave: sonic-slave-bullseye
       common_lib_artifact_name: common-lib
       swss_common_artifact_name: sonic-swss-common
@@ -137,8 +139,8 @@ stages:
       asan: true
 
 - stage: Gcov
+  condition: false
   dependsOn: Test
-  condition: in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues')
   jobs:
   - template: .azure-pipelines/gcov.yml
     parameters:

cfgmgr/Makefile.am

@@ -102,20 +102,20 @@ macsecmgrd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_SAI) $(
 macsecmgrd_LDADD = $(LDFLAGS_ASAN) $(COMMON_LIBS) $(SAIMETA_LIBS)
 
 if GCOV_ENABLED
-vlanmgrd_LDADD += -lgcovpreload
-teammgrd_LDADD += -lgcovpreload
-portmgrd_LDADD += -lgcovpreload
-intfmgrd_LDADD+= -lgcovpreload
-buffermgrd_LDADD += -lgcovpreload
-vrfmgrd_LDADD += -lgcovpreload
-nbrmgrd_LDADD += -lgcovpreload
-vxlanmgrd_LDADD += -lgcovpreload
-sflowmgrd_LDADD += -lgcovpreload
-natmgrd_LDADD += -lgcovpreload
-coppmgrd_LDADD += -lgcovpreload
-tunnelmgrd_LDADD += -lgcovpreload
-macsecmgrd_LDADD += -lgcovpreload
-fabricmgrd_LDADD += -lgcovpreload
+vlanmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+teammgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+portmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+fabricmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+intfmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+buffermgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+vrfmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+nbrmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+vxlanmgrd_SOURCES +=  ../gcovpreload/gcovpreload.cpp
+sflowmgrd_SOURCES +=  ../gcovpreload/gcovpreload.cpp
+natmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+coppmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+tunnelmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
+macsecmgrd_SOURCES += ../gcovpreload/gcovpreload.cpp
 endif
 
 if ASAN_ENABLED

debian/rules

@@ -33,7 +33,7 @@ ifeq ($(ENABLE_ASAN), y)
 endif
 
 ifeq ($(ENABLE_GCOV), y)
-	configure_opts += --enable-gcov CFLAGS="-g -O0" CXXFLAGS="-g -O0"
+	configure_opts += --enable-gcov --enable-code-coverage CFLAGS="-g -O0" CXXFLAGS="-g -O0"
 endif
 
 override_dh_auto_configure:
@@ -43,9 +43,10 @@ override_dh_auto_install:
 	dh_auto_install --destdir=debian/swss
 ifeq ($(ENABLE_GCOV), y)
 	mkdir -p debian/swss/tmp/gcov
-	sh ./tests/gcov_support.sh collect swss
+	lcov -c --directory . --no-external --exclude "$(shell pwd)/tests/*" --exclude "$(shell pwd)/**/tests/*" --output-file coverage.info
+	lcov_cobertura coverage.info -o coverage.xml
+	find ./ -type f -regex '.*\.\(h\|cpp\|gcno\|info\)' | tar -cf debian/swss/tmp/gcov/gcov-source.tar -T -
 endif
 
 override_dh_strip:
 	dh_strip --dbg-package=swss-dbg
-

fdbsyncd/Makefile.am

@@ -15,7 +15,7 @@ fdbsyncd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(COV_CFLAGS) $(CF
 fdbsyncd_LDADD = $(LDFLAGS_ASAN) -lnl-3 -lnl-route-3 -lswsscommon $(COV_LDFLAGS)
 
 if GCOV_ENABLED
-fdbsyncd_LDADD += -lgcovpreload
+fdbsyncd_SOURCES += ../gcovpreload/gcovpreload.cpp
 endif
 
 if ASAN_ENABLED

fpmsyncd/Makefile.am

@@ -15,7 +15,7 @@ fpmsyncd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_ASAN)
 fpmsyncd_LDADD = $(LDFLAGS_ASAN) -lnl-3 -lnl-route-3 -lswsscommon
 
 if GCOV_ENABLED
-fpmsyncd_LDADD += -lgcovpreload
+fpmsyncd_SOURCES += ../gcovpreload/gcovpreload.cpp
 endif
 
 if ASAN_ENABLED

gcovpreload/Makefile

@@ -6,7 +6,7 @@ CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || ec
 
 DYLIBSUFFIX=so
 DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX)
-DYLIB_MAKE_CMD=$(CC) -shared -fpic gcovpreload.c -o ${DYLIBNAME}
+DYLIB_MAKE_CMD=$(CC) -shared -fpic gcovpreload.cpp -o ${DYLIBNAME}
 
 all:
 	$(DYLIB_MAKE_CMD)

gcovpreload/gcovpreload.c → gcovpreload/gcovpreload.cpp

@@ -2,15 +2,15 @@
 #include <stdlib.h>
 #include <signal.h>
 #include <signal.h>
-#define SIMPLE_WAY
+
+extern "C" void __gcov_dump();
 
 void sighandler(int signo)
 {
 #ifdef SIMPLE_WAY
     exit(signo);
 #else
-    extern void __gcov_flush();
-    __gcov_flush(); /* flush out gcov stats data */
+    __gcov_dump();
     raise(signo); /* raise the signal again to crash process */
 #endif
 }
@@ -33,9 +33,9 @@ void ctor()
     struct sigaction sa;
     sa.sa_handler = sighandler;
     sigemptyset(&sa.sa_mask);
-    sa.sa_flags = SA_RESETHAND;
+    sa.sa_flags = (int)SA_RESETHAND;
 
-    for(i = 0; i < sizeof(sigs)/sizeof(sigs[0]); ++i) {
+    for(i = 0; i < (int)(sizeof(sigs)/sizeof(sigs[0])); ++i) {
         if (sigaction(sigs[i], &sa, NULL) == -1) {
             perror("Could not set signal handler");
         }

mclagsyncd/Makefile.am

@@ -15,7 +15,7 @@ mclagsyncd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_ASAN)
 mclagsyncd_LDADD = $(LDFLAGS_ASAN) -lnl-3 -lnl-route-3 -lswsscommon
 
 if GCOV_ENABLED
-mclagsyncd_LDADD += -lgcovpreload
+mclagsyncd_SOURCES += ../gcovpreload/gcovpreload.cpp
 endif
 
 if ASAN_ENABLED

natsyncd/Makefile.am

@@ -15,7 +15,7 @@ natsyncd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_ASAN)
 natsyncd_LDADD = $(LDFLAGS_ASAN) -lnl-3 -lnl-route-3 -lnl-nf-3 -lswsscommon
 
 if GCOV_ENABLED
-natsyncd_LDADD += -lgcovpreload
+natsyncd_SOURCES += ../gcovpreload/gcovpreload.cpp
 endif
 
 if ASAN_ENABLED

neighsyncd/Makefile.am

@@ -15,7 +15,7 @@ neighsyncd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_ASAN)
 neighsyncd_LDADD = $(LDFLAGS_ASAN) -lnl-3 -lnl-route-3 -lswsscommon
 
 if GCOV_ENABLED
-neighsyncd_LDADD += -lgcovpreload
+neighsyncd_SOURCES += ../gcovpreload/gcovpreload.cpp
 endif
 
 if ASAN_ENABLED

orchagent/Makefile.am

@@ -148,9 +148,9 @@ orchagent_restart_check_CPPFLAGS = $(DBGFLAGS) $(AM_CPPFLAGS) $(CFLAGS_COMMON) $
 orchagent_restart_check_LDADD = $(LDFLAGS_ASAN) -lhiredis -lswsscommon -lpthread
 
 if GCOV_ENABLED
-orchagent_LDADD += -lgcovpreload
-routeresync_LDADD += -lgcovpreload
-orchagent_restart_check_LDADD += -lgcovpreload
+orchagent_SOURCES += ../gcovpreload/gcovpreload.cpp
+routeresync_SOURCES += ../gcovpreload/gcovpreload.cpp
+orchagent_restart_check_SOURCES += ../gcovpreload/gcovpreload.cpp
 endif
 
 if ASAN_ENABLED

portsyncd/Makefile.am

@@ -15,7 +15,7 @@ portsyncd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_ASAN)
 portsyncd_LDADD = $(LDFLAGS_ASAN) -lnl-3 -lnl-route-3 -lswsscommon
 
 if GCOV_ENABLED
-portsyncd_LDADD += -lgcovpreload
+portsyncd_SOURCES += ../gcovpreload/gcovpreload.cpp
 endif
 
 if ASAN_ENABLED