From 24de0beb7a4a91f20101b52e09131ad42bacdc91 Mon Sep 17 00:00:00 2001 From: Yakiv Huryk Date: Sat, 25 Jun 2022 15:35:39 +0000 Subject: [PATCH 1/2] [ci][asan] add DVS tests run with ASAN This adds 3 new stages: * BuildAsan: same as 'Build' but with ENABLE_ASAN=y * BuildDockerAsan: same as 'BuildDocker' but uses asan-enabled swss and adds swss-dbg package. It also uses asan-enabled docker-sonic-vs from the sonic-buildimage ci. * TestAsan: same as 'Test' but uses asan-enabled docker and publishes the asan reports if any. It also fails if asan report dir is not empty (which means that there is some memory usage issue in swss) Signed-off-by: Yakiv Huryk --- .../build-docker-sonic-vs-template.yml | 21 +++++++--- .azure-pipelines/build-template.yml | 7 ++++ .azure-pipelines/docker-sonic-vs/Dockerfile | 2 + .../test-docker-sonic-vs-template.yml | 42 ++++++++++++++++--- azure-pipelines.yml | 37 ++++++++++++++++ 5 files changed, 98 insertions(+), 11 deletions(-) diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml index ff0ff6c0cb..602af8a320 100644 --- a/.azure-pipelines/build-docker-sonic-vs-template.yml +++ b/.azure-pipelines/build-docker-sonic-vs-template.yml @@ -23,6 +23,10 @@ parameters: - name: artifact_name type: string +- name: asan + type: boolean + default: false + jobs: - job: displayName: ${{ parameters.arch }} @@ -66,13 +70,13 @@ jobs: runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' path: $(Build.ArtifactStagingDirectory)/download - patterns: '**/target/docker-sonic-vs.gz' - displayName: "Download sonic-buildimage docker-sonic-vs" + patterns: '**/target/${{ parameters.artifact_name }}.gz' + displayName: "Download sonic-buildimage ${{ parameters.artifact_name }}" - script: | set -ex echo $(Build.DefinitionName).$(Build.BuildNumber) - docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz + docker load < $(Build.ArtifactStagingDirectory)/download/target/${{ parameters.artifact_name }}.gz mkdir -p .azure-pipelines/docker-sonic-vs/debs @@ -80,13 +84,18 @@ jobs: pushd .azure-pipelines - docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs + build_args="" + if [ '${{ parameters.asan }}' == True ]; then + build_args="--build-arg need_dbg=y" + fi + + docker build $build_args --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} docker-sonic-vs popd - docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz + docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz rm -rf $(Build.ArtifactStagingDirectory)/download - displayName: "Build docker-sonic-vs" + displayName: "Build ${{ parameters.artifact_name }}" - publish: $(Build.ArtifactStagingDirectory)/ artifact: ${{ parameters.artifact_name }} displayName: "Archive sonic docker vs image" diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index f8040bca56..b3719c341e 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -43,6 +43,10 @@ parameters: - name: common_lib_artifact_name type: string +- name: asan + type: boolean + default: false + jobs: - job: displayName: ${{ parameters.arch }} @@ -138,6 +142,9 @@ jobs: if [ '${{ parameters.archive_gcov }}' == True ]; then export ENABLE_GCOV=y fi + if [ '${{ parameters.asan }}' == True ]; then + export ENABLE_ASAN=y + fi ./autogen.sh dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb . displayName: "Compile sonic swss" diff --git a/.azure-pipelines/docker-sonic-vs/Dockerfile b/.azure-pipelines/docker-sonic-vs/Dockerfile index 33cdb7e8dc..a7098553b2 100644 --- a/.azure-pipelines/docker-sonic-vs/Dockerfile +++ b/.azure-pipelines/docker-sonic-vs/Dockerfile @@ -1,6 +1,7 @@ FROM docker-sonic-vs ARG docker_container_name +ARG need_dbg ADD ["debs", "/debs"] @@ -17,6 +18,7 @@ RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb RUN dpkg --purge swss RUN dpkg -i /debs/swss_1.0.0_amd64.deb +RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/swss-dbg_1.0.0_amd64.deb ; fi RUN apt-get update diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 83fd36dc09..2e3d582599 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -16,6 +16,14 @@ parameters: type: boolean default: false +- name: docker_sonic_vs_name + type: string + default: docker-sonic-vs + +- name: asan + type: boolean + default: false + jobs: - job: displayName: vstest @@ -30,9 +38,9 @@ jobs: - checkout: self - task: DownloadPipelineArtifact@2 inputs: - artifact: docker-sonic-vs + artifact: ${{ parameters.docker_sonic_vs_name }} path: $(Build.ArtifactStagingDirectory)/download - displayName: "Download pre-stage built docker-sonic-vs" + displayName: "Download pre-stage built ${{ parameters.docker_sonic_vs_name }}" - task: DownloadPipelineArtifact@2 inputs: source: specific @@ -68,11 +76,16 @@ jobs: sudo /sbin/ip link del Vrf1 type vrf table 1001 pushd tests + test_args="" if [ '${{ parameters.archive_gcov }}' == True ]; then - sudo py.test -v --force-flaky --junitxml=tr.xml --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) - else - sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) + test_args="--keeptb" fi + if [ '${{ parameters.asan }}' == True ]; then + test_args="--graceful-stop" + fi + + sudo py.test $test_args -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} + rm -rf $(Build.ArtifactStagingDirectory)/download displayName: "Run vs tests" @@ -85,6 +98,10 @@ jobs: - script: | cp -r tests/log $(Build.ArtifactStagingDirectory)/ + if [ '${{ parameters.asan }}' == True ]; then + cp -r tests/log/*/log/asan $(Build.ArtifactStagingDirectory)/ + fi + if [ '${{ parameters.archive_gcov }}' == True ]; then sudo apt-get install -y lcov ./tests/gcov_support.sh set_environment $(Build.ArtifactStagingDirectory) @@ -103,3 +120,18 @@ jobs: artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt) displayName: "Publish logs" condition: always() + + - publish: $(Build.ArtifactStagingDirectory)/asan + artifact: asan-reports + displayName: "Publish ASAN reports" + condition: eq('${{ parameters.asan }}', true) + + - script: | + if [ "$(ls -A $(Build.ArtifactStagingDirectory)/asan)" ]; then + echo "There are issues reported by ASAN" + exit 1 + else + echo "No issues reported by ASAN" + fi + displayName: "Check ASAN reports" + condition: eq('${{ parameters.asan }}', true) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 166905654e..7311c73d09 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,6 +49,19 @@ stages: archive_pytests: true archive_gcov: true +- stage: BuildAsan + + jobs: + - template: .azure-pipelines/build-template.yml + parameters: + arch: amd64 + sonic_slave: sonic-slave-buster + common_lib_artifact_name: common-lib + swss_common_artifact_name: sonic-swss-common + sairedis_artifact_name: sonic-sairedis + artifact_name: sonic-swss-asan + asan: true + - stage: BuildArm dependsOn: Build condition: succeeded('Build') @@ -88,6 +101,18 @@ stages: swss_artifact_name: sonic-swss artifact_name: docker-sonic-vs +- stage: BuildDockerAsan + dependsOn: BuildAsan + condition: succeeded('BuildAsan') + jobs: + - template: .azure-pipelines/build-docker-sonic-vs-template.yml + parameters: + swss_common_artifact_name: sonic-swss-common + sairedis_artifact_name: sonic-sairedis + swss_artifact_name: sonic-swss-asan + artifact_name: docker-sonic-vs-asan + asan: true + - stage: Test dependsOn: BuildDocker condition: succeeded('BuildDocker') @@ -99,6 +124,18 @@ stages: sonic_slave: sonic-slave-buster archive_gcov: true +- stage: TestAsan + dependsOn: BuildDockerAsan + condition: succeeded('BuildDockerAsan') + jobs: + - template: .azure-pipelines/test-docker-sonic-vs-template.yml + parameters: + log_artifact_name: log-asan + gcov_artifact_name: sonic-gcov + sonic_slave: sonic-slave-buster + docker_sonic_vs_name: docker-sonic-vs-asan + asan: true + - stage: Gcov dependsOn: Test condition: always() From 0cdf974e6e7ae4307c3e42c02b347a86c0781af9 Mon Sep 17 00:00:00 2001 From: Yakiv Huryk Date: Thu, 14 Jul 2022 14:25:15 +0000 Subject: [PATCH 2/2] [ci][asan] disabled ASAN report check Signed-off-by: Yakiv Huryk --- .azure-pipelines/test-docker-sonic-vs-template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 6a04a2feae..8f79acdc0e 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -140,7 +140,6 @@ jobs: - script: | if [ "$(ls -A $(Build.ArtifactStagingDirectory)/asan)" ]; then echo "There are issues reported by ASAN" - exit 1 else echo "No issues reported by ASAN" fi