diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml index 992bbb4097ee..74e5153e8b9f 100644 --- a/.azure-pipelines/azure-pipelines-build.yml +++ b/.azure-pipelines/azure-pipelines-build.yml @@ -100,7 +100,8 @@ jobs: platform_rpc: nephos buildSteps: - - template: template-skipvstest.yml + - template: .azure-pipelines/template-skipvstest.yml@buildimage + - template: .azure-pipelines/template-daemon.yml@buildimage - bash: | ( while true @@ -170,4 +171,4 @@ jobs: fi sudo kill $daemon_pid displayName: "Build sonic image" - - template: check-dirty-version.yml + - template: .azure-pipelines/check-dirty-version.yml@buildimage diff --git a/.azure-pipelines/azure-pipelines-image-template.yml b/.azure-pipelines/azure-pipelines-image-template.yml index 67d75abc6887..d4d2c96e1c61 100644 --- a/.azure-pipelines/azure-pipelines-image-template.yml +++ b/.azure-pipelines/azure-pipelines-image-template.yml @@ -25,7 +25,7 @@ jobs: jobFilters: ${{ parameters.jobFilters }} jobVariables: ${{ parameters.jobVariables }} preSteps: - - template: cleanup.yml + - template: .azure-pipelines/cleanup.yml@buildimage - ${{ parameters.preSteps }} - script: | if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then @@ -67,6 +67,6 @@ jobs: artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)' publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)' - ${{ parameters.postSteps }} - - template: cleanup.yml + - template: .azure-pipelines/cleanup.yml@buildimage jobGroups: ${{ parameters.jobGroups }} buildSteps: ${{ parameters.buildSteps }} diff --git a/.azure-pipelines/template-daemon.yml b/.azure-pipelines/template-daemon.yml new file mode 100644 index 000000000000..903e10c2d5c6 --- /dev/null +++ b/.azure-pipelines/template-daemon.yml @@ -0,0 +1,24 @@ +steps: +- bash: | + ( + while true + do + sleep 120 + now=$(date +%s) + pids=$(ps -C docker -o pid,etime,args | grep "docker build" | cut -d" " -f2) + for pid in $pids + do + start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s) + time_s=$(($now-$start)) + if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then + echo =========== $(date +%F%T) $time_s &>> target/daemon.log + ps $pid &>> target/daemon.log + sudo kill $pid + fi + done + done + ) & + daemon_pid=$! + ps $daemon_pid + echo $daemon_pid >> /tmp/azp_daemon_kill_docker_pid + displayName: start daemon to kill hang docker