From b23f79f463bd43a4e1f19cd6ad10b92425856a62 Mon Sep 17 00:00:00 2001
From: Liu Shilong <shilongliu@microsoft.com>
Date: Thu, 23 Feb 2023 14:19:56 +0800
Subject: [PATCH] [ci] Fix docker hang issue and change template reference
 branch (#13894) (#13921)

Why I did it
Cherry pick PR(#13894)
Azure pipeline change.
Use common template to make it easy to change common steps. Fix docker hang issue.

How I did it
How to verify
---
 .azure-pipelines/azure-pipelines-build.yml    |  5 ++--
 .../azure-pipelines-image-template.yml        |  4 ++--
 .azure-pipelines/template-daemon.yml          | 24 +++++++++++++++++++
 3 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 .azure-pipelines/template-daemon.yml

diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml
index 992bbb4097ee..74e5153e8b9f 100644
--- a/.azure-pipelines/azure-pipelines-build.yml
+++ b/.azure-pipelines/azure-pipelines-build.yml
@@ -100,7 +100,8 @@ jobs:
             platform_rpc: nephos
 
     buildSteps:
-      - template: template-skipvstest.yml
+      - template: .azure-pipelines/template-skipvstest.yml@buildimage
+      - template: .azure-pipelines/template-daemon.yml@buildimage
       - bash: |
           (
             while true
@@ -170,4 +171,4 @@ jobs:
           fi
           sudo kill $daemon_pid
         displayName: "Build sonic image"
-      - template: check-dirty-version.yml
+      - template: .azure-pipelines/check-dirty-version.yml@buildimage
diff --git a/.azure-pipelines/azure-pipelines-image-template.yml b/.azure-pipelines/azure-pipelines-image-template.yml
index 67d75abc6887..d4d2c96e1c61 100644
--- a/.azure-pipelines/azure-pipelines-image-template.yml
+++ b/.azure-pipelines/azure-pipelines-image-template.yml
@@ -25,7 +25,7 @@ jobs:
     jobFilters: ${{ parameters.jobFilters }}
     jobVariables: ${{ parameters.jobVariables }}
     preSteps:
-      - template: cleanup.yml
+      - template: .azure-pipelines/cleanup.yml@buildimage
       - ${{ parameters.preSteps }}
       - script: |
           if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
@@ -67,6 +67,6 @@ jobs:
             artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
             publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)'
       - ${{ parameters.postSteps }}
-      - template: cleanup.yml
+      - template: .azure-pipelines/cleanup.yml@buildimage
     jobGroups: ${{ parameters.jobGroups }}
     buildSteps: ${{ parameters.buildSteps }}
diff --git a/.azure-pipelines/template-daemon.yml b/.azure-pipelines/template-daemon.yml
new file mode 100644
index 000000000000..903e10c2d5c6
--- /dev/null
+++ b/.azure-pipelines/template-daemon.yml
@@ -0,0 +1,24 @@
+steps:
+- bash: |
+    (
+      while true
+      do
+        sleep 120
+        now=$(date +%s)
+        pids=$(ps -C docker -o pid,etime,args | grep "docker build" | cut -d" " -f2)
+        for pid in $pids
+        do
+          start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)
+          time_s=$(($now-$start))
+          if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then
+            echo =========== $(date +%F%T) $time_s &>> target/daemon.log
+            ps $pid &>> target/daemon.log
+            sudo kill $pid
+          fi
+        done
+      done
+    ) &
+    daemon_pid=$!
+    ps $daemon_pid
+    echo $daemon_pid >> /tmp/azp_daemon_kill_docker_pid
+  displayName: start daemon to kill hang docker