[Submodule][202211] Submodule update redis 202211

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -42,12 +42,32 @@ parameters:
   - mellanox
 
 stages:
+- stage: Prepare
+  jobs:
+  - job: Prepare
+    steps:
+    - script: |
+        DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
+        DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
+        DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
+        echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_TIMESTAMP;isOutput=true]$DEBIAN_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_SECURITY_TIMESTAMP;isOutput=true]$DEBIAN_SECURITY_TIMESTAMP"
+      name: SetVersions
+      displayName: 'Set snapshot versions'
 - stage: Build
+  dependsOn: Prepare
   variables:
     - name: CACHE_MODE
       value: none
     - name: VERSION_CONTROL_OPTIONS
       value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - name: SKIP_CHECKOUT
+      value: true
+    - name: DEBIAN_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_TIMESTAMP'] ]
+    - name: DEBIAN_SECURITY_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_SECURITY_TIMESTAMP'] ]
     - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
@@ -56,6 +76,21 @@ stages:
       buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
       - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
+      - checkout: self
+        submodules: recursive
+        fetchDepth: 0
+        path: s
+        displayName: 'Checkout code'
+      - script: |
+          echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+          if [ "$MIRROR_SNAPSHOT" == y ]; then
+            mkdir -p target/versions/default/
+            echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
+            echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
+            cat target/versions/default/versions-mirror
+          fi
+        displayName: 'Set snapshot versions'
+
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions

.azure-pipelines/azure-pipelines-build.yml

@@ -108,7 +108,8 @@ jobs:
             platform_rpc: nephos
 
     buildSteps:
-      - template: template-skipvstest.yml
+      - template: .azure-pipelines/template-skipvstest.yml@buildimage
+      - template: .azure-pipelines/template-daemon.yml@buildimage
       - bash: |
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
@@ -124,7 +125,7 @@ jobs:
             make $BUILD_OPTIONS target/docker-ptf-sai.gz
             if [ $(Build.Reason) != 'PullRequest' ];then
               gzip -kd target/sonic-vs.img.gz
-              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make $BUILD_OPTIONS sonic-slave-run
               rm target/sonic-vs.img
             fi
           else
@@ -139,9 +140,20 @@ jobs:
               make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
             fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
+              # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+              pushd ./src/sonic-sairedis/SAI
+              git stash
+              popd
               if [ $(GROUP_NAME) == broadcom ]; then
                 make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-dnx-rpc.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd 
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd
               fi
             fi
             if [ $(syncd_rpc_image) == yes ]; then
@@ -151,4 +163,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
-      - template: check-dirty-version.yml
+      - template: .azure-pipelines/check-dirty-version.yml@buildimage

.azure-pipelines/azure-pipelines-image-template.yml

@@ -25,7 +25,7 @@ jobs:
     jobFilters: ${{ parameters.jobFilters }}
     jobVariables: ${{ parameters.jobVariables }}
     preSteps:
-      - template: cleanup.yml
+      - template: .azure-pipelines/cleanup.yml@buildimage
       - ${{ parameters.preSteps }}
       - script: |
           if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
@@ -36,6 +36,7 @@ jobs:
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
+        fetchDepth: 0
         condition: and(succeeded(), eq(variables.SKIP_CHECKOUT, ''))
         displayName: 'Checkout code'
       - script: |
@@ -66,6 +67,6 @@ jobs:
             artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
             publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)'
       - ${{ parameters.postSteps }}
-      - template: cleanup.yml
+      - template: .azure-pipelines/cleanup.yml@buildimage
     jobGroups: ${{ parameters.jobGroups }}
     buildSteps: ${{ parameters.buildSteps }}

.azure-pipelines/build-template.yml

@@ -106,6 +106,16 @@ jobs:
         fi
         if [ ${{ parameters.sync_rpc_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
+          # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache
+          pushd ./src/sonic-sairedis/SAI
+          git stash
+          popd 
+          if [ ${{ parameters.platform }} == broadcom ]; then
+            make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+            pushd ./src/sonic-sairedis/SAI
+            git stash
+            popd 
+          fi
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin

.azure-pipelines/cleanup.yml

@@ -1,5 +1,11 @@
 steps:
 - script: |
+    set -x
+    # kill daemon process
+    ps $(cat /tmp/azp_daemon_kill_docker_pid)
+    sudo kill $(cat /tmp/azp_daemon_kill_docker_pid)
+    rm /tmp/azp_daemon_kill_docker_pid
+
     if sudo [ -f /var/run/march/docker.pid ] ; then
         pid=`sudo cat /var/run/march/docker.pid` ; sudo kill $pid
     fi
@@ -11,4 +17,5 @@ steps:
         pid=`sudo cat dockerfs/var/run/docker.pid` ; sudo kill $pid
     fi
     sudo rm -rf $(ls -A1)
+  condition: always()
   displayName: "Clean Workspace"

.azure-pipelines/docker-sonic-slave-template.yml

@@ -35,7 +35,7 @@ parameters:
   - sonicbld-armhf
 
 jobs:
-- job: Build_${{ parameters.dist }}_${{ parameters.march }}${{ parameters.arch }}
+- job: sonic_slave_${{ parameters.dist }}${{ parameters.march }}
   timeoutInMinutes: 360
   variables:
   - template: /.azure-pipelines/template-variables.yml@buildimage
@@ -46,7 +46,6 @@ jobs:
   - template: /.azure-pipelines/template-clean-sonic-slave.yml@buildimage
   - checkout: self
     clean: true
-    submodules: recursive
   - task: Docker@2
     displayName: Login to ACR
     inputs:
@@ -56,6 +55,10 @@ jobs:
       set -ex
       image_tag=$(BLDENV=${{ parameters.dist }} make -f Makefile.work showtag PLATFORM=generic PLATFORM_ARCH=${{ parameters.arch }} | grep sonic-slave | tail -n 1)
       image_latest=$(echo $(echo $image_tag | awk -F: '{print$1}'):latest)
+      if echo ${{ parameters.pool }} | grep ${{ parameters.arch }};then
+        image_latest=$(echo ${image_latest} | sed 's/:/-${{ parameters.arch }}:/')
+      fi
+      image_branch=$(echo $(echo $image_latest | awk -F: '{print$1}'):$(Build.SourceBranchName))
       docker rmi $image_tag || true
 
       if [[ "$(Build.Reason)" =~ [a-zA-Z]*CI ]] && docker pull ${{ parameters.registry_url }}/${image_tag};then
@@ -69,7 +72,9 @@ jobs:
 
       docker tag ${image_tag} ${REGISTRY_SERVER}/${image_tag}
       docker push ${REGISTRY_SERVER}/${image_tag}
-      if [[ "${{ parameters.arch }}" == "amd64" ]];then
+      docker tag ${image_tag} ${REGISTRY_SERVER}/${image_branch}
+      docker push ${REGISTRY_SERVER}/${image_branch}
+      if [[ "$(Build.SourceBranchName)" == "master" ]];then
         docker tag ${image_tag} ${REGISTRY_SERVER}/${image_latest}
         docker push ${REGISTRY_SERVER}/${image_latest}
       fi

.azure-pipelines/docker-sonic-slave.yml

@@ -30,7 +30,6 @@ trigger:
   paths:
     include:
     - sonic-slave-*
-    - src/sonic-build-hooks
     - files/build/versions
     - Makefile
     - Makefile.work
@@ -57,7 +56,7 @@ parameters:
   default: sonicdev
 
 stages:
-- stage: Build
+- stage: Build_in_amd64
   jobs:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
@@ -67,7 +66,9 @@ stages:
             pool: sonicbld
             arch: ${{ arch }}
             dist: ${{ dist }}
-- stage: Build_march
+            ${{ if ne(arch, 'amd64') }}:
+              march: _march_${{ arch }}
+- stage: Build_native_arm
   dependsOn: []
   jobs:
   - ${{ each dist in parameters.dists }}:
@@ -79,4 +80,4 @@ stages:
               pool: sonicbld-${{ arch }}
               arch: ${{ arch }}
               dist: ${{ dist }}
-              march: march_
+              march: _${{ arch }}

.azure-pipelines/run-test-scheduler-template.yml

@@ -11,12 +11,12 @@ parameters:
   default: 36000
 
 - name: MIN_WORKER
-  type: number
+  type: string
   default: 1
 
 - name: MAX_WORKER
-  type: number
-  default: 2
+  type: string
+  default: 1
 
 - name: TEST_SET
   type: string
@@ -81,11 +81,10 @@ steps:
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
       echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
       # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states PREPARE_TESTBED EXECUTING KVMDUMP FINISHED CANCELLED FAILED
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state LOCK_TESTBED
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Lock testbed
-    timeoutInMinutes: 240
 
   - script: |
       set -ex
@@ -95,36 +94,33 @@ steps:
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
       echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
       # When "PREPARE_TESTBED" finish, it changes into "EXECUTING"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 2400 --expected-states EXECUTING KVMDUMP FINISHED CANCELLED FAILED
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state PREPARE_TESTBED
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Prepare testbed
-    timeoutInMinutes: 40
 
   - script: |
       set -ex
       echo "Run test"
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
       echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
       # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 18000 --expected-states KVMDUMP FINISHED CANCELLED FAILED
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state EXECUTING
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Run test
-    timeoutInMinutes: 300
 
   - script: |
       set -ex
       echo "KVM dump"
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
       echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
       # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states FINISHED CANCELLED FAILED
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
     condition: succeededOrFailed()
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: KVM dump
-    timeoutInMinutes: 20
 
   - script: |
       set -ex

.azure-pipelines/template-daemon.yml

@@ -0,0 +1,24 @@
+steps:
+- bash: |
+    (
+      while true
+      do
+        sleep 120
+        now=$(date +%s)
+        pids=$(ps -C docker -o pid,etime,args | grep "docker build" | cut -d" " -f2)
+        for pid in $pids
+        do
+          start=$(date --date="$(ls -dl /proc/$pid --time-style full-iso | awk '{print$6,$7}')" +%s)
+          time_s=$(($now-$start))
+          if [[ $time_s -gt $(DOCKER_BUILD_TIMEOUT) ]]; then
+            echo =========== $(date +%F%T) $time_s &>> target/daemon.log
+            ps $pid &>> target/daemon.log
+            sudo kill $pid
+          fi
+        done
+      done
+    ) &
+    daemon_pid=$!
+    ps $daemon_pid
+    echo $daemon_pid >> /tmp/azp_daemon_kill_docker_pid
+  displayName: start daemon to kill hang docker

.azure-pipelines/template-variables.yml

@@ -2,3 +2,7 @@ variables:
   DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
   COMMON_LIB_BUILD_ENVS: 'bullseye'
   SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
+  SONIC_BUILD_RETRY_COUNT: 3
+  SONIC_BUILD_RETRY_INTERVAL: 600
+  DOCKER_BUILDKIT: 0
+  DOCKER_BUILD_TIMEOUT: 3600

.github/workflows/automerge_scan.yml

@@ -0,0 +1,59 @@
+name: AutoMergeScan
+on:
+  schedule:
+    - cron: '31 */2 * * *'
+  workflow_dispatch:
+
+jobs:
+  automerge_scan:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Debug
+      env:
+        TOKEN: ${{ secrets.TOKEN }}
+      run: |
+        set -e
+
+        echo ${TOKEN} | gh auth login --with-token
+        gh pr list -R sonic-net/sonic-buildimage -A mssonicbld --json additions,assignees,author,baseRefName,body,changedFiles,closed,closedAt,comments,commits,createdAt,deletions,files,headRefName,headRepository,headRepositoryOwner,id,isCrossRepository,isDraft,labels,latestReviews,maintainerCanModify,mergeCommit,mergeStateStatus,mergeable,mergedAt,mergedBy,milestone,number,potentialMergeCommit,projectCards,reactionGroups,reviewDecision,reviewRequests,reviews,state,statusCheckRollup,title,updatedAt,url > prs.log
+        cat prs.log | jq
+    - name: Main
+      run: |
+        set -e
+
+        count=$(cat prs.log | jq 'length')
+        for ((i=0;i<$count;i++))
+        do
+          url=$(cat prs.log | jq -r ".[$i].url")
+          created_at=$(cat prs.log | jq -r ".[$i].createdAt")
+          echo PR: $(($i+1))/$count, URL: $url, createdAt: $created_at, now: $(date -u +"%FT%TZ")
+          [[ "$url" == "" ]] && continue
+          [[ $created_at > $(date --date "1 hour ago" -u +"%FT%TZ") ]] && continue
+          checks=$(cat prs.log | jq ".[$i].statusCheckRollup")
+          checks_count=$(echo $checks | jq 'length')
+          echo Checks count: $checks_count
+          for ((j=0;j<$checks_count;j++))
+          do
+            check=$(echo $checks | jq ".[$j]")
+            state=$(echo $check | jq -r '.state')
+            conclusion=$(echo $check | jq -r '.conclusion')
+
+            # EasyCLA success flag:  state=SUCCESS
+            # Others success flag:   conclusion in SUCCESS,NEUTRAL
+            if [[ "$state" == "SUCCESS" ]];then
+              # check pass
+              continue
+            elif [[ "$conclusion" == "SUCCESS" ]] || [[ "$conclusion" == "NEUTRAL" ]];then
+              # check pass
+              continue
+            else
+              echo "$url Check failed!!!"
+              echo $check | jq
+              continue 2
+            fi
+          done
+          # merge the PR
+          echo ========Merging  PR========
+          gh pr merge --rebase --admin -R sonic-net/sonic-buildimage $url || true
+          echo ========Finished PR========
+        done