Merge branch 'master' into qiluo/tagged

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -11,32 +11,51 @@ schedules:
   displayName: Daily Build
   branches:
     include:
-    - 202012
+    - 20*
+    exclude:
+    - 200*
+    - 201*
+    - 202006
   always: true
 
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
 pool: sonicbld
 
+parameters:
+- name: 'jobFilters'
+  type: object
+  default:
+  - vs
+  - barefoot
+  - broadcom
+  - centec
+  - centec-arm64
+  - generic
+  - marvell-armhf
+  - mellanox
+
 stages:
 - stage: Build
   variables:
-    CACHE_MODE: none
-    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - name: CACHE_MODE
+      value: none
+    - name: VERSION_CONTROL_OPTIONS
+      value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
     parameters:
-      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
+      jobFilters: ${{ parameters.jobFilters }}
+      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
-      - script: |
-          containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
-          if [ ! -z "$containers" ]; then
-            docker container kill $containers || true
-            sleep 5
-          fi
-          images=$(docker images 'sonic-slave-*' -a -q)
-          if [ ! -z "$images" ]; then
-            docker rmi -f $images
-          fi
-        displayName: 'Cleanup sonic slave'
+      - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions
@@ -60,14 +79,14 @@ stages:
         default_platform=broadcom
         artifacts=$(find $(Pipeline.Workspace) -maxdepth 1 -type d -name 'sonic-buildimage.*' | grep -v "sonic-buildimage.${default_platform}")
         echo "artifacts$artifacts"
-        cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/versions target/
+        cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/target/versions target/
         make freeze FREEZE_VERSION_OPTIONS=-r
         find files/build/versions 
         ordered_artifacts=$(echo "$artifacts" | grep -v -E "arm64|armhf" && echo "$artifacts" | grep -E "arm64|armhf")
         for artifact in $ordered_artifacts
         do
           rm -rf target/versions
-          cp -r $artifact/versions target/
+          cp -r $artifact/target/versions target/
           OPTIONS="-a -d"
           [[ "$artifact" == *arm64* || "$artifact" == *armhf* ]] && OPTIONS="-d"
           make freeze FREEZE_VERSION_OPTIONS="$OPTIONS"

.azure-pipelines/azure-pipelines-build.yml

@@ -50,6 +50,7 @@ jobs:
             swi_image: yes
 
         - name: broadcom
+          timeoutInMinutes: 1440
           variables:
             dbg_image: yes
             swi_image: yes
@@ -64,7 +65,7 @@ jobs:
             platform_rpc: centec
 
         - name: centec-arm64
-          pool: sonicbld_8c
+          pool: sonicbld-arm64
           timeoutInMinutes: 2880
           variables:
             PLATFORM_ARCH: arm64
@@ -78,7 +79,7 @@ jobs:
             dbg_image: yes
 
         - name: marvell-armhf
-          pool: sonicbld_8c
+          pool: sonicbld-armhf
           timeoutInMinutes: 2880
           variables:
             PLATFORM_ARCH: armhf
@@ -101,7 +102,8 @@ jobs:
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
             if [ $(dbg_image) == yes ]; then
-              make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
+              make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz
+              mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
             fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
             if [ $(Build.Reason) != 'PullRequest' ];then
@@ -111,8 +113,8 @@ jobs:
             fi
           else
             if [ $(dbg_image) == yes ]; then
-              make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin && \
-                mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
+              make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin
+              mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
             fi
             if [ $(swi_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
@@ -130,3 +132,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
+      - template: check-dirty-version.yml

.azure-pipelines/azure-pipelines-image-template.yml

@@ -26,16 +26,17 @@ jobs:
     jobVariables: ${{ parameters.jobVariables }}
     preSteps:
       - template: cleanup.yml
-      - ${{ parameters. preSteps }}
+      - ${{ parameters.preSteps }}
       - script: |
-          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox)$"; then
+          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
             CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
             BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
             echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
           fi
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
+        condition: eq(variables.SKIP_CHECKOUT, '') 
         displayName: 'Checkout code'
       - script: |
           BRANCH_NAME=$(Build.SourceBranchName)
@@ -49,9 +50,14 @@ jobs:
     postSteps:
       - script: cp target -r $(Build.ArtifactStagingDirectory)/
         displayName: Copy Artifacts
+        condition: always()
       - publish:  $(Build.ArtifactStagingDirectory)
         artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
         displayName: "Archive sonic image"
+      - publish:  $(Build.ArtifactStagingDirectory)
+        condition: failed()
+        artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)$(System.JobAttempt)'
+        displayName: "Archive failed sonic image"
       - ${{ parameters.postSteps }}
       - template: cleanup.yml
     jobGroups: ${{ parameters.jobGroups }}

.azure-pipelines/azure-pipelines-job-groups.yml

@@ -19,7 +19,7 @@ parameters:
   default: ''
 - name: 'timeoutInMinutes'
   type: 'number'
-  default: 720
+  default: 900
 - name: 'jobFilters'
   type: object
   default: ''

.azure-pipelines/azure-pipelines-repd-build-variables.yml

@@ -0,0 +1,5 @@
+variables:
+  ${{ if and(ge(variables['Build.SourceBranchName'], '202012'), le(variables['Build.SourceBranchName'], '299999')) }}:
+    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
+  ${{ if or(lt(variables['Build.SourceBranchName'], '202012'), gt(variables['Build.SourceBranchName'], '299999')) }}:
+    VERSION_CONTROL_OPTIONS: ''

.azure-pipelines/build-commonlib.yml

@@ -0,0 +1,19 @@
+pr: none
+trigger: none
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily build
+  branches:
+    include:
+      - master
+      - 202???
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
+jobs:
+- template: .azure-pipelines/template-commonlib.yml@buildimage

.azure-pipelines/check-dirty-version.yml

@@ -0,0 +1,16 @@
+steps:
+- script: |
+    . functions.sh
+    SONIC_VERSION=$(sonic_get_version)
+    echo "SONIC_VERSION=$SONIC_VERSION"
+    if [[ "$SONIC_VERSION" == *dirty* ]]; then
+      # Print the detail dirty info
+      git status --untracked-files=no -s --ignore-submodules
+
+      # Exit with error, if it is a PR build
+      if [ "$(Build.Reason)" == "PullRequest" ]; then
+        echo "Build failed for the dirty version: $SONIC_VERSION" 1>&2
+        exit 1
+      fi
+    fi
+  displayName: "Check the dirty version"

.azure-pipelines/docker-sonic-slave-arm64.yml

@@ -0,0 +1,52 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
+schedules:
+- cron: "0 8 * * *"
+  branches:
+    include:
+    - master
+    - 202012
+  always: true
+
+trigger: none
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - sonic-slave-jessie
+    - sonic-slave-stretch
+    - sonic-slave-buster
+    - sonic-slave-bullseye
+    - .azure-pipelines
+
+parameters:
+- name: 'dists'
+  type: object
+  default:
+  - bullseye
+  - buster
+  - stretch
+
+stages:
+- stage: Build
+  jobs:
+  - ${{ each dist in parameters.dists }}:
+    - ${{ if contains(variables['Build.DefinitionName'], dist) }}:
+      - template: docker-sonic-slave-template.yml
+        parameters:
+          pool: sonicbld-arm64
+          arch: arm64
+          dist: ${{ dist }}

.azure-pipelines/docker-sonic-slave-armhf.yml

@@ -0,0 +1,52 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
+schedules:
+- cron: "0 8 * * *"
+  branches:
+    include:
+    - master
+    - 202012
+  always: true
+
+trigger: none
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - sonic-slave-jessie
+    - sonic-slave-stretch
+    - sonic-slave-buster
+    - sonic-slave-bullseye
+    - .azure-pipelines
+
+parameters:
+- name: 'dists'
+  type: object
+  default:
+  - bullseye
+  - buster
+  - stretch
+
+stages:
+- stage: Build
+  jobs:
+  - ${{ each dist in parameters.dists }}:
+    - ${{ if contains(variables['Build.DefinitionName'], dist) }}:
+      - template: docker-sonic-slave-template.yml
+        parameters:
+          pool: sonicbld-armhf
+          arch: armhf
+          dist: ${{ dist }}