Merge remote-tracking branch 'upsteam/master' into swssdk

.azure-pipelines/azure-pipelines-build.yml

@@ -132,3 +132,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
+      - template: check-dirty-version.yml

.azure-pipelines/azure-pipelines-download-certificate.yml

@@ -0,0 +1,33 @@
+parameters:
+- name: connectionName
+  type: string
+  default: sonic-dev-connection
+- name: kevaultName
+  type: string
+  default: sonic-kv
+- name: certificateName
+  type: string
+  default: sonic-secure-boot
+
+steps:
+- task: AzureKeyVault@2
+  inputs:
+    connectedServiceName: ${{ parameters.connectionName }}
+    keyVaultName: ${{ parameters.kevaultName }}
+    secretsFilter: ${{ parameters.certificateName }}
+
+- script: |
+    set -e
+    TMP_FILE=$(mktemp)
+    echo "$CERTIFICATE" | base64 -d > $TMP_FILE
+    sudo mkdir -p /etc/certificates
+    mkdir -p $(Build.StagingDirectory)/target
+    # Save the public key
+    openssl pkcs12 -in $TMP_FILE -clcerts --nokeys -nodes -passin pass: | sed -z -e "s/.*\(-----BEGIN CERTIFICATE\)/\1/" > $(SIGNING_CERT)
+    # Save the private key
+    openssl pkcs12 -in $TMP_FILE -nocerts -nodes -passin pass: | sed -z -e "s/.*\(-----BEGIN PRIVATE KEY\)/\1/" | sudo tee $(SIGNING_KEY) 1>/dev/null
+    ls -lt $(SIGNING_CERT) $(SIGNING_KEY)
+    rm $TMP_FILE
+  env:
+    CERTIFICATE: $(${{ parameters.certificateName }})
+  displayName: "Save certificate"

.azure-pipelines/azure-pipelines-image-template.yml

@@ -48,7 +48,7 @@ jobs:
           ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM_AZP) PLATFORM_ARCH=$(PLATFORM_ARCH) $(BUILD_OPTIONS) configure
         displayName: 'Make configure'
     postSteps:
-      - script: cp target -r $(Build.ArtifactStagingDirectory)/
+      - script: mv target $(Build.ArtifactStagingDirectory)/
         displayName: Copy Artifacts
         condition: always()
       - publish:  $(Build.ArtifactStagingDirectory)
@@ -58,6 +58,10 @@ jobs:
         condition: failed()
         artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)$(System.JobAttempt)'
         displayName: "Archive failed sonic image"
+      - template: trigger-publish-artifacts-build.yml
+        parameters:
+          artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
+          publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)'
       - ${{ parameters.postSteps }}
       - template: cleanup.yml
     jobGroups: ${{ parameters.jobGroups }}

.azure-pipelines/check-dirty-version.yml

@@ -0,0 +1,16 @@
+steps:
+- script: |
+    . functions.sh
+    SONIC_VERSION=$(sonic_get_version)
+    echo "SONIC_VERSION=$SONIC_VERSION"
+    if [[ "$SONIC_VERSION" == *dirty* ]]; then
+      # Print the detail dirty info
+      git status --untracked-files=no -s --ignore-submodules
+
+      # Exit with error, if it is a PR build
+      if [ "$(Build.Reason)" == "PullRequest" ]; then
+        echo "Build failed for the dirty version: $SONIC_VERSION" 1>&2
+        exit 1
+      fi
+    fi
+  displayName: "Check the dirty version"

.azure-pipelines/docker-sonic-slave.yml

@@ -60,7 +60,7 @@ stages:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
       - ${{ each arch in parameters.arches }}:
-        - ${{ if eq(variables[''Build.Reason], 'PullRequest') }}:
+        - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
           - template: docker-sonic-slave-template.yml
             parameters:
               pool: sonicbld

.azure-pipelines/official-build-cisco-8000.yml

@@ -22,10 +22,17 @@ resources:
     name: Cisco-8000-sonic/platform-cisco-8000
     endpoint: cisco-connection
 
+
 variables:
 - group: SONIC-AKV-STROAGE-1
 - name: StorageSASKey
   value: $(sonicstorage-SasToken)
+- name: SONIC_ENABLE_SECUREBOOT_SIGNATURE
+  value: y
+- name: SIGNING_KEY
+  value: /etc/certificates/sonic-secure-boot-private.pem
+- name: SIGNING_CERT
+  value: $(Build.StagingDirectory)/target/sonic-secure-boot-public.pem
 
 stages:
 - stage: Build
@@ -41,6 +48,7 @@ stages:
     parameters:
       buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
       preSteps:
+      - template: azure-pipelines-download-certificate.yml
       - checkout: self
         submodules: recursive
         path: s
@@ -90,5 +98,10 @@ stages:
           StorageSASKey: $(StorageSASKey)
         condition: ne(variables['Build.Reason'], 'PullRequest')
         displayName: "Override cisco sai packages"
+      - script: |
+          echo "SONIC_ENABLE_SECUREBOOT_SIGNATURE := y" >> rules/config.user
+          echo "SIGNING_KEY := $(SIGNING_KEY)" >> rules/config.user
+          echo "SIGNING_CERT := $(SIGNING_CERT)" >> rules/config.user
+        displayName: "Enable secure boot signature"
       jobGroups:
       - name: cisco-8000

.azure-pipelines/run-test-template.yml

@@ -7,6 +7,9 @@ parameters:
   type: string
 - name: ptf_name
   type: string
+- name: vmtype
+  type: string
+  default: 'ceos'
 - name: section
   type: string
   default: ''
@@ -41,7 +44,7 @@ steps:
     git reset --hard origin/master
     sed -i s/use_own_value/${username}/ ansible/veos_vtb
     echo aaa > ansible/password.txt
-    docker exec sonic-mgmt bash -c "pushd /data/sonic-mgmt/ansible;./testbed-cli.sh -d /data/sonic-vm -m $(inventory) -t $(testbed_file) -k ceos refresh-dut ${{ parameters.tbname }} password.txt" && sleep 180
+    docker exec sonic-mgmt bash -c "pushd /data/sonic-mgmt/ansible;./testbed-cli.sh -d /data/sonic-vm -m $(inventory) -t $(testbed_file) -k ${{ parameters.vmtype }} refresh-dut ${{ parameters.tbname }} password.txt" && sleep 180
   displayName: "Setup testbed"
 
 - script: |

.azure-pipelines/template-commonlib.yml

@@ -2,33 +2,39 @@ jobs:
 - job: Build
   timeoutInMinutes: 120
   pool: sonicbld
+  variables:
+  - template: template-variables.yml
   steps:
   - checkout: self
     clean: true
     submodules: recursive
   - script: |
       set -ex
-      case $(Build.SourceBranchName) in
-        202012 | 202106)
-          bldenv=buster
-          ;;
-        *)
-          bldenv=bullseye
-          ;;
-      esac
-      BLDENV=$bldenv make -f Makefile.work configure PLATFORM=vs ENABLE_DOCKER_BASE_PULL=y
-      echo "##vso[task.setvariable variable=bldenv;]$bldenv"
+      branch=$(Build.SourceBranchName)
+      # DIST_MASTER is set in variable.
+      BRANCH=DIST_${branch^^}
+      bldenvs=${!BRANCH}
+      [ "$bldenvs" == "" ] && bldenvs="$(COMMON_LIB_BUILD_ENVS)"
+      for bldenv in $bldenvs
+      do
+        BLDENV=$bldenv make -f Makefile.work configure PLATFORM=vs ENABLE_DOCKER_BASE_PULL=y
+      done
+      set +x
+      echo "##vso[task.setvariable variable=bldenvs;]$bldenvs"
     displayName: Make configure
   - script: |
       set -ex
-      LIBNL3_VERSION_BASE=$(grep "LIBNL3_VERSION_BASE =" rules/libnl3.mk | awk '{print$3}')
-      LIBNL3_VERSION=$(grep "LIBNL3_VERSION =" rules/libnl3.mk | awk '{print$3}' | sed -e "s/(//" -e "s/)//" -e "s/\\$//" -e "s/LIBNL3_VERSION_BASE/$LIBNL3_VERSION_BASE/")
-      BLDENV=$(bldenv) make -f Makefile.work target/debs/$(bldenv)/libnl-3-200_${LIBNL3_VERSION}_amd64.deb ENABLE_DOCKER_BASE_PULL=y
+      for bldenv in $(bldenvs)
+      do
+        LIBNL3_VERSION_BASE=$(grep "LIBNL3_VERSION_BASE =" rules/libnl3.mk | awk '{print$3}')
+        LIBNL3_VERSION=$(grep "LIBNL3_VERSION =" rules/libnl3.mk | awk '{print$3}' | sed -e "s/(//" -e "s/)//" -e "s/\\$//" -e "s/LIBNL3_VERSION_BASE/$LIBNL3_VERSION_BASE/")
+        SONIC_BUILD_JOBS=$(nproc) BLDENV=$bldenv make -f Makefile.work target/debs/$bldenv/libnl-3-200_${LIBNL3_VERSION}_amd64.deb ENABLE_DOCKER_BASE_PULL=y
 
-      LIBYANG_VERSION_BASE=$(grep "LIBYANG_VERSION_BASE =" rules/libyang.mk | awk '{print$3}')
-      LIBYANG_VERSION=$(grep "LIBYANG_VERSION =" rules/libyang.mk | awk '{print$3}' | sed -e "s/\\$//" -e "s/(//" -e "s/)//" -e "s/LIBYANG_VERSION_BASE/$LIBYANG_VERSION_BASE/")
-      BLDENV=$(bldenv) make -f Makefile.work target/debs/$(bldenv)/libyang_${LIBYANG_VERSION}_amd64.deb
-      find target -name *.deb | xargs -i cp {} $(Build.ArtifactStagingDirectory)
+        LIBYANG_VERSION_BASE=$(grep "LIBYANG_VERSION_BASE =" rules/libyang.mk | awk '{print$3}')
+        LIBYANG_VERSION=$(grep "LIBYANG_VERSION =" rules/libyang.mk | awk '{print$3}' | sed -e "s/\\$//" -e "s/(//" -e "s/)//" -e "s/LIBYANG_VERSION_BASE/$LIBYANG_VERSION_BASE/")
+        SONIC_BUILD_JOBS=$(nproc) BLDENV=$bldenv make -f Makefile.work target/debs/$bldenv/libyang_${LIBYANG_VERSION}_amd64.deb
+      done
+      mv target $(Build.ArtifactStagingDirectory)
     displayName: Make common lib packages
   - publish: $(Build.ArtifactStagingDirectory)
     artifact: common-lib

.azure-pipelines/template-variables.yml

@@ -1,2 +1,3 @@
 variables:
   DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
+  COMMON_LIB_BUILD_ENVS: 'bullseye'

.azure-pipelines/trigger-publish-artifacts-build.yml

@@ -0,0 +1,64 @@
+# The steps to trigger the pipeline to publish the artifacts
+
+parameters:
+- name: artifactName
+  type: string
+  default: ""
+- name: publishPrefix
+  type: string
+  default: "$(Build.DefinitionName)/$(Build.SourceBranchName)"
+
+steps:
+- script: |
+    . functions.sh
+    sonic_version=$(sonic_get_version)
+    latest_tag=$(git describe --tags --abbrev=0)
+    docker_tags="$sonic_version $(Build.SourceBranchName)"
+    if [ "$(Build.SourceBranchName)" == "master" ]; then
+      docker_tags="$docker_tags latest"
+    fi
+    echo "##vso[task.setvariable variable=sonic_version]$sonic_version"
+    echo "##vso[task.setvariable variable=latest_tag]$latest_tag"
+    echo "##vso[task.setvariable variable=docker_tags]$docker_tags"
+  condition: ne(variables['Build.Reason'], 'PullRequest')
+  displayName: 'Set trigger build variables'
+- task: TriggerBuild@4
+  condition: ne(variables['Build.Reason'], 'PullRequest')
+  inputs:
+    definitionIsInCurrentTeamProject: false
+    teamProject: internal
+    tfsServer: $(System.CollectionUri)
+    buildDefinition: 'publish-artifacts'
+    queueBuildForUserThatTriggeredBuild: true
+    ignoreSslCertificateErrors: false
+    useSameSourceVersion: false
+    useCustomSourceVersion: false
+    useSameBranch: false
+    waitForQueuedBuildsToFinish: false
+    storeInEnvironmentVariable: true
+    authenticationMethod: 'Personal Access Token'
+    password: '$(system.accesstoken)'
+    enableBuildInQueueCondition: false
+    dependentOnSuccessfulBuildCondition: false
+    dependentOnFailedBuildCondition: false
+    checkbuildsoncurrentbranch: false
+    failTaskIfConditionsAreNotFulfilled: false
+    buildParameters: ''
+    templateParameters: |
+      pipelineContext: {"buildId":"$(Build.BuildId)",
+        "pipelineId":"$(System.DefinitionId)",
+        "project": "$(System.TeamProject)",
+        "branchName":"$(Build.SourceBranchName)"},
+      artifactContext: {"artifactName":"${{ parameters.artifactName }}",
+        "artifactPatterns":"**/*.bin\n
+              **/*.swi\n
+              **/*.raw\n
+              **/*.img.gz\n
+              **/*-rpc.gz\n
+              **/python-saithrift*.deb"},
+      publishContext: {"publishPrefix":"${{ parameters.publishPrefix }}",
+        "keepArtifactName":false,
+        "dockerImagePatterns":"target/*-rpc.gz",
+        "dockerTags":"$(docker_tags)",
+        "version":"$(sonic_version)",
+        "latestTag":"$(latest_tag)"}

.github/pull_request_template.md

@@ -42,7 +42,7 @@ pull request for inclusion in the changelog:
 <!--
 Provide a link to config_db schema for the table for which YANG model
 is defined
-Link should point to correct section on https://github.com/Azure/SONiC/wiki/Configuration.
+Link should point to correct section on https://github.com/Azure/sonic-buildimage/blob/master/src/sonic-yang-models/doc/Configuration.md
 -->
 
 #### A picture of a cute animal (not mandatory but encouraged)

.gitmodules

@@ -103,6 +103,6 @@
 [submodule "src/sonic-p4rt/sonic-pins"]
 	path = src/sonic-p4rt/sonic-pins
 	url = https://github.com/Azure/sonic-pins.git
-[submodule "src/thrift_0_14_1/thrift"]
-	path = src/thrift_0_14_1/thrift
-	url = https://github.com/apache/thrift.git
+[submodule "src/ptf-py3"]
+	path = src/ptf-py3
+	url = https://github.com/p4lang/ptf.git

Makefile

@@ -40,6 +40,7 @@ endif
 ifeq ($(NOBULLSEYE), 0)
 	BLDENV=bullseye make -f Makefile.work $@
 endif
+	BLDENV=bullseye make -f Makefile.work docker-cleanup
 
 jessie:
 	@echo "+++ Making $@ +++"
@@ -83,7 +84,7 @@ $(PLATFORM_PATH):
 configure : $(PLATFORM_PATH)
 	$(call make_work, $@)
 
-clean reset showtag sonic-slave-build sonic-slave-bash :
+clean reset showtag docker-cleanup sonic-slave-build sonic-slave-bash :
 	$(call make_work, $@)
 
 # Freeze the versions, see more detail options: scripts/versions_manager.py freeze -h

Makefile.cache

@@ -186,7 +186,7 @@ define GET_MOD_DEP_SHA
 	$(if $($(1)_DEP_FILES_MISSING), $(warning "[ DPKG ] Dependecy file(s) are not found for $(1) : $($(1)_DEP_FILES_MISSING)))
 
 	# Include package dependencies hash values into package hash calculation
-	$(eval $(1)_DEP_PKGS_SHA := $(foreach dfile,$(1)_MOD_DEP_PKGS,$(dfile)_DEP_MOD_SHA $(dfile)_MOD_HASH))
+	$(eval $(1)_DEP_PKGS_SHA := $(foreach dfile,$($(1)_MOD_DEP_PKGS),$($(dfile)_DEP_MOD_SHA) $($(dfile)_MOD_HASH)))
 
 	$(eval $(1)_DEP_MOD_SHA := $(shell bash -c "git hash-object  $($(1)_DEP_MOD_SHA_FILES) && echo $($(1)_DEP_PKGS_SHA)" \
                             | sha1sum | awk '{print substr($$1,0,23);}'))