Merge remote-tracking branch 'upstream/master' into celestica_e1031_s…

…ecurity

.azure-pipelines/azure-pipelines-build.yml

@@ -139,6 +139,9 @@ jobs:
             fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+              if [ $(GROUP_NAME) == broadcom ]; then
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-dnx-rpc.gz
+              fi
             fi
             if [ $(syncd_rpc_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/sonic-$(GROUP_NAME).bin

.azure-pipelines/azure-pipelines-repd-build-variables.yml

@@ -1,6 +1,6 @@
 variables:
   ${{ if eq(variables['Build.Reason'],'PullRequest') }}:
-    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=$([ "$(System.PullRequest.TargetBranch)" != "master" ] && echo deb,py2,py3,web,git,docker)'
+    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=$([[ "$(System.PullRequest.TargetBranch)" =~ ^20[2-9][0-9]{3}$ ]] && echo deb,py2,py3,web,git,docker)'
   ${{ elseif ne(variables['Build.SourceBranchName'],'master') }}:
     VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
   ${{ else }}:

.azure-pipelines/docker-sonic-slave-template.yml

@@ -10,6 +10,9 @@ parameters:
   - amd64
   - armhf
   - arm64
+- name: march
+  type: string
+  default: ''
 - name: dist
   type: string
   values:
@@ -32,89 +35,44 @@ parameters:
   - sonicbld-armhf
 
 jobs:
-- job: Build_${{ parameters.dist }}_${{ parameters.arch }}
+- job: Build_${{ parameters.dist }}_${{ parameters.march }}${{ parameters.arch }}
   timeoutInMinutes: 360
+  variables:
+  - template: /.azure-pipelines/template-variables.yml@buildimage
+  - template: /.azure-pipelines/azure-pipelines-repd-build-variables.yml@buildimage
   pool: ${{ parameters.pool }}
   steps:
   - template: cleanup.yml
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-    - template: template-clean-sonic-slave.yml
-  - ${{ else }}:
-    - template: '/.azure-pipelines/template-clean-sonic-slave.yml@buildimage'
+  - template: /.azure-pipelines/template-clean-sonic-slave.yml@buildimage
   - checkout: self
     clean: true
     submodules: recursive
+  - task: Docker@2
+    displayName: Login to ACR
+    inputs:
+      command: login
+      containerRegistry: ${{ parameters.registry_conn }}
   - bash: |
       set -ex
+      image_tag=$(BLDENV=${{ parameters.dist }} make -f Makefile.work showtag PLATFORM=generic PLATFORM_ARCH=${{ parameters.arch }} | grep sonic-slave | tail -n 1)
+      image_latest=$(echo $(echo $image_tag | awk -F: '{print$1}'):latest)
+      docker rmi $image_tag || true
 
-      SLAVE_DIR=sonic-slave-${{ parameters.dist }}
-      if [ x${{ parameters.pool }} == x"sonicbld" ]; then
-        if [ x${{ parameters.arch }} == x"amd64" ]; then
-          SLAVE_BASE_IMAGE=${SLAVE_DIR}
-          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}
-        elif [ x${{ parameters.pool }} == x"sonicbld" ]; then
-          SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ parameters.arch }}
-          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-march-${{ parameters.arch }}
-        fi
-      elif [[ x${{ parameters.pool }} == x"sonicbld-armhf" && x${{ parameters.arch }} == x"armhf" ]]; then
-        SLAVE_BASE_IMAGE=${SLAVE_DIR}
-        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-armhf
-      elif [[ x${{ parameters.pool }} == x"sonicbld-arm64" && x${{ parameters.arch }} == x"arm64" ]]; then
-        SLAVE_BASE_IMAGE=${SLAVE_DIR}
-        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-arm64
-      else
-        echo "do not support build ${{ parameters.arch }} on ${{ parameters.pool }}"
-        exit 1
+      if [[ "$(Build.Reason)" =~ [a-zA-Z]*CI ]] && docker pull ${{ parameters.registry_url }}/${image_tag};then
+        exit 0
       fi
 
-      if [ x"$(Build.SourceBranchName)" == x"202012" ]; then
-        BUILD_OPTIONS = 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
+      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ parameters.dist }} make -f Makefile.work configure PLATFORM=generic PLATFORM_ARCH=${{ parameters.arch }} $args || docker image ls $image_tag
+      if [[ "$(Build.Reason)" == "PullRequest" ]];then
+        exit 0
       fi
 
-      tmpfile=$(mktemp)
-
-      echo ${{ parameters.arch }} > .arch
-
-      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ parameters.dist }} $(BUILD_OPTIONS) make -f Makefile.work sonic-slave-build | tee $tmpfile
-      SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
-      SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
-
-      mkdir -p target
-
-      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:latest
-      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:$SLAVE_BASE_TAG
-      if [ "$SLAVE_BASE_IMAGE_UPLOAD" != "$SLAVE_DIR" ]; then
-        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:latest-${{ parameters.arch }}
-        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:$SLAVE_BASE_TAG
+      docker tag ${image_tag} ${REGISTRY_SERVER}/${image_tag}
+      docker push ${REGISTRY_SERVER}/${image_tag}
+      if [[ "${{ parameters.arch }}" == "amd64" ]];then
+        docker tag ${image_tag} ${REGISTRY_SERVER}/${image_latest}
+        docker push ${REGISTRY_SERVER}/${image_latest}
       fi
-      set +x
-      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE_UPLOAD"
-      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
     env:
       REGISTRY_SERVER: ${{ parameters.registry_url }}
     displayName: Build sonic-slave-${{ parameters.dist }}-${{ parameters.arch }}
-
-  - task: Docker@2
-    condition: ne(variables['Build.Reason'], 'PullRequest')
-    displayName: Upload image
-    inputs:
-      containerRegistry: ${{ parameters.registry_conn }}
-      repository: $(VARIABLE_SLAVE_BASE_IMAGE)
-      command: push
-      ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG)
-          latest
-      ${{ else }}:
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG)
-  - ${{ if ne(parameters.arch, 'amd64') }}:
-    - task: Docker@2
-      condition: ne(variables['Build.Reason'], 'PullRequest')
-      displayName: Upload image ${{ parameters.dist }}
-      inputs:
-        containerRegistry: ${{ parameters.registry_conn }}
-        repository: "sonic-slave-${{ parameters.dist }}"
-        command: push
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG) 

.azure-pipelines/docker-sonic-slave.yml

@@ -12,26 +12,28 @@ resources:
     endpoint: sonic-net
 
 schedules:
-- cron: "0 8 * * *"
+- cron: "0 0 * * 0"
+  displayName: Weekly build
   branches:
     include:
     - master
-    - 202012
+    - 202???
   always: true
 
-trigger: none
-pr:
+pr: none
+trigger:
+  batch: true
   branches:
     include:
     - master
+    - 202???
   paths:
     include:
-    - sonic-slave-jessie
-    - sonic-slave-stretch
-    - sonic-slave-buster
-    - sonic-slave-bullseye
+    - sonic-slave-*
     - src/sonic-build-hooks
-    - .azure-pipelines
+    - files/build/versions
+    - Makefile
+    - Makefile.work
 
 parameters:
 - name: 'arches'
@@ -60,15 +62,21 @@ stages:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
       - ${{ each arch in parameters.arches }}:
-        - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-          - template: docker-sonic-slave-template.yml
-            parameters:
-              pool: sonicbld
-              arch: ${{ arch }}
-              dist: ${{ dist }}
-        - ${{ else }}:
-          - template: '/.azure-pipelines/docker-sonic-slave-template.yml@buildimage'
+        - template: .azure-pipelines/docker-sonic-slave-template.yml@buildimage
+          parameters:
+            pool: sonicbld
+            arch: ${{ arch }}
+            dist: ${{ dist }}
+- stage: Build_march
+  dependsOn: []
+  jobs:
+  - ${{ each dist in parameters.dists }}:
+    - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
+      - ${{ each arch in parameters.arches }}:
+        - ${{ if ne(arch, 'amd64') }}:
+          - template: .azure-pipelines/docker-sonic-slave-template.yml@buildimage
             parameters:
-              pool: sonicbld
+              pool: sonicbld-${{ arch }}
               arch: ${{ arch }}
               dist: ${{ dist }}
+              march: march_

.azure-pipelines/official-build.yml

@@ -38,7 +38,7 @@ stages:
   variables:
   - name: CACHE_MODE
     value: wcache
-  - template: azure-pipelines-repd-build-variables.yml
+  - template: .azure-pipelines/azure-pipelines-repd-build-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
     parameters:

.azure-pipelines/run-test-template.yml

@@ -51,9 +51,6 @@ steps:
     rm -rf $(Build.ArtifactStagingDirectory)/*
     docker exec sonic-mgmt bash -c "/data/sonic-mgmt/tests/kvmtest.sh -en -T ${{ parameters.tbtype }} ${{ parameters.tbname }} ${{ parameters.dut }} ${{ parameters.section }}"
   displayName: "Run tests"
-  ${{ if eq(parameters.tbtype, 'multi-asic-t1-lag-pr') }}:
-      continueOnError: true
-
 - script: |
     # save dut state if test fails
     virsh_version=$(virsh --version)

.github/pull_request_template.md

@@ -39,6 +39,8 @@ Write a short (one line) summary that describes the changes in this
 pull request for inclusion in the changelog:
 -->
 
+#### Ensure to add label/tag for the feature raised. example - [PR#2174](https://github.com/sonic-net/sonic-utilities/pull/2174) where, Generic Config and Update feature has been labelled as GCU.
+
 #### Link to config_db schema for YANG module changes
 <!--
 Provide a link to config_db schema for the table for which YANG model

.github/workflows/automerge.yml

@@ -15,7 +15,7 @@ jobs:
     - name: automerge
       uses: 'pascalgn/automerge-action@v0.13.1'
       env:
-        GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        GITHUB_TOKEN: '${{ secrets.TOKEN }}'
         MERGE_LABELS: 'automerge'
         MERGE_METHOD: 'squash'
         MERGE_FILTER_AUTHOR: 'mssonicbld'

.github/workflows/semgrep.yml

@@ -0,0 +1,21 @@
+name: Semgrep
+
+on:
+  pull_request: {}
+  push:
+    branches:
+      - master
+      - '201[7-9][0-1][0-9]'
+      - '202[0-9][0-1][0-9]'
+
+jobs:
+  semgrep:
+    name: Semgrep
+    runs-on: ubuntu-latest
+    container:
+      image: returntocorp/semgrep
+    steps:
+      - uses: actions/checkout@v3
+      - run: semgrep ci
+        env:
+           SEMGREP_RULES: p/default