Merged PR 7418678: [202012] manual merge github 202012 branch to inte…

…rnal-202012 branch

Related work items: sonic-net#2108, sonic-net#2560, sonic-net#9486, sonic-net#12325, sonic-net#12626, sonic-net#12739, sonic-net#12835, sonic-net#12836, sonic-net#12865, sonic-net#12880, sonic-net#12903, sonic-net#12914, sonic-net#12922, sonic-net#12925, sonic-net#12930, sonic-net#12989, sonic-net#12990, sonic-net#13006, sonic-net#13014, sonic-net#13040, sonic-net#13092, sonic-net#13124, sonic-net#13132, sonic-net#13146, sonic-net#13157, sonic-net#13168, sonic-net#13175, sonic-net#13177, sonic-net#13182, sonic-net#13188, sonic-net#13226, sonic-net#13233, sonic-net#13244, sonic-net#13249

.azure-pipelines/azure-pipelines-build.yml

@@ -123,7 +123,17 @@ jobs:
               make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
             fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
+              # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+              pushd ./src/sonic-sairedis/SAI
+              git stash
+              popd
+              if [ $(GROUP_NAME) == broadcom ]; then
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd
+              fi
             fi
             if [ $(syncd_rpc_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/sonic-$(GROUP_NAME).bin

.azure-pipelines/build-template.yml

@@ -105,6 +105,16 @@ jobs:
         fi
         if [ ${{ parameters.sync_rpc_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
+          # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache
+          pushd ./src/sonic-sairedis/SAI
+          git stash
+          popd 
+          if [ ${{ parameters.platform }} == broadcom ]; then
+            make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+            pushd ./src/sonic-sairedis/SAI
+            git stash
+            popd 
+          fi
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin

.azure-pipelines/docker-sonic-slave.yml

@@ -30,7 +30,6 @@ trigger:
   paths:
     include:
     - sonic-slave-*
-    - src/sonic-build-hooks
     - files/build/versions
     - Makefile
     - Makefile.work
@@ -56,7 +55,7 @@ parameters:
   default: sonicdev
 
 stages:
-- stage: Build
+- stage: Build_in_amd64
   jobs:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
@@ -66,7 +65,9 @@ stages:
             pool: sonicbld
             arch: ${{ arch }}
             dist: ${{ dist }}
-- stage: Build_march
+            ${{ if ne(arch, 'amd64') }}:
+              march: _march_${{ arch }}
+- stage: Build_native_arm
   dependsOn: []
   jobs:
   - ${{ each dist in parameters.dists }}:
@@ -78,4 +79,4 @@ stages:
               pool: sonicbld-${{ arch }}
               arch: ${{ arch }}
               dist: ${{ dist }}
-              march: march_
+              march: _${{ arch }}

.azure-pipelines/run-test-scheduler-template.yml

@@ -11,11 +11,11 @@ parameters:
   default: 36000
 
 - name: MIN_WORKER
-  type: number
+  type: string
   default: 1
 
 - name: MAX_WORKER
-  type: number
+  type: string
   default: 2
 
 - name: TEST_SET
@@ -67,11 +67,10 @@ steps:
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
       echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
       # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states PREPARE_TESTBED EXECUTING KVMDUMP FINISHED CANCELLED FAILED
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state LOCK_TESTBED
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Lock testbed
-    timeoutInMinutes: 240
 
   - script: |
       set -ex
@@ -81,36 +80,33 @@ steps:
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
       echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
       # When "PREPARE_TESTBED" finish, it changes into "EXECUTING"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 2400 --expected-states EXECUTING KVMDUMP FINISHED CANCELLED FAILED
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state PREPARE_TESTBED
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Prepare testbed
-    timeoutInMinutes: 40
 
   - script: |
       set -ex
       echo "Run test"
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
       echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
       # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 18000 --expected-states KVMDUMP FINISHED CANCELLED FAILED
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state EXECUTING
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Run test
-    timeoutInMinutes: 300
 
   - script: |
       set -ex
       echo "KVM dump"
       echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
       echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
       # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states FINISHED CANCELLED FAILED
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
     condition: succeededOrFailed()
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: KVM dump
-    timeoutInMinutes: 20
 
   - script: |
       set -ex

.azure-pipelines/template-variables.yml

@@ -0,0 +1,6 @@
+variables:
+  DEFAULT_CONTAINER_REGISTRY: 'publicmirror.azurecr.io'
+  COMMON_LIB_BUILD_ENVS: 'bullseye'
+  SONIC_SLAVE_DOCKER_DRIVER: 'overlay2'
+  SONIC_BUILD_RETRY_COUNT: 3
+  SONIC_BUILD_RETRY_INTERVAL: 600

Makefile

@@ -15,16 +15,17 @@ PLATFORM_PATH := platform/$(if $(PLATFORM),$(PLATFORM),$(CONFIGURED_PLATFORM))
 PLATFORM_CHECKOUT := platform/checkout
 PLATFORM_CHECKOUT_FILE := $(PLATFORM_CHECKOUT)/$(PLATFORM).ini
 PLATFORM_CHECKOUT_CMD := $(shell if [ -f $(PLATFORM_CHECKOUT_FILE) ]; then PLATFORM_PATH=$(PLATFORM_PATH) j2 $(PLATFORM_CHECKOUT)/template.j2 $(PLATFORM_CHECKOUT_FILE); fi)
+MAKE_WITH_RETRY := ./scripts/run_with_retry $(MAKE)
 
 %::
 	@echo "+++ --- Making $@ --- +++"
 ifeq ($(NOJESSIE), 0)
-	EXTRA_DOCKER_TARGETS=$(notdir $@) make -f Makefile.work jessie
+	$(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) -f Makefile.work jessie
 endif
 ifeq ($(NOSTRETCH), 0)
-	EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch make -f Makefile.work stretch
+	$(MAKE_WITH_RETRY) EXTRA_DOCKER_TARGETS=$(notdir $@) BLDENV=stretch -f Makefile.work stretch
 endif
-	BLDENV=buster make -f Makefile.work $@
+	$(MAKE_WITH_RETRY) BLDENV=buster -f Makefile.work $@
 
 jessie:
 	@echo "+++ Making $@ +++"

azure-pipelines.yml

@@ -166,30 +166,31 @@ stages:
     pool:
       vmImage: 'ubuntu-20.04'
     displayName: "kvmtest-t0 by TestbedV2"
-    timeoutInMinutes: 1080
+    timeoutInMinutes: 240
     condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
     continueOnError: false
     steps:
     - template: .azure-pipelines/run-test-scheduler-template.yml
       parameters:
         TOPOLOGY: t0
-        MIN_WORKER: 2
-        MAX_WORKER: 3
+        MIN_WORKER: $(T0_INSTANCE_NUM)
+        MAX_WORKER: $(T0_INSTANCE_NUM)
         MGMT_BRANCH: 202012
 
   - job: t0_2vlans_testbedv2
     pool:
       vmImage: 'ubuntu-20.04'
     displayName: "kvmtest-t0-2vlans by TestbedV2"
-    timeoutInMinutes: 1080
+    timeoutInMinutes: 240
     condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
     continueOnError: false
     steps:
     - template: .azure-pipelines/run-test-scheduler-template.yml
       parameters:
         TOPOLOGY: t0
         TEST_SET: t0-2vlans
-        MAX_WORKER: 1
+        MIN_WORKER: $(T0_2VLANS_INSTANCE_NUM)
+        MAX_WORKER: $(T0_2VLANS_INSTANCE_NUM)
         DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a"
         MGMT_BRANCH: 202012
 
@@ -243,15 +244,15 @@ stages:
     pool:
       vmImage: 'ubuntu-20.04'
     displayName: "kvmtest-t1-lag by TestbedV2"
-    timeoutInMinutes: 600
+    timeoutInMinutes: 240
     condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
     continueOnError: false
     steps:
     - template: .azure-pipelines/run-test-scheduler-template.yml
       parameters:
         TOPOLOGY: t1-lag
-        MIN_WORKER: 2
-        MAX_WORKER: 3
+        MIN_WORKER: $(T1_LAG_INSTANCE_NUM)
+        MAX_WORKER: $(T1_LAG_INSTANCE_NUM)
         MGMT_BRANCH: 202012
 
   - job:
@@ -285,14 +286,14 @@ stages:
     pool:
       vmImage: 'ubuntu-20.04'
     displayName: "kvmtest-dualtor-t0 by TestbedV2"
-    timeoutInMinutes: 1080
+    timeoutInMinutes: 240
     condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES'))
     continueOnError: false
     steps:
       - template: .azure-pipelines/run-test-scheduler-template.yml
         parameters:
           TOPOLOGY: dualtor
-          MIN_WORKER: 1
-          MAX_WORKER: 1
+          MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
+          MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
           COMMON_EXTRA_PARAMS: "--disable_loganalyzer "
           MGMT_BRANCH: 202012

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/BALANCED

@@ -0,0 +1 @@
+../../../common/profiles/td2/gen/BALANCED

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/RDMA-CENTRIC

@@ -0,0 +1 @@
+../../../common/profiles/td2/gen/RDMA-CENTRIC

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/TCP-CENTRIC

@@ -0,0 +1 @@
+../../../common/profiles/td2/gen/TCP-CENTRIC

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/buffer_ports_t0.j2

@@ -0,0 +1,6 @@
+{%- macro generate_port_lists(PORT_ALL) %}
+    {# Generate list of ports #}
+    {% for port_idx in range(0,32) %}
+        {% if PORT_ALL.append("Ethernet%d" % (port_idx * 4)) %}{% endif %}
+    {% endfor %}
+{%- endmacro %}

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/buffer_ports_t1.j2

@@ -0,0 +1,6 @@
+{%- macro generate_port_lists(PORT_ALL) %}
+    {# Generate list of ports #}
+    {% for port_idx in range(0,32) %}
+        {% if PORT_ALL.append("Ethernet%d" % (port_idx * 4)) %}{% endif %}
+    {% endfor %}
+{%- endmacro %}

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/buffers_defaults_t0.j2

@@ -0,0 +1 @@
+BALANCED/buffers_defaults_t0.j2

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/buffers_defaults_t1.j2

@@ -0,0 +1 @@
+BALANCED/buffers_defaults_t1.j2

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/pg_profile_lookup.ini

@@ -0,0 +1 @@
+BALANCED/pg_profile_lookup.ini

device/arista/x86_64-arista_7050_qx32/Arista-7050-QX32/qos.json.j2

@@ -0,0 +1 @@
+BALANCED/qos.json.j2

device/arista/x86_64-arista_7050_qx32s/Arista-7050QX-32S-S4Q31/BALANCED

@@ -0,0 +1 @@
+../../../common/profiles/td2/gen/BALANCED

device/arista/x86_64-arista_7050_qx32s/Arista-7050QX-32S-S4Q31/RDMA-CENTRIC

@@ -0,0 +1 @@
+../../../common/profiles/td2/gen/RDMA-CENTRIC