Bringing changes from sonic-build image

.azure-pipelines/azure-pipelines-build-ubuntu-2004.yml

@@ -0,0 +1,55 @@
+# C/C++ with GCC
+# Build your C/C++ project with GCC using make.
+# Add steps that publish test results, save build artifacts, deploy, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/apps/c-cpp/gcc
+pr: none
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+
+stages:
+- stage: Build
+
+  jobs:
+  - job:
+    displayName: "amd64/ubuntu-20.04"
+    pool:
+      vmImage: 'ubuntu-20.04'
+
+    steps:
+    - checkout: self
+      submodules: true
+    - script: |
+        sudo apt-get update
+        sudo apt-get install -y make wget libtool m4 autoconf dh-exec libdebhelper-perl=13.6ubuntu1~bpo20.04.1 debhelper=13.6ubuntu1~bpo20.04.1 \
+                         cmake pkg-config python3-pip python cmake libgtest-dev libgmock-dev libyang-dev \
+                         debhelper-compat dh-elpa dh-sequence-python3 python3-all \
+                         libpython3-all-dev python3-six xmlto unzip rake-compiler gem2deb pkg-php-tools \
+                         ant default-jdk maven-repo-helper libguava-java
+        wget http://ftp.us.debian.org/debian/pool/main/libg/libgoogle-gson-java/libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
+        sudo dpkg -i libgoogle-gson-java_2.8.6-1+deb11u1_all.deb
+        mkdir -p /tmp/artifacts
+      displayName: "Install dependencies"
+    - script: |
+        SONIC_CONFIG_MAKE_JOBS=$(nproc) CONFIGURED_ARCH=amd64 DEST=/tmp/artifacts make -f ../rules/protobuf.mk -f protobuf/Makefile
+      workingDirectory: src
+      displayName: "Build protobuf"
+    - script: |
+        sudo dpkg -i protobuf-compiler_3.21.12-3_amd64.deb libprotoc32_3.21.12-3_amd64.deb \
+                      libprotobuf32_3.21.12-3_amd64.deb libprotobuf-dev_3.21.12-3_amd64.deb \
+                      libprotobuf-lite32_3.21.12-3_amd64.deb
+      workingDirectory: /tmp/artifacts
+      displayName: "Install protobuf"
+    - script: |
+        dpkg-buildpackage -rfakeroot -b -us -uc
+      workingDirectory: src/sonic-dash-api
+      displayName: "Build sonic-dash-api"
+    - script: |
+        cp *.deb /tmp/artifacts
+      workingDirectory: src
+    - publish: /tmp/artifacts
+      artifact: sonic-buildimage.amd64.ubuntu20_04
+      displayName: "Archive sonic-buildimage debian packages for ubuntu20.04"

.azure-pipelines/run-test-scheduler-template.yml

@@ -18,6 +18,10 @@ parameters:
   type: string
   default: 1
 
+- name: NUM_ASIC
+  type: number
+  default: 1
+
 - name: TEST_SET
   type: string
   default: ""
@@ -34,17 +38,73 @@ parameters:
   type: string
   default: "ceos"
 
-- name: SPECIFIED_PARAMS
+- name: TESTBED_NAME
+  type: string
+  default: ""
+
+- name: IMAGE_URL
+  type: string
+  default: ""
+
+- name: HWSKU
+  type: string
+  default: ""
+
+- name: TEST_PLAN_TYPE
+  type: string
+  default: ""
+
+- name: PLATFORM
+  type: string
+  default: ""
+
+- name: SCRIPTS
+  type: string
+  default: ""
+
+- name: FEATURES
+  type: string
+  default: ""
+
+- name: SCRIPTS_EXCLUDE
+  type: string
+  default: ""
+
+- name: FEATURES_EXCLUDE
   type: string
-  default: "{}"
+  default: ""
+
+- name: REPO_NAME
+  type: string
+  default: ""
 
 - name: MGMT_BRANCH
   type: string
-  default: master
+  default: ""
 
-- name: NUM_ASIC
+- name: STOP_ON_FAILURE
+  type: string
+  default: ""
+
+- name: RETRY_TIMES
+  type: string
+  default: ""
+
+- name: DUMP_KVM_IF_FAIL
+  type: string
+  default: "True"
+  values:
+  - "True"
+  - "False"
+
+- name: REQUESTER
+  type: string
+  default: ""
+
+- name: MAX_RUN_TEST_MINUTES
   type: number
-  default: 1
+  default: 480
+
 
 steps:
   - script: |
@@ -57,12 +117,20 @@ steps:
       set -e
       pip install PyYAML
       rm -f new_test_plan_id.txt
-      python ./.azure-pipelines/test_plan.py create -t ${{ parameters.TOPOLOGY }} -o new_test_plan_id.txt \
-      --min-worker ${{ parameters.MIN_WORKER }} --max-worker ${{ parameters.MAX_WORKER }} \
-      --test-set ${{ parameters.TEST_SET }} --kvm-build-id $(KVM_BUILD_ID) \
-      --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" --common-extra-params "${{ parameters.COMMON_EXTRA_PARAMS }}" \
-      --mgmt-branch ${{ parameters.MGMT_BRANCH }} --vm-type ${{ parameters.VM_TYPE }} --specified-params "${{ parameters.SPECIFIED_PARAMS }}" \
+
+      python ./.azure-pipelines/test_plan.py create \
+      -t ${{ parameters.TOPOLOGY }} \
+      -o new_test_plan_id.txt \
+      --min-worker ${{ parameters.MIN_WORKER }} \
+      --max-worker ${{ parameters.MAX_WORKER }} \
+      --test-set ${{ parameters.TEST_SET }} \
+      --kvm-build-id $(KVM_BUILD_ID) \
+      --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" \
+      --common-extra-params "${{ parameters.COMMON_EXTRA_PARAMS }}" \
+      --mgmt-branch ${{ parameters.MGMT_BRANCH }} \
+      --vm-type ${{ parameters.VM_TYPE }} \
       --num-asic ${{ parameters.NUM_ASIC }}
+
       TEST_PLAN_ID=`cat new_test_plan_id.txt`
 
       echo "Created test plan $TEST_PLAN_ID"
@@ -108,18 +176,20 @@ steps:
     env:
       TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
     displayName: Run test
-
-  - script: |
-      set -e
-      echo "KVM dump"
-      echo "SONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com"
-      echo "Runtime detailed progress at $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
-      # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
-      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
-    condition: succeededOrFailed()
-    env:
-      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
-    displayName: KVM dump
+    timeoutInMinutes: ${{ parameters.MAX_RUN_TEST_MINUTES }}
+
+  - ${{ if eq(parameters.DUMP_KVM_IF_FAIL, 'True') }}:
+    - script: |
+        set -e
+        echo "KVM dump"
+        echo "SONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com"
+        echo "Runtime detailed progress at $(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
+        # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
+        python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
+      condition: succeededOrFailed()
+      env:
+        TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+      displayName: KVM dump
 
   - script: |
       set -e

.gitmodules

@@ -118,3 +118,6 @@
 [submodule "src/sonic-device-health"]
 	path = src/sonic-device-health
 	url = https://github.com/renukamanavalan/sonic-device-health.git
+[submodule "src/sonic-dash-api/sonic-dash-api"]
+	path = src/sonic-dash-api/sonic-dash-api
+	url = https://github.com/sonic-net/sonic-dash-api.git

Makefile.work

@@ -177,13 +177,13 @@ DOCKER_ROOT = $(PWD)/fsroot.docker.$(BLDENV)
 
 # Support FIPS feature, armhf not supported yet
 ifeq ($(PLATFORM_ARCH),armhf)
-ENABLE_FIPS_FEATURE := n
+INCLUDE_FIPS := n
 ENABLE_FIPS := n
 endif
 
-ifeq ($(ENABLE_FIPS_FEATURE), n)
+ifeq ($(INCLUDE_FIPS), n)
 ifeq ($(ENABLE_FIPS), y)
-    $(error Cannot set fips config ENABLE_FIPS=y when ENABLE_FIPS_FEATURE=n)
+    $(error Cannot set fips config ENABLE_FIPS=y when INCLUDE_FIPS=n)
 endif
 endif
 
@@ -210,7 +210,7 @@ $(shell \
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
 	MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \
 	CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \
-	ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) \
+	INCLUDE_FIPS=$(INCLUDE_FIPS) \
 	DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) \
 	DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
 	GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \
@@ -561,7 +561,7 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
                            ENABLE_ASAN=$(ENABLE_ASAN) \
                            SONIC_INCLUDE_BOOTCHART=$(INCLUDE_BOOTCHART) \
                            SONIC_ENABLE_BOOTCHART=$(ENABLE_BOOTCHART) \
-                           ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) \
+                           INCLUDE_FIPS=$(INCLUDE_FIPS) \
                            ENABLE_FIPS=$(ENABLE_FIPS) \
                            SONIC_SLAVE_DOCKER_DRIVER=$(SONIC_SLAVE_DOCKER_DRIVER) \
                            MIRROR_URLS=$(MIRROR_URLS) \

README.md

@@ -7,9 +7,34 @@
 [![Innovium](https://dev.azure.com/mssonic/build/_apis/build/status/innovium/Azure.sonic-buildimage.official.innovium?branchName=master&label=Innovium)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=148&branchName=master)
 [![Mellanox](https://dev.azure.com/mssonic/build/_apis/build/status/mellanox/Azure.sonic-buildimage.official.mellanox?branchName=master&label=Mellanox)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=139&branchName=master)
 [![Marvell(armhf)](https://dev.azure.com/mssonic/build/_apis/build/status/marvell/Azure.sonic-buildimage.official.marvell-armhf?branchName=master&label=Marvell-armhf)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=141&branchName=master)
+[![Marvell(arm64)](https://dev.azure.com/mssonic/build/_apis/build/status/marvell/Azure.sonic-buildimage.official.marvell-arm64?branchName=master&label=Marvell-arm64)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=999&branchName=master)
 [![Nephos](https://dev.azure.com/mssonic/build/_apis/build/status/nephos/Azure.sonic-buildimage.official.nephos?branchName=master&label=Nephos)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=149&branchName=master)
 [![VS](https://dev.azure.com/mssonic/build/_apis/build/status/vs/Azure.sonic-buildimage.official.vs?branchName=master&label=VS)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=142&branchName=master)
 
+*202305 builds*:
+
+[![Barefoot](https://dev.azure.com/mssonic/build/_apis/build/status/barefoot/Azure.sonic-buildimage.official.barefoot?branchName=202205&label=Barefoot)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=146&branchName=202305)
+[![Broadcom](https://dev.azure.com/mssonic/build/_apis/build/status/broadcom/Azure.sonic-buildimage.official.broadcom?branchName=202305&label=Broadcom)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=138&branchName=202305)
+[![Centec](https://dev.azure.com/mssonic/build/_apis/build/status/centec/Azure.sonic-buildimage.official.centec?branchName=202305&label=Centec)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=143&branchName=202305)
+[![Centec(arm64)](https://dev.azure.com/mssonic/build/_apis/build/status/centec/Azure.sonic-buildimage.official.centec-arm64?branchName=202305&label=Centec-arm64)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=140&branchName=202305)
+[![Innovium](https://dev.azure.com/mssonic/build/_apis/build/status/innovium/Azure.sonic-buildimage.official.innovium?branchName=202305&label=Innovium)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=148&branchName=202305)
+[![Mellanox](https://dev.azure.com/mssonic/build/_apis/build/status/mellanox/Azure.sonic-buildimage.official.mellanox?branchName=202305&label=Mellanox)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=139&branchName=202305)
+[![Marvell(armhf)](https://dev.azure.com/mssonic/build/_apis/build/status/marvell/Azure.sonic-buildimage.official.marvell-armhf?branchName=202305&label=Marvell-armhf)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=141&branchName=202305)
+[![Nephos](https://dev.azure.com/mssonic/build/_apis/build/status/nephos/Azure.sonic-buildimage.official.nephos?branchName=202305&label=Nephos)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=149&branchName=202305)
+[![VS](https://dev.azure.com/mssonic/build/_apis/build/status/vs/Azure.sonic-buildimage.official.vs?branchName=202305&label=VS)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=142&branchName=202305)
+
+*202211 builds*:
+
+[![Barefoot](https://dev.azure.com/mssonic/build/_apis/build/status/barefoot/Azure.sonic-buildimage.official.barefoot?branchName=202205&label=Barefoot)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=146&branchName=202211)
+[![Broadcom](https://dev.azure.com/mssonic/build/_apis/build/status/broadcom/Azure.sonic-buildimage.official.broadcom?branchName=202211&label=Broadcom)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=138&branchName=202211)
+[![Centec](https://dev.azure.com/mssonic/build/_apis/build/status/centec/Azure.sonic-buildimage.official.centec?branchName=202211&label=Centec)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=143&branchName=202211)
+[![Centec(arm64)](https://dev.azure.com/mssonic/build/_apis/build/status/centec/Azure.sonic-buildimage.official.centec-arm64?branchName=202211&label=Centec-arm64)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=140&branchName=202211)
+[![Innovium](https://dev.azure.com/mssonic/build/_apis/build/status/innovium/Azure.sonic-buildimage.official.innovium?branchName=202211&label=Innovium)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=148&branchName=202211)
+[![Mellanox](https://dev.azure.com/mssonic/build/_apis/build/status/mellanox/Azure.sonic-buildimage.official.mellanox?branchName=202211&label=Mellanox)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=139&branchName=202211)
+[![Marvell(armhf)](https://dev.azure.com/mssonic/build/_apis/build/status/marvell/Azure.sonic-buildimage.official.marvell-armhf?branchName=202211&label=Marvell-armhf)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=141&branchName=202211)
+[![Nephos](https://dev.azure.com/mssonic/build/_apis/build/status/nephos/Azure.sonic-buildimage.official.nephos?branchName=202211&label=Nephos)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=149&branchName=202211)
+[![VS](https://dev.azure.com/mssonic/build/_apis/build/status/vs/Azure.sonic-buildimage.official.vs?branchName=202211&label=VS)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=142&branchName=202211)
+
 *202205 builds*:
 
 [![Barefoot](https://dev.azure.com/mssonic/build/_apis/build/status/barefoot/Azure.sonic-buildimage.official.barefoot?branchName=202205&label=Barefoot)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=146&branchName=202205)

azure-pipelines.yml

@@ -108,27 +108,26 @@ stages:
         sudo rm -rf ../*.deb
       displayName: "Cleanup"
 
-    - task: DownloadPipelineArtifact@2
-      inputs:
-        source: specific
-        project: build
-        pipeline: 9
-        artifact: sonic-swss-common.amd64.ubuntu20_04
-        runVersion: 'latestFromBranch'
-        runBranch: 'refs/heads/master'
-      displayName: "Download sonic swss common deb packages"
-
     - task: DownloadPipelineArtifact@2
       inputs:
         artifact: sonic-buildimage.vs
       displayName: "Download sonic-buildimage.vs artifact"
 
     - script: |
-        set -x
         sudo apt-get update
-        sudo apt-get install libyang0.16 -y
-        sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb
-        sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
+        sudo apt-get install -y make libtool m4 autoconf dh-exec debhelper cmake pkg-config \
+                         libhiredis-dev libnl-3-dev libnl-genl-3-dev libnl-route-3-dev libnl-nf-3-dev swig3.0 \
+                         libpython2.7-dev libboost-dev libboost-serialization-dev uuid-dev libzmq5 libzmq3-dev  python3-pip \
+                         cmake libgtest-dev libgmock-dev libyang-dev
+        sudo pip3 install pytest
+        cd src/sonic-swss-common
+        ./autogen.sh
+        dpkg-buildpackage -rfakeroot -us -uc -b -j$(nproc)
+        sudo dpkg -i --force-confask,confnew ../libswsscommon_*.deb
+        sudo dpkg -i ../python3-swsscommon_*.deb
+      displayName: "Compile sonic swss common"
+
+    - script: |
         sudo docker load -i ../target/docker-sonic-vs.gz
         docker tag docker-sonic-vs:latest docker-sonic-vs:$(Build.BuildNumber)
         username=$(id -un)
@@ -158,6 +157,7 @@ stages:
         TOPOLOGY: t0
         MIN_WORKER: $(T0_INSTANCE_NUM)
         MAX_WORKER: $(T0_INSTANCE_NUM)
+        MGMT_BRANCH: master
 
   - job: t0_2vlans_elastictest
     pool: ubuntu-20.04
@@ -171,6 +171,7 @@ stages:
         TEST_SET: t0-2vlans
         MIN_WORKER: $(T0_2VLANS_INSTANCE_NUM)
         MAX_WORKER: $(T0_2VLANS_INSTANCE_NUM)
+        MGMT_BRANCH: master
         DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a"
 
   - job: t1_lag_elastictest
@@ -184,6 +185,7 @@ stages:
         TOPOLOGY: t1-lag
         MIN_WORKER: $(T1_LAG_INSTANCE_NUM)
         MAX_WORKER: $(T1_LAG_INSTANCE_NUM)
+        MGMT_BRANCH: master
 
   - job: multi_asic_elastictest
     displayName: "kvmtest-multi-asic-t1-lag by Elastictest"
@@ -198,6 +200,7 @@ stages:
           MIN_WORKER: $(MULTI_ASIC_INSTANCE_NUM)
           MAX_WORKER: $(MULTI_ASIC_INSTANCE_NUM)
           NUM_ASIC: 4
+          MGMT_BRANCH: master
 
   - job: dualtor_elastictest
     pool: ubuntu-20.04
@@ -210,6 +213,7 @@ stages:
           TOPOLOGY: dualtor
           MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
           MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
+          MGMT_BRANCH: master
           COMMON_EXTRA_PARAMS: "--disable_loganalyzer "
 
   - job: sonic_t0_elastictest
@@ -224,19 +228,19 @@ stages:
           MIN_WORKER: $(T0_SONIC_INSTANCE_NUM)
           MAX_WORKER: $(T0_SONIC_INSTANCE_NUM)
           TEST_SET: t0-sonic
-          COMMON_EXTRA_PARAMS: "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"
+          MGMT_BRANCH: master
+          COMMON_EXTRA_PARAMS: "--neighbor_type=sonic "
           VM_TYPE: vsonic
-          SPECIFIED_PARAMS: '{\"test_pretest.py\":[\"--completeness_level=confident\",\"--allow_recover\"],\"test_posttest.py\":[\"--completeness_level=confident\",\"--allow_recover\"]}'
 
-  - job: wan_elastictest
-    displayName: "kvmtest-wan by Elastictest"
-    pool: ubuntu-20.04
-    timeoutInMinutes: 240
-    continueOnError: false
-    steps:
-      - template: .azure-pipelines/run-test-scheduler-template.yml
-        parameters:
-          TOPOLOGY: wan-pub
-          MIN_WORKER: $(WAN_INSTANCE_NUM)
-          MAX_WORKER: $(WAN_INSTANCE_NUM)
-          COMMON_EXTRA_PARAMS: "--skip_sanity "
+#  - job: wan_elastictest
+#    displayName: "kvmtest-wan by Elastictest"
+#    pool: ubuntu-20.04
+#    timeoutInMinutes: 240
+#    continueOnError: false
+#    steps:
+#      - template: .azure-pipelines/run-test-scheduler-template.yml
+#        parameters:
+#          TOPOLOGY: wan-pub
+#          MIN_WORKER: $(WAN_INSTANCE_NUM)
+#          MAX_WORKER: $(WAN_INSTANCE_NUM)
+#          COMMON_EXTRA_PARAMS: "--skip_sanity "