Update the Azure Pipeline build to use Bookworm and Ubuntu 22.04

.azure-pipelines/build-sairedis-template.yml

@@ -66,7 +66,6 @@ jobs:
       set -ex
       sudo apt-get update
       sudo apt-get install -qq -y \
-        qtbase5-dev \
         libdbus-glib-1-dev \
         libpcsclite-dev \
         docbook-to-man \
@@ -90,7 +89,7 @@ jobs:
       sudo mkdir -m 755 /var/run/sswsyncd
 
       sudo apt-get install -y rsyslog
-      sudo service rsyslog start
+      sudo rsyslogd
 
     displayName: "Install dependencies"
   - task: DownloadPipelineArtifact@2
@@ -137,7 +136,8 @@ jobs:
     displayName: "Compile sonic sairedis"
   - script: |
       sudo cp azsyslog.conf /etc/rsyslog.conf
-      sudo service rsyslog restart
+      sudo killall rsyslogd
+      sudo rsyslogd
     displayName: "Update rsyslog.conf"
   - ${{ if eq(parameters.run_unit_test, true) }}:
     - script: |

.azure-pipelines/build-template.yml

@@ -98,16 +98,6 @@ jobs:
       mv ../*.deb .
     displayName: "Compile sonic swss common with coverage enabled"
   - ${{ if eq(parameters.run_unit_test, true) }}:
-    - script: |
-        set -ex
-        git clone https://github.com/gcovr/gcovr.git
-        cd gcovr/
-        git checkout 5.2
-        sudo pip3 install setuptools
-        sudo python3 setup.py install
-        cd ..
-        sudo rm -rf gcovr
-      displayName: "Install gcovr 5.2 (for --exclude-throw-branches support)"
     - script: |
         set -ex
         sudo pip install Pympler==0.8 pytest
@@ -142,9 +132,9 @@ jobs:
         set -ex
         # Install .NET CORE
         curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-        sudo apt-add-repository https://packages.microsoft.com/debian/11/prod
+        sudo apt-add-repository https://packages.microsoft.com/debian/12/prod
         sudo apt-get update
-        sudo apt-get install -y dotnet-sdk-6.0
+        sudo apt-get install -y dotnet-sdk-8.0
       displayName: "Install .NET CORE"
     - task: PublishCodeCoverageResults@1
       inputs:

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -1,11 +1,31 @@
 FROM docker-sonic-vs
 
 ARG docker_container_name
+ARG need_dbg
 
 COPY ["debs", "/debs"]
 
 # Remove existing packages first before installing the new/current packages. This is to overcome limitations with
 # Docker's diff detection mechanism, where only the file size and the modification timestamp (which will remain the
 # same, even though contents have changed) are checked between the previous and current layer.
-RUN dpkg --purge libswsscommon python3-swsscommon sonic-db-cli libsaimetadata libsairedis libsaivs syncd-vs swss sonic-eventd  libdashapi
-RUN dpkg -i /debs/libdashapi_1.0.0_amd64.deb /debs/libswsscommon_1.0.0_amd64.deb /debs/python3-swsscommon_1.0.0_amd64.deb /debs/sonic-db-cli_1.0.0_amd64.deb /debs/libsaimetadata_1.0.0_amd64.deb /debs/libsairedis_1.0.0_amd64.deb /debs/libsaivs_1.0.0_amd64.deb /debs/syncd-vs_1.0.0_amd64.deb /debs/swss_1.0.0_amd64.deb
+RUN dpkg --purge libswsscommon python3-swsscommon sonic-db-cli libsaimetadata libsairedis libsaivs syncd-vs swss sonic-eventd libdashapi
+
+RUN apt-get update
+
+RUN apt install -y /debs/libdashapi_1.0.0_amd64.deb \
+            /debs/libswsscommon_1.0.0_amd64.deb \
+            /debs/python3-swsscommon_1.0.0_amd64.deb \
+            /debs/sonic-db-cli_1.0.0_amd64.deb \
+            /debs/libsaimetadata_1.0.0_amd64.deb \
+            /debs/libsairedis_1.0.0_amd64.deb \
+            /debs/libsaivs_1.0.0_amd64.deb \
+            /debs/syncd-vs_1.0.0_amd64.deb \
+            /debs/swss_1.0.0_amd64.deb
+
+RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/libswsscommon-dbgsym_1.0.0_amd64.deb ; fi
+
+COPY ["start.sh", "/usr/bin/"]
+
+RUN pip3 install scapy==2.5.0
+
+RUN apt-get -y install software-properties-common libdatetime-perl libcapture-tiny-perl build-essential libcpanel-json-xs-perl git python3-protobuf

.azure-pipelines/docker-sonic-vs/start.sh

@@ -0,0 +1,187 @@
+#!/bin/bash -e
+
+# Generate configuration
+
+# NOTE: 'PLATFORM' and 'HWSKU' environment variables are set
+# in the Dockerfile so that they persist for the life of the container
+
+ln -sf /usr/share/sonic/device/$PLATFORM /usr/share/sonic/platform
+ln -sf /usr/share/sonic/device/$PLATFORM/$HWSKU /usr/share/sonic/hwsku
+
+SWITCH_TYPE=switch
+PLATFORM_CONF=platform.json
+if [[ $HWSKU == "DPU-2P" ]]; then
+    SWITCH_TYPE=dpu
+    PLATFORM_CONF=platform-dpu-2p.json
+fi
+
+pushd /usr/share/sonic/hwsku
+
+# filter available front panel ports in lanemap.ini
+[ -f lanemap.ini.orig ] || cp lanemap.ini lanemap.ini.orig
+for p in $(ip link show | grep -oE "eth[0-9]+" | grep -v eth0); do
+    grep ^$p: lanemap.ini.orig
+done > lanemap.ini
+
+# filter available sonic front panel ports in port_config.ini
+[ -f port_config.ini.orig ] || cp port_config.ini port_config.ini.orig
+grep ^# port_config.ini.orig > port_config.ini
+for lanes in $(awk -F ':' '{print $2}' lanemap.ini); do
+    grep -E "\s$lanes\s" port_config.ini.orig
+done >> port_config.ini
+
+popd
+
+[ -d /etc/sonic ] || mkdir -p /etc/sonic
+
+# Note: libswsscommon requires a dabase_config file in /var/run/redis/sonic-db/
+# Prepare this file before any dependent application, such as sonic-cfggen
+mkdir -p /var/run/redis/sonic-db
+cp /etc/default/sonic-db/database_config.json /var/run/redis/sonic-db/
+
+SYSTEM_MAC_ADDRESS=$(ip link show eth0 | grep ether | awk '{print $2}')
+sonic-cfggen -t /usr/share/sonic/templates/init_cfg.json.j2 -a "{\"system_mac\": \"$SYSTEM_MAC_ADDRESS\", \"switch_type\": \"$SWITCH_TYPE\"}" > /etc/sonic/init_cfg.json
+
+if [[ -f /usr/share/sonic/virtual_chassis/default_config.json ]]; then
+    sonic-cfggen -j /etc/sonic/init_cfg.json -j /usr/share/sonic/virtual_chassis/default_config.json --print-data > /tmp/init_cfg.json
+    mv /tmp/init_cfg.json /etc/sonic/init_cfg.json
+fi
+
+if [ -f /etc/sonic/config_db.json ]; then
+    sonic-cfggen -j /etc/sonic/init_cfg.json -j /etc/sonic/config_db.json --print-data > /tmp/config_db.json
+    mv /tmp/config_db.json /etc/sonic/config_db.json
+else
+    # generate and merge buffers configuration into config file
+    if [ -f /usr/share/sonic/hwsku/buffers.json.j2 ]; then
+        sonic-cfggen -k $HWSKU -p /usr/share/sonic/device/$PLATFORM/$PLATFORM_CONF -t /usr/share/sonic/hwsku/buffers.json.j2 > /tmp/buffers.json
+        buffers_cmd="-j /tmp/buffers.json"
+    fi
+    if [ -f /usr/share/sonic/hwsku/qos.json.j2 ]; then
+        sonic-cfggen -j /etc/sonic/init_cfg.json -t /usr/share/sonic/hwsku/qos.json.j2 > /tmp/qos.json
+        qos_cmd="-j /tmp/qos.json"
+    fi
+
+    sonic-cfggen -p /usr/share/sonic/device/$PLATFORM/$PLATFORM_CONF -k $HWSKU --print-data > /tmp/ports.json
+    # change admin_status from up to down; Test cases dependent
+    sed -i "s/up/down/g" /tmp/ports.json
+    sonic-cfggen -j /etc/sonic/init_cfg.json $buffers_cmd $qos_cmd -j /tmp/ports.json --print-data > /etc/sonic/config_db.json
+fi
+
+sonic-cfggen -t /usr/share/sonic/templates/copp_cfg.j2 > /etc/sonic/copp_cfg.json
+
+if [ "$HWSKU" == "Mellanox-SN2700" ]; then
+    cp /usr/share/sonic/hwsku/sai_mlnx.profile /usr/share/sonic/hwsku/sai.profile
+elif [ "$HWSKU" == "DPU-2P" ]; then
+    cp /usr/share/sonic/hwsku/sai_dpu_2p.profile /usr/share/sonic/hwsku/sai.profile
+fi
+
+mkdir -p /etc/swss/config.d/
+
+rm -f /var/run/rsyslogd.pid
+
+supervisorctl start rsyslogd
+
+supervisord_cfg="/etc/supervisor/conf.d/supervisord.conf"
+chassisdb_cfg_file="/usr/share/sonic/virtual_chassis/default_config.json"
+chassisdb_cfg_file_default="/etc/default/sonic-db/default_chassis_cfg.json"
+host_template="/usr/share/sonic/templates/hostname.j2"
+db_cfg_file="/var/run/redis/sonic-db/database_config.json"
+db_cfg_file_tmp="/var/run/redis/sonic-db/database_config.json.tmp"
+
+if [ -r "$chassisdb_cfg_file" ]; then
+   echo $(sonic-cfggen -j $chassisdb_cfg_file -t $host_template) >> /etc/hosts
+else
+   chassisdb_cfg_file="$chassisdb_cfg_file_default"
+   echo "10.8.1.200 redis_chassis.server" >> /etc/hosts
+fi
+
+supervisorctl start redis-server
+
+start_chassis_db=`sonic-cfggen -v DEVICE_METADATA.localhost.start_chassis_db -y $chassisdb_cfg_file`
+if [[ "$HOSTNAME" == *"supervisor"* ]] || [ "$start_chassis_db" == "1" ]; then
+   supervisorctl start redis-chassis
+fi
+
+conn_chassis_db=`sonic-cfggen -v DEVICE_METADATA.localhost.connect_to_chassis_db -y $chassisdb_cfg_file`
+if [ "$start_chassis_db" != "1" ] && [ "$conn_chassis_db" != "1" ]; then
+   cp $db_cfg_file $db_cfg_file_tmp
+   update_chassisdb_config -j $db_cfg_file_tmp -d
+   cp $db_cfg_file_tmp $db_cfg_file
+fi
+
+if [ "$conn_chassis_db" == "1" ]; then
+   if [ -f /usr/share/sonic/virtual_chassis/coreportindexmap.ini ]; then
+      cp /usr/share/sonic/virtual_chassis/coreportindexmap.ini /usr/share/sonic/hwsku/
+
+      pushd /usr/share/sonic/hwsku
+
+      # filter available front panel ports in coreportindexmap.ini
+      [ -f coreportindexmap.ini.orig ] || cp coreportindexmap.ini coreportindexmap.ini.orig
+      for p in $(ip link show | grep -oE "eth[0-9]+" | grep -v eth0); do
+          grep ^$p: coreportindexmap.ini.orig
+      done > coreportindexmap.ini
+
+      popd
+   fi
+fi
+
+/usr/bin/configdb-load.sh
+
+if [ "$HWSKU" = "brcm_gearbox_vs" ]; then
+    supervisorctl start gbsyncd
+    supervisorctl start gearsyncd
+fi
+
+supervisorctl start syncd
+
+supervisorctl start portsyncd
+
+supervisorctl start orchagent
+
+supervisorctl start coppmgrd
+
+supervisorctl start neighsyncd
+
+supervisorctl start fdbsyncd
+
+supervisorctl start teamsyncd
+
+supervisorctl start fpmsyncd
+
+supervisorctl start teammgrd
+
+supervisorctl start vrfmgrd
+
+supervisorctl start portmgrd
+
+supervisorctl start intfmgrd
+
+supervisorctl start vlanmgrd
+
+supervisorctl start zebra
+
+supervisorctl start mgmtd
+
+supervisorctl start staticd
+
+supervisorctl start buffermgrd
+
+supervisorctl start nbrmgrd
+
+supervisorctl start vxlanmgrd
+
+supervisorctl start sflowmgrd
+
+supervisorctl start natmgrd
+
+supervisorctl start natsyncd
+
+supervisorctl start tunnelmgrd
+
+supervisorctl start fabricmgrd
+
+# Start arp_update when VLAN exists
+VLAN=`sonic-cfggen -d -v 'VLAN.keys() | join(" ") if VLAN'`
+if [ "$VLAN" != "" ]; then
+    supervisorctl start arp_update
+fi

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -15,7 +15,7 @@ jobs:
   displayName: vstest
   timeoutInMinutes: ${{ parameters.timeout }}
 
-  pool: sonic-common
+  pool: sonictest
 
   steps:
   - checkout: self
@@ -52,11 +52,10 @@ jobs:
 
   - script: |
       set -ex
-      ls -l
       sudo sonic-swss-common/.azure-pipelines/build_and_install_module.sh
 
       sudo apt-get install -y libhiredis0.14 libyang0.16
-      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb 
       sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi*.deb
       sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
       sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
@@ -72,34 +71,17 @@ jobs:
       sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz
       docker ps
       ip netns list
+      sudo /sbin/ip link add Vrf1 type vrf table 1001 || { echo 'vrf command failed' ; exit 1; }
+      sudo /sbin/ip link del Vrf1 type vrf table 1001
       pushd sonic-swss/tests
 
-      # run pytests in sets of 20   
-      all_tests=$(ls test_*.py)
+      all_tests=$(ls test_*.py | xargs)
       all_tests="${all_tests} p4rt dash"
-      test_set=()
-      for test in ${all_tests}; do
-        test_set+=("${test}")
-        if [ ${#test_set[@]} -ge 20 ]; then
-          test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-          echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
-          container_count=$(docker ps -q -a | wc -l)
-          if [ ${container_count} -gt 0 ]; then
-            docker stop $(docker ps -q -a)
-            docker rm $(docker ps -q -a)
-          fi
-          test_set=()
-        fi
-      done
-      if [ ${#test_set[@]} -gt 0 ]; then
-        test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-        echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
-        container_count=$(docker ps -q -a | wc -l)
-        if [ ${container_count} -gt 0 ]; then
-          docker stop $(docker ps -q -a)
-          docker rm $(docker ps -q -a)
-        fi
-      fi
+
+      # Run the tests in parallel and retry
+      retry=3
+      IMAGE_NAME=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+      echo $all_tests | xargs -n 1 | xargs -P 8 -I TEST_MODULE sudo ./run-tests.sh "$IMAGE_NAME" "--force-recreate-dvs" "TEST_MODULE" 3
 
       rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"