[ci]: Support Azure pipelines on 201911 branch (#1806)

Merge azure pipeline yaml files from master
Fix branch reference issue
Disable arm64/armhf build

.artifactignore

@@ -0,0 +1,2 @@
+**/*
+!*.deb

.azure-pipelines/build-docker-sonic-vs-template.yml

@@ -0,0 +1,85 @@
+parameters:
+- name: arch
+  type: string
+  values:
+  - amd64
+  - armhf
+  - arm64
+  default: amd64
+
+- name: timeout
+  type: number
+  default: 60
+
+- name: swss_artifact_name
+  type: string
+
+- name: sairedis_artifact_name
+  type: string
+
+- name: swss_common_artifact_name
+  type: string
+
+- name: artifact_name
+  type: string
+
+jobs:
+- job:
+  displayName: ${{ parameters.arch }}
+  timeoutInMinutes: ${{ parameters.timeout }}
+
+  pool:
+    vmImage: 'ubuntu-20.04'
+
+  steps:
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: 9
+      artifact: ${{ parameters.swss_common_artifact_name }}
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/201911'
+    displayName: "Download sonic swss common deb packages"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: 12
+      artifact: ${{ parameters.sairedis_artifact_name }}
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/201911'
+    displayName: "Download sonic sairedis deb packages"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifact: ${{ parameters.swss_artifact_name }}
+    displayName: "Download sonic swss artifact"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: 142
+      artifact: sonic-buildimage.vs
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/201911'
+    displayName: "Download sonic buildimage"
+  - script: |
+      echo $(Build.DefinitionName).$(Build.BuildNumber)
+
+      docker load < ../target/docker-sonic-vs.gz
+
+      mkdir -p .azure-pipelines/docker-sonic-vs/debs
+
+      cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs
+
+      pushd .azure-pipelines
+
+      docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs
+
+      popd
+
+      docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
+
+  - publish: $(Build.ArtifactStagingDirectory)/
+    artifact: ${{ parameters.artifact_name }}
+    displayName: "Archive sonic docker vs image"

.azure-pipelines/build-template.yml

@@ -0,0 +1,134 @@
+parameters:
+- name: arch
+  type: string
+  values:
+  - amd64
+  - armhf
+  - arm64
+  default: amd64
+
+- name: pool
+  type: string
+  values:
+  - sonicbld
+  - default
+  default: default
+
+- name: timeout
+  type: number
+  default: 60
+
+- name: sonic_slave
+  type: string
+
+- name: sairedis_artifact_name
+  type: string
+
+- name: swss_common_artifact_name
+  type: string
+
+- name: artifact_name
+  type: string
+
+- name: archive_pytests
+  type: boolean
+  default: false
+
+jobs:
+- job:
+  displayName: ${{ parameters.arch }}
+  timeoutInMinutes: ${{ parameters.timeout }}
+
+  pool:
+    ${{ if ne(parameters.pool, 'default') }}:
+      name: ${{ parameters.pool }}
+    ${{ if eq(parameters.pool, 'default') }}:
+      vmImage: 'ubuntu-20.04'
+
+  container:
+    image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest
+
+  steps:
+  - script: |
+      sudo apt-get install -y libhiredis0.14 libhiredis-dev
+      sudo apt-get install -y libzmq5 libzmq3-dev
+      sudo apt-get install -qq -y \
+          libhiredis-dev \
+          libnl-3-dev \
+          libnl-genl-3-dev \
+          libnl-route-3-dev \
+          libnl-nf-3-dev \
+          swig3.0
+      sudo apt-get install -y libdbus-1-3
+      sudo apt-get install -y libteam-dev \
+          libteam5 \
+          libteamdctl0
+    displayName: "Install dependencies"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: 9
+      artifact: ${{ parameters.swss_common_artifact_name }}
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/201911'
+    displayName: "Download sonic swss common deb packages"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: 12
+      artifact: ${{ parameters.sairedis_artifact_name }}
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/201911'
+    displayName: "Download sonic sairedis deb packages"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: 142
+      artifact: sonic-buildimage.vs
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/201911'
+      patterns: target/debs/stretch/*.deb
+    displayName: "Download sonic image deb packages"
+  - script: |
+      pushd $(Pipeline.Workspace)/target/debs/stretch
+      cp libhiredis0.14_*.deb \
+         libhiredis-dev_*.deb \
+         libnl-3-200_*.deb \
+         libnl-3-dev_*.deb \
+         libnl-genl-3-200_*.deb \
+         libnl-genl-3-dev_*.deb \
+         libnl-route-3-200_*.deb \
+         libnl-route-3-dev_*.deb \
+         libnl-nf-3-200_*.deb \
+         libnl-nf-3-dev_*.deb \
+         libnl-cli-3-200_*.deb \
+         swig3.0_*.deb \
+         $(Pipeline.Workspace)/
+         cp libteam-dev_1.28-1_amd64.deb libteam5_1.28-1_amd64.deb libteamdctl0_1.28-1_amd64.deb $(Pipeline.Workspace)/
+      popd
+      ls -l $(Pipeline.Workspace)/*.deb
+    displayName: "Copy deb packages"
+  - script: |
+      sudo apt-get install -y libzmq5 libzmq3-dev
+      sudo dpkg -i --force-confask,confnew *.deb || sudo apt-get install -f -y
+    workingDirectory: $(Pipeline.Workspace)
+    displayName: "Install dependencies"
+  - checkout: self
+    submodules: true
+  - script: |
+      set -x
+      tar czf pytest.tgz tests
+      cp -r pytest.tgz $(Build.ArtifactStagingDirectory)/
+      ./autogen.sh
+      dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
+    displayName: "Compile sonic swss"
+  - publish: $(System.DefaultWorkingDirectory)/
+    artifact: ${{ parameters.artifact_name }}
+    displayName: "Archive swss debian packages"
+  - publish: $(Build.ArtifactStagingDirectory)/
+    artifact: sonic-swss-pytests
+    displayName: "Archive swss pytests"
+    condition: eq('${{ parameters.archive_pytests }}', true)

.azure-pipelines/build_and_install_module.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# build and install team/vrf driver
+#
+
+set -e
+
+source /etc/os-release
+
+function build_and_install_kmodule()
+{
+    if sudo modprobe team 2>/dev/null && sudo modprobe vrf 2>/dev/null && sudo modprobe macsec 2>/dev/null; then
+        echo "The module team, vrf and macsec exist."
+        return
+    fi
+
+    [ -z "$WORKDIR" ] && WORKDIR=$(mktemp -d)
+    cd $WORKDIR
+
+    KERNEL_RELEASE=$(uname -r)
+    KERNEL_MAINVERSION=$(echo $KERNEL_RELEASE | cut -d- -f1)
+    EXTRAVERSION=$(echo $KERNEL_RELEASE | cut -d- -f2)
+    LOCALVERSION=$(echo $KERNEL_RELEASE | cut -d- -f3)
+    VERSION=$(echo $KERNEL_MAINVERSION | cut -d. -f1)
+    PATCHLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f2)
+    SUBLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f3)
+
+    # Install the required debian packages to build the kernel modules
+    apt-get install -y build-essential linux-headers-${KERNEL_RELEASE} autoconf pkg-config fakeroot
+    apt-get install -y flex bison libssl-dev libelf-dev
+    apt-get install -y libnl-route-3-200 libnl-route-3-dev libnl-cli-3-200 libnl-cli-3-dev libnl-3-dev
+
+    # Add the apt source mirrors and download the linux image source code
+    cp /etc/apt/sources.list /etc/apt/sources.list.bk
+    sed -i "s/^# deb-src/deb-src/g" /etc/apt/sources.list
+    apt-get update
+    apt-get source linux-image-unsigned-$(uname -r) > source.log
+
+    # Recover the original apt sources list
+    cp /etc/apt/sources.list.bk /etc/apt/sources.list
+    apt-get update
+
+    # Build the Linux kernel module drivers/net/team and vrf
+    cd $(find . -maxdepth 1 -type d | grep -v "^.$")
+    make  allmodconfig
+    mv .config .config.bk
+    cp /boot/config-$(uname -r) .config
+    grep NET_TEAM .config.bk >> .config
+    echo CONFIG_NET_VRF=m >> .config
+    echo CONFIG_MACSEC=m >> .config
+    echo CONFIG_NET_VENDOR_MICROSOFT=y >> .config
+    echo CONFIG_MICROSOFT_MANA=m >> .config
+    make VERSION=$VERSION PATCHLEVEL=$PATCHLEVEL SUBLEVEL=$SUBLEVEL EXTRAVERSION=-${EXTRAVERSION} LOCALVERSION=-${LOCALVERSION} modules_prepare
+    make M=drivers/net/team
+    mv drivers/net/Makefile drivers/net/Makefile.bak
+    echo 'obj-$(CONFIG_NET_VRF) += vrf.o' > drivers/net/Makefile
+    echo 'obj-$(CONFIG_MACSEC) += macsec.o' >> drivers/net/Makefile
+    make M=drivers/net
+
+    # Install the module
+    TEAM_DIR=$(echo /lib/modules/$(uname -r)/kernel/net/team)
+    NET_DIR=$(echo /lib/modules/$(uname -r)/kernel/net)
+    if [ ! -e "$TEAM_DIR/team.ko" ]; then
+        mkdir -p $TEAM_DIR
+        cp drivers/net/team/*.ko $TEAM_DIR/
+        modinfo $TEAM_DIR/team.ko
+        depmod
+        modprobe team
+    fi
+    if [ ! -e "$NET_DIR/vrf.ko" ]; then
+        mkdir -p $NET_DIR
+        cp drivers/net/vrf.ko $NET_DIR/
+        modinfo $NET_DIR/vrf.ko
+        depmod
+        modprobe vrf
+    fi
+    if [ ! -e "$NET_DIR/macsec.ko" ]; then
+        mkdir -p $NET_DIR
+        cp drivers/net/macsec.ko $NET_DIR/
+        modinfo $NET_DIR/macsec.ko
+        depmod
+        modprobe macsec
+    fi
+
+    cd /tmp
+    rm -rf $WORKDIR
+}
+
+build_and_install_kmodule

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -0,0 +1,8 @@
+FROM docker-sonic-vs
+
+ARG docker_container_name
+
+ADD ["debs", "/debs"]
+
+RUN dpkg --purge swss
+RUN dpkg -i /debs/*.deb || apt-get install -f -y

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -0,0 +1,70 @@
+parameters:
+- name: timeout
+  type: number
+  default: 180
+
+- name: log_artifact_name
+  type: string
+
+jobs:
+- job:
+  displayName: vstest
+  timeoutInMinutes: ${{ parameters.timeout }}
+
+  pool:
+    vmImage: 'ubuntu-20.04'
+
+  steps:
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifact: docker-sonic-vs
+    displayName: "Download docker sonic vs image"
+
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: 9
+      artifact: sonic-swss-common.amd64.ubuntu20_04
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/201911'
+    displayName: "Download sonic swss common deb packages"
+
+  - script: |
+      set -x
+      sudo .azure-pipelines/build_and_install_module.sh
+
+      sudo apt-get install -y libhiredis0.14
+      sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
+      sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
+
+      # install packages for vs test
+      sudo apt-get install -y net-tools bridge-utils vlan
+      sudo apt-get install -y python3-pip
+      sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker==4.4.1 redis==3.3.4 flaky==3.7.0
+    displayName: "Install dependencies"
+
+  - script: |
+      set -x
+      sudo docker load -i ../docker-sonic-vs.gz
+      docker ps
+      ip netns list
+      pushd tests
+      sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+    displayName: "Run vs tests"
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/tr.xml'
+      testRunTitle: vstest
+    condition: always()
+
+  - script: |
+      cp -r tests/log $(Build.ArtifactStagingDirectory)/
+    displayName: "Collect logs"
+    condition: always()
+
+  - publish: $(Build.ArtifactStagingDirectory)/
+    artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)
+    displayName: "Publish logs"
+    condition: always()