Reland 'Make changes to building and packaging sairedis (#1116)' (#1194)

* Reland 'Make changes to building and packaging sairedis (#1116)'

* Make changes to building and packaging sairedis

This commit includes the following changes:
1. Use Debian build profiles instead of custom build targets to build
different configurations of sairedis. Build profiles were designed for
this purpose. This also makes the debian/rules file a bit cleaner.
2. Rely on the debug packages being automatically created, instead of us
explicitly specifying it in debian/control.
3. Add actual support for excluding Python 2 binding during build.
4. Make sure the compile flags used for building Python 2 and Python 3
are actually correct.

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

* Update pipeline file

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

* Fix argument for profile

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

* Exclude libsai package from the list of dependencies

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

* Allow multiple syslog artifacts during tests

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

* Fix debug poackage name in dockerfile

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

* Fix armhf build due to python binary not existing

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

---------

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

.azure-pipelines/build-template.yml

@@ -159,7 +159,7 @@ jobs:
       if [ '${{ parameters.asan }}' == True ]; then
          extraflags='--enable-asan'
       fi
-      DEB_BUILD_OPTIONS=nocheck fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS=$extraflags DEB_BUILD_PROFILES=nopython2 CFLAGS="" CXXFLAGS="" binary-syncd-vs
+      DEB_BUILD_OPTIONS=nocheck DEB_CONFIGURE_EXTRA_FLAGS=$extraflags dpkg-buildpackage -us -uc -b -Psyncd,vs,nopython2 -j$(nproc)
       mv ../*.deb .
     displayName: "Compile sonic sairedis with coverage enabled"
   - script: |
@@ -227,6 +227,6 @@ jobs:
         contents: 'syslog-all.tgz'
         targetFolder: $(Build.ArtifactStagingDirectory)
   - publish: $(Build.ArtifactStagingDirectory)/
-    artifact: ${{ parameters.syslog_artifact_name }}
+    artifact: ${{ parameters.syslog_artifact_name }}@$(System.JobAttempt)
     displayName: "Publish syslog artifacts"
     condition: always()

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -14,4 +14,4 @@ RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb \
             /debs/syncd-vs_1.0.0_amd64.deb \
             /debs/swss_1.0.0_amd64.deb
 
-RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/syncd-vs-dbg_1.0.0_amd64.deb ; fi
+RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/syncd-vs-dbgsym_1.0.0_amd64.deb ; fi

configure.ac

@@ -10,8 +10,8 @@ AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_LIBTOOL
 AC_HEADER_STDC
-AM_PATH_PYTHON
 AM_PATH_PYTHON3
+AX_PYTHON3_DEVEL
 AX_CODE_COVERAGE
 AX_ADD_AM_MACRO_STATIC([])
 
@@ -101,6 +101,14 @@ AC_ARG_ENABLE(asan,
 	*) AC_MSG_ERROR(bad value ${enableval} for --enable-asan) ;;
 esac],[asan_enabled=false])
 
+AC_ARG_ENABLE(python2,
+[  --enable-python2     Generate Python 2 bindings],
+[case "${enableval}" in
+	yes) python2=true ;;
+	no)  python2=false ;;
+	*) AC_MSG_ERROR(bad value ${enableval} for --enable-python2) ;;
+esac],[python2=true])
+
 if test "x$asan_enabled" = "xtrue"; then
     CFLAGS_ASAN+=" -fsanitize=address"
     CFLAGS_ASAN+=" -DASAN_ENABLED"
@@ -111,13 +119,31 @@ if test "x$asan_enabled" = "xtrue"; then
     AC_SUBST(LDFLAGS_ASAN)
 fi
 
+if test "x$python2" = "xtrue"; then
+	m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
+					  [python2 python2.7 python2.6 python2.5 python2.4 python2.3 python2.2 python2.1 python2.0])
+	AM_PATH_PYTHON
+	AX_PYTHON_DEVEL([< '3'])
+	PYTHON2_CPPFLAGS=$PYTHON_CPPFLAGS
+	PYTHON2_LIBS=$PYTHON_LIBS
+	AC_SUBST(PYTHON2_CPPFLAGS)
+	AC_SUBST(PYTHON2_LIBS)
+else
+	# The py-compile script either uses $PYTHON (which won't be set if we're not building python 2 bindings),
+	# or assumes a default of `python` (which may or may not exist as a symlink). Therefore, just assign
+	# PYTHON3 to PYTHON to make sure it always points to a valid binary.
+	PYTHON=$PYTHON3
+fi
+
 AM_CONDITIONAL(ASAN_ENABLED, test x$asan_enabled = xtrue)
+AM_CONDITIONAL(PYTHON2, test x$python2 = xtrue)
 
 AC_PATH_PROGS(SWIG, [swig3.0 swig])
 
 CXXFLAGS_COMMON=""
 CXXFLAGS_COMMON+=" -ansi"
 CXXFLAGS_COMMON+=" -fPIC"
+CXXFLAGS_COMMON+=" -pipe"
 CXXFLAGS_COMMON+=" -std=c++14"
 CXXFLAGS_COMMON+=" -Wall"
 CXXFLAGS_COMMON+=" -Wcast-align"

debian/compat

@@ -1 +1 @@
-10
+12

debian/control

@@ -2,24 +2,33 @@ Source: sonic
 Maintainer: Kamil Cudnik <kcudnik@microsoft.com>
 Section: net
 Priority: optional
-Build-Depends: debhelper (>=9), autotools-dev, libzmq5-dev
+Build-Depends: debhelper (>= 12), autotools-dev, libzmq5-dev
 Standards-Version: 1.0.0
 
 Package: syncd
 Architecture: any
+Build-Profiles: <syncd !rpc !vs>
+Depends: ${misc:Pre-Depends}
+Recommends: ${shlibs:Depends}
 Conflicts: syncd-rpc, syncd-vs
 Description: This package contains sync daemon for SONiC project.
   This sync daemon syncs the ASIC_DB in Redis database and the real ASIC via SAI.
 
 Package: syncd-rpc
 Architecture: any
+Build-Profiles: <syncd rpc !vs>
+Depends: ${misc:Pre-Depends}
+Recommends: ${shlibs:Depends}
 Conflicts: syncd, syncd-vs
 Description: This package contains sync daemon with rpc for SONiC project.
   This sync daemon syncs the ASIC_DB in Redis database and the real ASIC via SAI.
   This daemon contains saithrift rpc library for remote control of an ASIC.
 
 Package: syncd-vs
 Architecture: any
+Build-Profiles: <syncd !rpc vs>
+Depends: ${misc:Pre-Depends}
+Recommends: ${shlibs:Depends}
 Conflicts: syncd-rpc, syncd
 Description: This package contains sync daemon for SONiC project linked with virtual switch.
   This sync daemon syncs the ASIC_DB in Redis database and the real ASIC via SAI.
@@ -38,6 +47,7 @@ Description: This package contains development files for SAI-Redis.
 
 Package: libsaivs
 Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Pre-Depends}
 Section: libs
 Description: This package contains SAI-VirtualSwitch implementation for SONiC project.
 
@@ -49,6 +59,7 @@ Description: This package contains development files for SAI-VirtualSwitch.
 
 Package: libsaimetadata
 Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Pre-Depends}
 Section: libs
 Description: This package contains SAI-Metadata implementation for SONiC project.
 
@@ -58,63 +69,6 @@ Depends: libsaimetadata (= ${binary:Version})
 Section: libdevel
 Description: This package contains development files for SAI-Metadata.
 
-Package: syncd-dbg
-Architecture: any
-Section: debug
-Priority: extra
-Conflicts: syncd-rpc-dbg, syncd-vs
-Depends:
-    syncd (= ${binary:Version}),
-    ${misc:Depends}
-Description: debugging symbols for syncd
-
-Package: syncd-rpc-dbg
-Architecture: any
-Section: debug
-Priority: extra
-Conflicts: syncd-dbg, syncd-vs
-Depends:
-    syncd-rpc (= ${binary:Version}),
-    ${misc:Depends}
-Description: debugging symbols for syncd-rpc
-
-Package: syncd-vs-dbg
-Architecture: any
-Section: debug
-Priority: extra
-Conflicts: syncd-dbg, syncd-rpc-dbg
-Depends:
-    syncd-vs (= ${binary:Version}),
-    ${misc:Depends}
-Description: debugging symbols for syncd-vs
-
-Package: libsairedis-dbg
-Architecture: any
-Section: debug
-Priority: extra
-Depends:
-    libsairedis (= ${binary:Version}),
-    ${misc:Depends}
-Description: debugging symbols for libsairedis
-
-Package: libsaivs-dbg
-Architecture: any
-Section: debug
-Priority: extra
-Depends:
-    libsaivs (= ${binary:Version}),
-    ${misc:Depends}
-Description: debugging symbols for libsaivs
-
-Package: libsaimetadata-dbg
-Architecture: any
-Section: debug
-Priority: extra
-Depends:
-    libsaimetadata (= ${binary:Version}),
-    ${misc:Depends}
-Description: debugging symbols for libsaimetadata
-
 Package: python-pysairedis
 Architecture: any
 Build-Profiles: <!nopython2>

debian/libsaimetadata-dev.install

@@ -1,3 +1,5 @@
 meta/sai*.h usr/include/sai
 meta/Sai*.h usr/include/sai
 SAI/meta/sai*.h usr/include/sai
+usr/lib/*/libsaimetadata.so
+usr/lib/*/libsaimeta.so

debian/libsairedis-dev.install

@@ -1 +1,2 @@
 lib/sairedis.h usr/include/sai
+usr/lib/*/libsairedis.so

debian/rules

@@ -3,14 +3,6 @@
 # output every command that modifies files on the build system.
 #export DH_VERBOSE = 1
 
-.ONESHELL:
-SHELL = /bin/bash
-.SHELLFLAGS += -ex
-
-# see EXAMPLES in dpkg-buildflags(1) and read /usr/share/dpkg/*
-DPKG_EXPORT_BUILDFLAGS = 1
-include /usr/share/dpkg/default.mk
-
 ifneq (${SWSS_COMMON_INC},)
 	SWSS_COMMON_CONFIG = "--with-swss-common-inc=${SWSS_COMMON_INC}"
 endif
@@ -21,7 +13,7 @@ ifneq (${SWSS_COMMON_LIB},)
 endif
 
 # see FEATURE AREAS in dpkg-buildflags(1)
-#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 
 # see ENVIRONMENT in dpkg-buildflags(1)
 # package maintainers to append CFLAGS
@@ -31,78 +23,53 @@ endif
 
 DOPACKAGES = $(shell dh_listpackages)
 
+configure_opts = --disable-static
+ifeq ($(ENABLE_ASAN), y)
+	configure_opts += --enable-asan
+endif
+
 # For Debian jessie, stretch, and buster, and Ubuntu bionic and focal, build
 # Python 2 bindings. This is controlled by the build profile being used.
-ifneq (,$(filter python-pysairedis,$(DOPACKAGES)))
+ifeq (,$(filter nopython2,$(DEB_BUILD_PROFILES)))
 configure_opts += --enable-python2
 else
 configure_opts += --disable-python2
 endif
 
-clean:
-	dh $@ --with autotools-dev
-
-build:
-	echo build stage is skipped. Please use binary to generate debian packages
-
-binary: binary-syncd-rpc binary-syncd
-
-binary-sairedis:
-	echo '--enable-syncd=no' > /tmp/syncd-build
-	ENABLESYNCD=no dh clean  --with autotools-dev
-	dh build  -N syncd -N syncd-dbg -N syncd-rpc -N syncd-rpc-dbg -N syncd-vs -N syncd-vs-dbg --with autotools-dev
-	dh binary -N syncd -N syncd-dbg -N syncd-rpc -N syncd-rpc-dbg -N syncd-vs -N syncd-vs-dbg --with autotools-dev
-
-binary-syncd:
-	echo > /tmp/syncd-build
-	dh clean  --with autotools-dev
-	dh build  -N syncd-rpc -N syncd-rpc-dbg -N syncd-vs -N syncd-vs-dbg --with autotools-dev
-	dh binary -N syncd-rpc -N syncd-rpc-dbg -N syncd-vs -N syncd-vs-dbg --with autotools-dev
-
-binary-syncd-rpc: | binary-syncd
-	echo '--enable-rpcserver=yes' > /tmp/syncd-build
-	dh clean  --with autotools-dev
-	dh build  -N syncd -N syncd-dbg -N syncd-vs -N syncd-vs-dbg --with autotools-dev
-	dh binary -N syncd -N syncd-dbg -N syncd-vs -N syncd-vs-dbg --with autotools-dev
+ifneq ($(filter syncd,$(DEB_BUILD_PROFILES)),)
+ifneq ($(filter rpc,$(DEB_BUILD_PROFILES)),)
+configure_opts += --enable-rpcserver
+endif
+ifneq ($(filter vs,$(DEB_BUILD_PROFILES)),)
+configure_opts += --with-sai=vs
+endif
+else
+configure_opts += --disable-syncd
+endif
 
-binary-syncd-vs:
-	echo '--with-sai=vs' > /tmp/syncd-build
-	dh clean  --with autotools-dev
-	dh build  -N syncd -N syncd-dbg -N syncd-rpc -N syncd-rpc-dbg --with autotools-dev
-	dh binary -N syncd -N syncd-dbg -N syncd-rpc -N syncd-rpc-dbg --with autotools-dev
+%:
+	dh $@
 
 # dh_make generated override targets
 # This is example for Cmake (See https://bugs.debian.org/641051 )
 #override_dh_auto_configure:
 #	dh_auto_configure -- \
 #	-DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH)
 
-configure_opts =
-ifeq ($(ENABLE_ASAN), y)
-	configure_opts += --enable-asan
-endif
-
 override_dh_auto_configure:
-	./autogen.sh
-	dh_auto_configure -- $(DEB_CONFIGURE_EXTRA_FLAGS) $(shell cat /tmp/syncd-build) ${SWSS_COMMON_CONFIG} $(configure_opts)
+	dh_auto_configure -- ${SWSS_COMMON_CONFIG} $(configure_opts)
 
 override_dh_install:
 	dh_install
 	# Note: escape $ with an extra $ symbol
-	if egrep -q '(^| )--enable-rpcserver=yes( |$$)' /tmp/syncd-build && [ -f debian/syncd-rpc/usr/bin/syncd_init_common.sh ] ; then
+ifneq ($(filter rpc,$(DEB_BUILD_PROFILES)),)
+	if [ -f debian/syncd-rpc/usr/bin/syncd_init_common.sh ] ; then
 		sed -i 's|ENABLE_SAITHRIFT=0|ENABLE_SAITHRIFT=1 # Add a comment to fix https://github.com/Azure/sonic-buildimage/issues/2694 |' debian/syncd-rpc/usr/bin/syncd_init_common.sh
 	fi
+endif
 
 override_dh_installinit:
 	dh_installinit --init-script=syncd
 
 override_dh_shlibdeps:
-	$(LD_LIBRARY_PATH_CONFIG) dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info
-
-override_dh_strip:
-	dh_strip -psyncd-rpc --dbg-package=syncd-rpc-dbg
-	dh_strip -psyncd --dbg-package=syncd-dbg
-	dh_strip -psyncd-vs --dbg-package=syncd-vs-dbg
-	dh_strip -plibsairedis --dbg-package=libsairedis-dbg
-	dh_strip -plibsaivs --dbg-package=libsaivs-dbg
-	dh_strip -plibsaimetadata --dbg-package=libsaimetadata-dbg
+	$(LD_LIBRARY_PATH_CONFIG) dh_shlibdeps -- --ignore-missing-info -xlibsai