From 6cca5cd349684521e6eea596ebe85f21849c94c4 Mon Sep 17 00:00:00 2001 From: lguohan Date: Mon, 27 Jul 2020 10:46:20 -0700 Subject: [PATCH 1/5] [build]: wait for conflicts package to be uninstalled (#5039) when parallel build is enabled, both docker-fpm-frr and docker-syncd-brcm is built at the same time, docker-fpm-frr requires swss which requires to install libsaivs-dev. docker-syncd-brcm requires syncd package which requires to install libsaibcm-dev. since libsaivs-dev and libsaibcm-dev install the sai header in the same location, these two packages cannot be installed at the same time. Therefore, we need to serialize the build between these two packages. Simply uninstall the conflict package is not enough to solve this issue. The correct solution is to have one package wait for another package to be uninstalled. For example, if syncd is built first, then it will install libsaibcm-dev. Meanwhile, if the swss build job starts and tries to install libsaivs-dev, it will first try to query if libsaibcm-dev is installed or not. if it is installed, then it will wait until libsaibcm-dev is uninstalled. After syncd job is finished, it will uninstall libsaibcm-dev and swss build job will be unblocked. To solve this issue, _UNINSTALLS is introduced to uninstall a package that is no longer needed and to allow blocked job to continue. Signed-off-by: Guohan Lu --- platform/barefoot/bfn-sai.mk | 3 ++- platform/barefoot/rules.mk | 1 + platform/broadcom/rules.mk | 2 ++ platform/broadcom/sai.mk | 2 +- platform/cavium/rules.mk | 2 ++ platform/centec/rules.mk | 2 ++ platform/centec/sdk.mk | 2 +- platform/innovium/invm-sai.mk | 2 +- platform/innovium/rules.mk | 1 + platform/marvell/rules.mk | 2 ++ platform/marvell/sai.mk | 2 +- platform/mellanox/mlnx-sai.mk | 2 +- platform/mellanox/rules.mk | 2 ++ platform/nephos/rules.mk | 2 ++ platform/nephos/sai.mk | 2 +- rules/functions | 24 +++++++++++++++++++ rules/sairedis.mk | 1 + rules/swss.mk | 1 + slave.mk | 45 ++++++++++++++++------------------- 19 files changed, 68 insertions(+), 32 deletions(-) diff --git a/platform/barefoot/bfn-sai.mk b/platform/barefoot/bfn-sai.mk index 5d067448d795..f3f83a621658 100644 --- a/platform/barefoot/bfn-sai.mk +++ b/platform/barefoot/bfn-sai.mk @@ -2,5 +2,6 @@ BFN_SAI = bfnsdk_8.5.x.59217b4.deb $(BFN_SAI)_URL = "https://github.com/barefootnetworks/sonic-release-pkgs/raw/rel_8_5/bfnsdk_8.5.x.59217b4.deb" SONIC_ONLINE_DEBS += $(BFN_SAI) # $(BFN_SAI_DEV) +$(eval $(call add_conflict_package,$(BFN_SAI),$(LIBSAIVS_DEV))) + $(BFN_SAI_DEV)_DEPENDS += $(BFN_SAI) -$(BFN_SAI_DEV)_CONFLICTS += $(LIBSAIVS_DEV) diff --git a/platform/barefoot/rules.mk b/platform/barefoot/rules.mk index 17b7dec1c5f6..f8e8d0dfe423 100644 --- a/platform/barefoot/rules.mk +++ b/platform/barefoot/rules.mk @@ -23,6 +23,7 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) $(SONIC_ONE_ABOOT) \ # Inject sai into syncd #$(SYNCD)_DEPENDS += $(BFN_SAI) $(WNC_OSW1800_PLATFORM) $(BFN_INGRASYS_PLATFORM) $(BFN_PLATFORM) $(SYNCD)_DEPENDS += $(BFN_SAI) $(BFN_INGRASYS_PLATFORM) $(BFN_PLATFORM) +$(SYNCD)_UNINSTALLS += $(BFN_SAI) ifeq ($(ENABLE_SYNCD_RPC),y) $(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV) diff --git a/platform/broadcom/rules.mk b/platform/broadcom/rules.mk index dfa75d2455e6..e46e5d012428 100644 --- a/platform/broadcom/rules.mk +++ b/platform/broadcom/rules.mk @@ -33,6 +33,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) $(SONIC_ONE_ABOOT_IMAGE) \ # Inject brcm sai into syncd $(SYNCD)_DEPENDS += $(BRCM_SAI) $(BRCM_SAI_DEV) +$(SYNCD)_UNINSTALLS += $(BRCM_SAI_DEV) + ifeq ($(ENABLE_SYNCD_RPC),y) $(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV) endif diff --git a/platform/broadcom/sai.mk b/platform/broadcom/sai.mk index d51de0cd7523..5f7de76fc324 100644 --- a/platform/broadcom/sai.mk +++ b/platform/broadcom/sai.mk @@ -7,4 +7,4 @@ $(BRCM_SAI_DEV)_URL = "https://sonicstorage.blob.core.windows.net/packages/bcmsa SONIC_ONLINE_DEBS += $(BRCM_SAI) $(BRCM_SAI_DEV)_DEPENDS += $(BRCM_SAI) -$(BRCM_SAI_DEV)_CONFLICTS += $(LIBSAIVS_DEV) +$(eval $(call add_conflict_package,$(BRCM_SAI_DEV),$(LIBSAIVS_DEV))) diff --git a/platform/cavium/rules.mk b/platform/cavium/rules.mk index cdcbcddf8f9b..343ae2a732b4 100644 --- a/platform/cavium/rules.mk +++ b/platform/cavium/rules.mk @@ -16,6 +16,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \ # Inject cavium sai into syncd $(SYNCD)_DEPENDS += $(CAVM_SAI) $(CAVM_LIBSAI) +$(SYNCD)_UNINSTALLS += $(CAVM_LIBSAI) + ifeq ($(ENABLE_SYNCD_RPC),y) $(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV_CAVM) endif diff --git a/platform/centec/rules.mk b/platform/centec/rules.mk index 751c5155d48d..7fe351fc5e7c 100644 --- a/platform/centec/rules.mk +++ b/platform/centec/rules.mk @@ -14,6 +14,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \ # Inject centec sai into syncd $(SYNCD)_DEPENDS += $(CENTEC_SAI) +$(SYNCD)_UNINSTALLS += $(CENTEC_SAI) + ifeq ($(ENABLE_SYNCD_RPC),y) $(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV) endif diff --git a/platform/centec/sdk.mk b/platform/centec/sdk.mk index 308f0b3bf40b..7b6571bdcadd 100644 --- a/platform/centec/sdk.mk +++ b/platform/centec/sdk.mk @@ -1,6 +1,6 @@ # Centec SAI CENTEC_SAI = libsai_1.3.3_amd64.deb $(CENTEC_SAI)_URL = https://github.com/CentecNetworks/goldengate-sai/raw/master/lib/SONiC_1.3.3/libsai_1.3.3-1.0_amd64.deb -$(CENTEC_SAI)_CONFLICTS += $(LIBSAIVS_DEV) +$(eval $(call add_conflict_package,$(CENTEC_SAI),$(LIBSAIVS_DEV))) SONIC_ONLINE_DEBS += $(CENTEC_SAI) diff --git a/platform/innovium/invm-sai.mk b/platform/innovium/invm-sai.mk index 389eb6bd76a5..30ebce0d8aa2 100755 --- a/platform/innovium/invm-sai.mk +++ b/platform/innovium/invm-sai.mk @@ -10,7 +10,7 @@ $(INVM_LIBSAI)_URL = $(INVM_SAI_ONLINE)/$(INVM_LIBSAI) $(INVM_HSAI)_URL = $(INVM_SAI_ONLINE)/$(INVM_HSAI) $(INVM_DRV)_URL = $(INVM_SAI_ONLINE)/$(INVM_DRV) -$(INVM_HSAI)_CONFLICTS += $(LIBSAIVS_DEV) +$(eval $(call add_conflict_package,$(INVM_HSAI),$(LIBSAIVS_DEV))) SONIC_ONLINE_DEBS += $(INVM_LIBSAI) $(INVM_HSAI) $(INVM_DRV) SONIC_STRETCH_DEBS += $(INVM_DRV) diff --git a/platform/innovium/rules.mk b/platform/innovium/rules.mk index 00fc8ad1062e..14bbed09ec4b 100755 --- a/platform/innovium/rules.mk +++ b/platform/innovium/rules.mk @@ -16,6 +16,7 @@ SONIC_ALL += $(SONIC_INVM_ONE_IMAGE) \ # Inject invm sai into syncd $(SYNCD)_DEPENDS += $(INVM_HSAI) $(INVM_LIBSAI) $(LIBSAITHRIFT_DEV_INVM) +$(SYNCD)_UNINSTALLS += $(INVM_HSAI) # Runtime dependency on invm sai is set only for syncd $(SYNCD)_RDEPENDS += $(INVM_HSAI) diff --git a/platform/marvell/rules.mk b/platform/marvell/rules.mk index d0a0f954ac33..3d0c99d8eda8 100644 --- a/platform/marvell/rules.mk +++ b/platform/marvell/rules.mk @@ -13,6 +13,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \ # Inject mrvl sai into syncd $(SYNCD)_DEPENDS += $(MRVL_FPA) $(MRVL_SAI) +$(SYNCD)_UNINSTALLS += $(MRVL_SAI) + ifeq ($(ENABLE_SYNCD_RPC),y) $(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV) endif diff --git a/platform/marvell/sai.mk b/platform/marvell/sai.mk index c6c898a567a1..920f9ea4d9c6 100644 --- a/platform/marvell/sai.mk +++ b/platform/marvell/sai.mk @@ -6,6 +6,6 @@ export MRVL_SAI = mrvllibsai_$(MRVL_SAI_VERSION).deb $(MRVL_SAI)_SRC_PATH = $(PLATFORM_PATH)/sai $(MRVL_SAI)_DEPENDS += $(MRVL_FPA) -$(MRVL_SAI)_CONFLICTS = $(LIBSAIVS_DEV) +$(eval $(call add_conflict_package,$(MRVL_SAI),$(LIBSAIVS_DEV))) SONIC_MAKE_DEBS += $(MRVL_SAI) diff --git a/platform/mellanox/mlnx-sai.mk b/platform/mellanox/mlnx-sai.mk index 2e30a35072ff..1ca32bd2e250 100644 --- a/platform/mellanox/mlnx-sai.mk +++ b/platform/mellanox/mlnx-sai.mk @@ -9,5 +9,5 @@ MLNX_SAI = mlnx-sai_1.mlnx.$(MLNX_SAI_VERSION)_amd64.deb $(MLNX_SAI)_SRC_PATH = $(PLATFORM_PATH)/mlnx-sai $(MLNX_SAI)_DEPENDS += $(MLNX_SDK_DEBS) $(MLNX_SAI)_RDEPENDS += $(MLNX_SDK_RDEBS) $(MLNX_SDK_DEBS) -$(MLNX_SAI)_CONFLICTS += $(LIBSAIVS_DEV) +$(eval $(call add_conflict_package,$(MLNX_SAI),$(LIBSAIVS_DEV))) SONIC_MAKE_DEBS += $(MLNX_SAI) diff --git a/platform/mellanox/rules.mk b/platform/mellanox/rules.mk index 67cef6eb8fee..f48a2aef43a6 100644 --- a/platform/mellanox/rules.mk +++ b/platform/mellanox/rules.mk @@ -23,6 +23,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) \ # Inject mlnx sai into syncd $(SYNCD)_DEPENDS += $(MLNX_SAI) +$(SYNCD)_UNINSTALLS += $(MLNX_SAI) + ifeq ($(ENABLE_SYNCD_RPC),y) $(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV) endif diff --git a/platform/nephos/rules.mk b/platform/nephos/rules.mk index 6424ee76e515..2dc831562a0e 100644 --- a/platform/nephos/rules.mk +++ b/platform/nephos/rules.mk @@ -22,6 +22,8 @@ SONIC_ALL += $(SONIC_ONE_IMAGE) $(DOCKER_FPM) # Inject nephos sai into syncd $(SYNCD)_DEPENDS += $(NEPHOS_SAI) $(NEPHOS_SAI_DEV) +$(SYNCD)_UNINSTALLS += $(NEPHOS_SAI_DEV) + ifeq ($(ENABLE_SYNCD_RPC),y) $(SYNCD)_DEPENDS += $(LIBSAITHRIFT_DEV) endif diff --git a/platform/nephos/sai.mk b/platform/nephos/sai.mk index 40276d55ae1d..be24443b19dd 100644 --- a/platform/nephos/sai.mk +++ b/platform/nephos/sai.mk @@ -11,4 +11,4 @@ $(NEPHOS_SAI_DEV)_URL = "https://github.com/NephosInc/SONiC/raw/master/sai/libsa SONIC_ONLINE_DEBS += $(NEPHOS_SAI) $(NEPHOS_SAI_DEV) $(NEPHOS_SAI_DEV)_DEPENDS += $(NEPHOS_SAI) -$(NEPHOS_SAI_DEV)_CONFLICTS += $(LIBSAIVS_DEV) +$(eval $(call add_conflict_package,$(NEPHOS_SAI_DEV),$(LIBSAIVS_DEV))) diff --git a/rules/functions b/rules/functions index 93e2a346274e..1416372487de 100644 --- a/rules/functions +++ b/rules/functions @@ -91,8 +91,32 @@ $(1)_EXTRA_DEBS += $(2) SONIC_EXTRA_DEBS += $(2) endef +############################################################################### +## Definition of conflict packages +############################################################################### + +# call: +# add_conflict_package some_deb.deb, conflict_deb +define add_conflict_package +$(1)_CONFLICT_DEBS += $(2) +$(2)_CONFLICT_DEBS += $(1) +endef + ############################################################################### ## Utility functions ############################################################################### expand = $(foreach d,$(1),$(call expand,$($(d)_$(2)),$(2))) $(1) + +############################################################################### +## Uninstall debs +############################################################################### +define UNINSTALL_DEBS +$(info $(1)) +$(foreach deb,$(1), \ + { while true; do \ + if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then \ + { sudo DEBIAN_FRONTEND=noninteractive dpkg -P $(firstword $(subst _, ,$(basename $(deb)))) $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; } \ + fi; \ + done; } ) +endef diff --git a/rules/sairedis.mk b/rules/sairedis.mk index e4f8d1d7a5d9..024835a6016d 100644 --- a/rules/sairedis.mk +++ b/rules/sairedis.mk @@ -16,6 +16,7 @@ $(eval $(call add_derived_package,$(LIBSAIREDIS),$(LIBSAIVS))) LIBSAIVS_DEV = libsaivs-dev_1.0.0_amd64.deb $(eval $(call add_derived_package,$(LIBSAIREDIS),$(LIBSAIVS_DEV))) +$(LIBSAIVS_DEV)_DEPENDS += $(LIBSAIVS) LIBSAIMETADATA = libsaimetadata_1.0.0_amd64.deb $(eval $(call add_derived_package,$(LIBSAIREDIS),$(LIBSAIMETADATA))) diff --git a/rules/swss.mk b/rules/swss.mk index daa48cdfcea9..9050f89ba668 100644 --- a/rules/swss.mk +++ b/rules/swss.mk @@ -6,6 +6,7 @@ $(SWSS)_DEPENDS += $(LIBSAIREDIS_DEV) $(LIBSAIMETADATA_DEV) $(LIBTEAM_DEV) \ $(LIBTEAMDCT) $(LIBTEAM_UTILS) $(LIBSWSSCOMMON_DEV) \ $(LIBSAIVS) $(LIBSAIVS_DEV) $(SWSS)_RDEPENDS += $(LIBSAIREDIS) $(LIBSAIMETADATA) $(LIBTEAM) $(LIBSWSSCOMMON) $(PYTHON_SWSSCOMMON) +$(SWSS)_UNINSTALLS = $(LIBSAIVS_DEV) SONIC_DPKG_DEBS += $(SWSS) diff --git a/slave.mk b/slave.mk index e233d15ad7a0..a96639277680 100644 --- a/slave.mk +++ b/slave.mk @@ -254,6 +254,10 @@ $(addprefix $(FILES_PATH)/, $(SONIC_MAKE_FILES)) : $(FILES_PATH)/% : .platform $ make DEST=$(shell pwd)/$(FILES_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(FILES_PATH)/$* $(LOG) # Clean up if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi + + # Uninstall unneeded build dependency + $(call UNINSTALL_DEBS,$($*_UNINSTALLS)) + $(FOOTER) SONIC_TARGET_LIST += $(addprefix $(FILES_PATH)/, $(SONIC_MAKE_FILES)) @@ -280,6 +284,10 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC}" make DEST=$(shell pwd)/$(DEBS_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(DEBS_PATH)/$* $(LOG) # Clean up if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi + + # Uninstall unneeded build dependency + $(call UNINSTALL_DEBS,$($*_UNINSTALLS)) + $(FOOTER) SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) @@ -308,6 +316,10 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi # Take built package(s) mv $(addprefix $($*_SRC_PATH)/../, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS)) $(DEBS_PATH) $(LOG) + + # Uninstall unneeded build dependency + $(call UNINSTALL_DEBS,$($*_UNINSTALLS)) + $(FOOTER) SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) @@ -375,35 +387,18 @@ SONIC_INSTALL_TARGETS = $(addsuffix -install,$(addprefix $(DEBS_PATH)/, \ $(SONIC_PYTHON_STDEB_DEBS) \ $(SONIC_DERIVED_DEBS) \ $(SONIC_EXTRA_DEBS))) -$(SONIC_INSTALL_TARGETS) : $(DEBS_PATH)/%-install : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) $(DEBS_PATH)/$$* $$(addsuffix -uninstall,$$(addprefix $(DEBS_PATH)/,$$($$*_CONFLICTS))) - $(HEADER) - [ -f $(DEBS_PATH)/$* ] || { echo $(DEBS_PATH)/$* does not exist $(LOG) && false $(LOG) } - # put a lock here because dpkg does not allow installing packages in parallel - while true; do - if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then - { sudo dpkg -i $(DEBS_PATH)/$* $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; } - fi - done - $(FOOTER) -# Targets for installing debian packages prior to build one that depends on them -SONIC_UNINSTALL_TARGETS = $(addsuffix -uninstall,$(addprefix $(DEBS_PATH)/, \ - $(SONIC_ONLINE_DEBS) \ - $(SONIC_COPY_DEBS) \ - $(SONIC_MAKE_DEBS) \ - $(SONIC_DPKG_DEBS) \ - $(SONIC_PYTHON_STDEB_DEBS) \ - $(SONIC_DERIVED_DEBS) \ - $(SONIC_EXTRA_DEBS))) - -$(SONIC_UNINSTALL_TARGETS) : $(DEBS_PATH)/%-uninstall : .platform +$(SONIC_INSTALL_TARGETS) : $(DEBS_PATH)/%-install : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) $(DEBS_PATH)/$$* $(HEADER) [ -f $(DEBS_PATH)/$* ] || { echo $(DEBS_PATH)/$* does not exist $(LOG) && false $(LOG) } - # put a lock here because dpkg does not allow installing packages in parallel while true; do - if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then - { sudo DEBIAN_FRONTEND=noninteractive dpkg -P $(firstword $(subst _, ,$(basename $*))) $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; } - fi + # wait for conflicted packages to be uninstalled + $(foreach deb, $($*_CONFLICT_DEBS), \ + { while dpkg -s $(firstword $(subst _, ,$(basename $(deb)))) &> /dev/null; do echo "waiting for $(deb) to be uninstalled" $(LOG); sleep 1; done } ) + # put a lock here because dpkg does not allow installing packages in parallel + if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then + { sudo DEBIAN_FRONTEND=noninteractive dpkg -i $(DEBS_PATH)/$* $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; } + fi done $(FOOTER) From fcc867935fe933dd5d4170268ac96e67ed343447 Mon Sep 17 00:00:00 2001 From: lguohan Date: Tue, 12 Jan 2021 06:03:12 -0800 Subject: [PATCH 2/5] [build]: fix dpkg admindir corruption issue in parallel build (#6408) Fix #119 when parallel build is enable, multiple dpkg-buildpackage instances are running at the same time. /var/lib/dpkg is shared by all instances and the /var/lib/dpkg/updates could be corrupted and cause the build failure. the fix is to use overlay fs to mount separate /var/lib/dpkg for each dpkg-buildpackage instance so that they are not affecting each other. Signed-off-by: Guohan Lu --- rules/functions | 12 ++++++++++++ slave.mk | 8 ++++++-- src/bash/Makefile | 2 +- src/hiredis/Makefile | 2 +- src/initramfs-tools/Makefile | 2 +- src/iproute2/Makefile | 2 +- src/isc-dhcp/Makefile | 2 +- src/libnl3/Makefile | 2 +- src/libteam/Makefile | 2 +- src/lldpd/Makefile | 2 +- src/monit/Makefile | 2 +- src/mpdecimal/Makefile | 2 +- src/python-click/Makefile | 2 +- src/python3/Makefile | 2 +- src/radvd/Makefile | 2 +- src/redis/Makefile | 2 +- src/snmpd/Makefile | 2 +- src/socat/Makefile | 2 +- src/sonic-device-data/Makefile | 2 +- src/sonic-frr/Makefile | 4 ++-- src/tacacs/nss/Makefile | 2 +- src/tacacs/pam/Makefile | 4 ++-- src/thrift/Makefile | 2 +- 23 files changed, 41 insertions(+), 25 deletions(-) diff --git a/rules/functions b/rules/functions index 1416372487de..774d3df5f5e2 100644 --- a/rules/functions +++ b/rules/functions @@ -120,3 +120,15 @@ $(foreach deb,$(1), \ fi; \ done; } ) endef + +############################################################################### +## Setup overlay fs for dpkg admin directory /var/lib/dpkg +############################################################################### +define SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR +upperdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH)) +workdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH)) +mergedir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH)) +sudo mount -t overlay overlay -olowerdir=/var/lib/dpkg,upperdir=$$upperdir,workdir=$$workdir $$mergedir +export SONIC_DPKG_ADMINDIR=$$mergedir +trap "sudo umount $$mergedir && rm -rf $$mergedir $$upperdir $$workdir" EXIT +endef diff --git a/slave.mk b/slave.mk index a96639277680..d69e42875799 100644 --- a/slave.mk +++ b/slave.mk @@ -30,6 +30,7 @@ PYTHON_WHEELS_PATH = $(TARGET_PATH)/python-wheels PROJECT_ROOT = $(shell pwd) DBG_IMAGE_MARK = dbg DBG_SRC_ARCHIVE_FILE = $(TARGET_PATH)/sonic_src.tar.gz +DPKG_ADMINDIR_PATH = /sonic/dpkg CONFIGURED_PLATFORM := $(shell [ -f .platform ] && cat .platform || echo generic) PLATFORM_PATH = platform/$(CONFIGURED_PLATFORM) @@ -52,6 +53,7 @@ configure : @mkdir -p target/debs @mkdir -p target/files @mkdir -p target/python-wheels + @mkdir -p $(DPKG_ADMINDIR_PATH) @echo $(PLATFORM) > .platform distclean : .platform clean @@ -280,6 +282,7 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a rm -f $(addprefix $(DEBS_PATH)/, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS)) # Apply series of patches if exist if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && QUILT_PATCHES=../$(notdir $($*_SRC_PATH)).patch quilt push -a; popd; fi + $(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR) # Build project and take package DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC}" make DEST=$(shell pwd)/$(DEBS_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(DEBS_PATH)/$* $(LOG) # Clean up @@ -307,9 +310,10 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a # Build project pushd $($*_SRC_PATH) $(LOG) [ ! -f ./autogen.sh ] || ./autogen.sh $(LOG) + $(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR) $(if $($*_DPKG_TARGET), - DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) $(LOG), - DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) $(LOG) + DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) --admindir $$mergedir $(LOG), + DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $$mergedir $(LOG) ) popd $(LOG) # Clean up diff --git a/src/bash/Makefile b/src/bash/Makefile index 2e9f093aedb4..ec888d47c658 100644 --- a/src/bash/Makefile +++ b/src/bash/Makefile @@ -10,7 +10,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dget -u https://launchpad.net/debian/+archive/primary/+sourcefiles/bash/$(BASH_VERSION_FULL)/bash_$(BASH_VERSION_FULL).dsc pushd bash-$(BASH_VERSION_MAJOR) - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DEST)/ diff --git a/src/hiredis/Makefile b/src/hiredis/Makefile index d302b327d312..cb7f6316d03e 100644 --- a/src/hiredis/Makefile +++ b/src/hiredis/Makefile @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x hiredis_$(HIREDIS_VERSION_FULL).dsc pushd hiredis-$(HIREDIS_VERSION) - fakeroot debian/rules -j$(SONIC_CONFIG_MAKE_JOBS) binary + dpkg-buildpackage -rfakeroot -d -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DERIVED_TARGETS) $(DEST)/ diff --git a/src/initramfs-tools/Makefile b/src/initramfs-tools/Makefile index 0b08e4aa431e..b412dee54bd7 100644 --- a/src/initramfs-tools/Makefile +++ b/src/initramfs-tools/Makefile @@ -19,7 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # Build the package rm -f debian/*.debhelper.log - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/iproute2/Makefile b/src/iproute2/Makefile index 0181c9ca6d31..eaddbd304757 100644 --- a/src/iproute2/Makefile +++ b/src/iproute2/Makefile @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x iproute2_$(IPROUTE2_VERSION_FULL).dsc pushd iproute2-$(IPROUTE2_VERSION) - dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DEST)/ diff --git a/src/isc-dhcp/Makefile b/src/isc-dhcp/Makefile index 4117b7f75665..aaa2353dbafe 100644 --- a/src/isc-dhcp/Makefile +++ b/src/isc-dhcp/Makefile @@ -24,7 +24,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/libnl3/Makefile b/src/libnl3/Makefile index bf5ba60ab91d..591fc952b6be 100644 --- a/src/libnl3/Makefile +++ b/src/libnl3/Makefile @@ -22,7 +22,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x libnl3_$(LIBNL3_VERSION).dsc pushd ./libnl3-$(LIBNL3_VERSION_BASE) - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/libteam/Makefile b/src/libteam/Makefile index dade8f949aaa..b5dece4eaaff 100644 --- a/src/libteam/Makefile +++ b/src/libteam/Makefile @@ -30,7 +30,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : mv tmp/debian libteam/ rm -rf tmp pushd ./libteam - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/lldpd/Makefile b/src/lldpd/Makefile index 5f502b86863c..387733d2d68a 100644 --- a/src/lldpd/Makefile +++ b/src/lldpd/Makefile @@ -23,7 +23,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - env "with_netlink_receive_bufsize=1024*1024" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + env "with_netlink_receive_bufsize=1024*1024" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/monit/Makefile b/src/monit/Makefile index 8c9817e52731..35e35e2ef63d 100644 --- a/src/monit/Makefile +++ b/src/monit/Makefile @@ -24,7 +24,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/mpdecimal/Makefile b/src/mpdecimal/Makefile index 5603c44b445f..dc5d8a0add12 100644 --- a/src/mpdecimal/Makefile +++ b/src/mpdecimal/Makefile @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x mpdecimal_$(MPDECIMAL_VERSION_FULL).dsc pushd mpdecimal-$(MPDECIMAL_VERSION) - dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DERIVED_TARGETS) $(DEST)/ diff --git a/src/python-click/Makefile b/src/python-click/Makefile index 4deb27257fd2..31716e03616f 100644 --- a/src/python-click/Makefile +++ b/src/python-click/Makefile @@ -19,7 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : git reset --hard debian/$(PYTHON_CLICK_VERSION) # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb package to the destination directory diff --git a/src/python3/Makefile b/src/python3/Makefile index d7718cbeccf6..2d7efeb4b1f9 100644 --- a/src/python3/Makefile +++ b/src/python3/Makefile @@ -32,7 +32,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : fi done - dpkg-buildpackage -rfakeroot -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd cp $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/radvd/Makefile b/src/radvd/Makefile index 9064a43c25c1..148a53aa002d 100644 --- a/src/radvd/Makefile +++ b/src/radvd/Makefile @@ -23,7 +23,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/redis/Makefile b/src/redis/Makefile index f229d321a11f..6685a389ed4b 100644 --- a/src/redis/Makefile +++ b/src/redis/Makefile @@ -16,7 +16,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x redis_$(REDIS_VERSION_FULL).dsc pushd redis-$(REDIS_VERSION) - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/snmpd/Makefile b/src/snmpd/Makefile index e9645b93b686..a774db8ac38e 100644 --- a/src/snmpd/Makefile +++ b/src/snmpd/Makefile @@ -31,7 +31,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg init stg import -s ../patch-$(SNMPD_VERSION)/series - fakeroot debian/rules -j$(SONIC_CONFIG_MAKE_JOBS) binary + dpkg-buildpackage -rfakeroot -b -d -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/socat/Makefile b/src/socat/Makefile index e82208946ac1..08fc5883590d 100644 --- a/src/socat/Makefile +++ b/src/socat/Makefile @@ -18,7 +18,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # Build source and Debian packages pushd socat-1.7.3.1 patch -p0 < ../enable_readline.patch - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/sonic-device-data/Makefile b/src/sonic-device-data/Makefile index ecac92b00e0e..ea4ca75d5b5f 100644 --- a/src/sonic-device-data/Makefile +++ b/src/sonic-device-data/Makefile @@ -22,7 +22,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : done; # Build the package - dpkg-buildpackage -rfakeroot -b -us -uc + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd diff --git a/src/sonic-frr/Makefile b/src/sonic-frr/Makefile index 1b087a78185e..9b7d8b65fb2a 100644 --- a/src/sonic-frr/Makefile +++ b/src/sonic-frr/Makefile @@ -25,7 +25,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : cp -a debianpkg debian make -f debian/rules backports - # new directory to build the package + # new directory to build the package rm -rf frrpkg mkdir frrpkg cd frrpkg @@ -34,7 +34,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : tar xf ../../frr_*sonic.debian8*.debian.tar.xz # build package - dpkg-buildpackage -rfakeroot -b -us -uc + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) cd .. mv $(DERIVED_TARGET) $* $(DEST)/ diff --git a/src/tacacs/nss/Makefile b/src/tacacs/nss/Makefile index 63abfcf2a3c9..9665faf58cc9 100644 --- a/src/tacacs/nss/Makefile +++ b/src/tacacs/nss/Makefile @@ -18,7 +18,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : git am ../0004-Skip-accessing-tacacs-servers-for-local-non-tacacs-u.patch git am ../0005-libnss-Modify-parsing-of-IP-addr-and-port-number-str.patch - dpkg-buildpackage -rfakeroot -b -us -uc + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/tacacs/pam/Makefile b/src/tacacs/pam/Makefile index 627242af5a71..a726f523e357 100644 --- a/src/tacacs/pam/Makefile +++ b/src/tacacs/pam/Makefile @@ -9,7 +9,7 @@ DERIVED_TARGETS = libtac2_$(PAM_TACPLUS_VERSION)_amd64.deb \ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # Obtain pam_tacplus rm -rf ./pam_tacplus - git clone https://github.com/jeroennijhof/pam_tacplus.git + git clone https://github.com/jeroennijhof/pam_tacplus.git pushd ./pam_tacplus git checkout -f v1.4.1 @@ -20,7 +20,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : git apply ../0004-management-vrf-support.patch git apply ../0005-pam-Modify-parsing-of-IP-address-and-port-number-to-.patch - dpkg-buildpackage -rfakeroot -b -us -uc + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/thrift/Makefile b/src/thrift/Makefile index 077b7fb308bd..5ca4b77bfff6 100644 --- a/src/thrift/Makefile +++ b/src/thrift/Makefile @@ -23,7 +23,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x thrift_$(THRIFT_VERSION_FULL).dsc pushd thrift-$(THRIFT_VERSION) patch -p1 < ../patch/THRIFT-3577-assertion-failed.patch - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -d -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -d -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ From ccec723d599841f0ed42c2164448e84bdb5ef8fe Mon Sep 17 00:00:00 2001 From: Guohan Lu Date: Fri, 1 Jan 2021 01:00:47 -0800 Subject: [PATCH 3/5] [build]: fix dpkg uninstall bug fix a bug when there are multiple debian packages to be uninstalled Signed-off-by: Guohan Lu --- rules/functions | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rules/functions b/rules/functions index 774d3df5f5e2..90018f2abf3f 100644 --- a/rules/functions +++ b/rules/functions @@ -112,13 +112,13 @@ expand = $(foreach d,$(1),$(call expand,$($(d)_$(2)),$(2))) $(1) ## Uninstall debs ############################################################################### define UNINSTALL_DEBS -$(info $(1)) -$(foreach deb,$(1), \ - { while true; do \ - if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then \ - { sudo DEBIAN_FRONTEND=noninteractive dpkg -P $(firstword $(subst _, ,$(basename $(deb)))) $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; } \ - fi; \ - done; } ) +if [ -n "$(1)" ]; then \ + while true; do \ + if mkdir $(DEBS_PATH)/dpkg_lock &> /dev/null; then \ + { sudo DEBIAN_FRONTEND=noninteractive dpkg -P $(foreach deb,$(1),$(firstword $(subst _, ,$(basename $(deb))))) $(LOG) && rm -d $(DEBS_PATH)/dpkg_lock && break; } || { rm -d $(DEBS_PATH)/dpkg_lock && exit 1 ; } \ + fi; \ + done; \ +fi endef ############################################################################### From dd625842d8f16502457f0f22488433c93ee1cb86 Mon Sep 17 00:00:00 2001 From: lguohan Date: Wed, 30 Dec 2020 04:57:44 -0800 Subject: [PATCH 4/5] [build]: setup -t option in docker run correctly (#6320) use bash -t test flag to check if input device is tty or not Signed-off-by: Guohan Lu --- Makefile.work | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.work b/Makefile.work index 7536681dd3c6..5a8f9e991148 100644 --- a/Makefile.work +++ b/Makefile.work @@ -80,7 +80,7 @@ DOCKER_RUN := docker run --rm=true --privileged \ -w $(DOCKER_BUILDER_WORKDIR) \ -e "http_proxy=$(http_proxy)" \ -e "https_proxy=$(https_proxy)" \ - -i$(if $(TERM),t,) + -i$(shell { if [ -t 0 ]; then echo t; fi }) include rules/config From 7aca949698928fd7c1539b0ce0133bd48649e682 Mon Sep 17 00:00:00 2001 From: Guohan Lu Date: Wed, 20 Jan 2021 01:06:40 -0800 Subject: [PATCH 5/5] [ci]: add azure pipeline yaml Signed-off-by: Guohan Lu --- .artifactignore | 5 +++ azure-pipelines.yml | 81 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 .artifactignore create mode 100644 azure-pipelines.yml diff --git a/.artifactignore b/.artifactignore new file mode 100644 index 000000000000..4e039ee7c04c --- /dev/null +++ b/.artifactignore @@ -0,0 +1,5 @@ +**/* +!target/*.bin +!target/*.log +!target/*.img.gz +!target/docker-sonic-vs.gz diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000000..3b9776e51cdf --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,81 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- main + +name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) + +resources: + repositories: + - repository: sonic-mgmt + type: github + name: Azure/sonic-mgmt + endpoint: build + +stages: +- stage: Build + pool: sonicbld + + jobs: + - job: + displayName: "broadcom" + timeoutInMinutes: 600 + steps: + - checkout: self + submodules: recursive + displayName: 'Checkout code' + + - script: | + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=rcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" + make configure PLATFORM=broadcom + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.broadcom.201811 + displayName: "Archive sonic image" + + - job: + displayName: "mellanox" + timeoutInMinutes: 600 + steps: + - checkout: self + submodules: recursive + displayName: 'Checkout code' + + - script: | + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=rcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/mellanox" + make configure PLATFORM=mellanox + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-mellanox.bin + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.mellanox.201811 + displayName: "Archive sonic image" + + - job: + displayName: "kvm" + timeoutInMinutes: 600 + steps: + - checkout: self + submodules: recursive + displayName: 'Checkout code' + + - script: | + echo $(Build.BuildNumber) + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=rcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/vs" + ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=vs + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \ + target/docker-sonic-vs.gz target/sonic-vs.img.gz && \ + sudo cp target/sonic-vs.img.gz /nfs/azpl/kvmimage/sonic-vs.$(Build.BuildNumber).img.gz + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.kvm.201811 + displayName: "Archive sonic image"