Backport of Version caching PR into 202211 branch

Back porting of version caching Code PRs into 202211 branch
sonic-net/SONiC#942

  1. Restructuring of Makefile.work
  2. Build cache framework
  3. Environment settings
  4. Debug framework
  5. Docker cache framework

Makefile.work

@@ -179,12 +179,23 @@ ifeq ($(ENABLE_FIPS), y)
 endif
 endif
 
+SONIC_VERSION_CACHE := $(filter-out none,$(SONIC_VERSION_CACHE_METHOD))
+SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE)
+SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE)
+export SONIC_VERSION_CACHE SONIC_VERSION_CACHE_SOURCE
+$(shell test -d $(SONIC_VERSION_CACHE_SOURCE) || \
+    mkdir -p $(SONIC_VERSION_CACHE_SOURCE) && chmod -f 777 $(SONIC_VERSION_CACHE_SOURCE) 2>/dev/null )
+
 # Generate the version control build info
 $(shell \
 	SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
 	TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
 	PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
 	MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
+	DISTRO=$(BLDENV) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
@@ -203,6 +214,8 @@ $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
 
 PREPARE_DOCKER=BUILD_SLAVE=y \
 	DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/prepare_docker_buildinfo.sh \
 		$(SLAVE_BASE_IMAGE) \
 		$(SLAVE_DIR)/Dockerfile \
@@ -216,22 +229,23 @@ $(shell $(PREPARE_DOCKER) )
 SLAVE_BASE_TAG = $(shell \
 	cat $(SLAVE_DIR)/Dockerfile \
 		$(SLAVE_DIR)/sources.list.* \
-		$(SLAVE_DIR)/buildinfo/versions/versions-* \
-		src/sonic-build-hooks/hooks/* 2>/dev/null \
+		$(SLAVE_DIR)/buildinfo/versions/versions-* 2>/dev/null \
 	| sha1sum \
 	| awk '{print substr($$1,0,11);}')
 
 # Calculate the slave TAG based on $(USER)/$(PWD)/$(CONFIGURED_PLATFORM) to get unique SHA ID
 SLAVE_TAG = $(shell \
 	(cat $(SLAVE_DIR)/Dockerfile.user \
 		 $(SLAVE_DIR)/Dockerfile \
+		 $(SLAVE_DIR)/sources.list.* \
 		 $(SLAVE_DIR)/buildinfo/versions/versions-* \
-		 .git/HEAD \
 	&& echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
 	| sha1sum \
 	| awk '{print substr($$1,0,11);}')
 
 COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/collect_docker_version_files.sh \
 		$(SLAVE_BASE_IMAGE) \
 		target \
@@ -292,6 +306,10 @@ ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
 	DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache:rw"
 endif
 
+ifneq ($(SONIC_VERSION_CACHE_SOURCE),)
+	DOCKER_RUN += -v "$(SONIC_VERSION_CACHE_SOURCE):/vcache:rw"
+endif
+
 ifeq ($(SONIC_ENABLE_SECUREBOOT_SIGNATURE), y)
 ifneq ($(SIGNING_KEY),)
 	DOCKER_SIGNING_SOURCE := $(shell dirname $(SIGNING_KEY))
@@ -374,7 +392,7 @@ endif
 
 endif
 
-SPLIT_LOG = | tee
+SPLIT_LOG = 2>&1 | tee
 
 DOCKER_BASE_LOG = $(SLAVE_DIR)/$(SLAVE_BASE_IMAGE)_$(SLAVE_BASE_TAG).log
 DOCKER_LOG = $(SLAVE_DIR)/$(SLAVE_IMAGE)_$(SLAVE_TAG).log
@@ -385,6 +403,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
 		    --build-arg http_proxy=$(http_proxy) \
 		    --build-arg https_proxy=$(https_proxy) \
 		    --build-arg no_proxy=$(no_proxy) \
+		    --build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+		    --build-arg SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
 		    $(SLAVE_DIR) \
 		    $(SPLIT_LOG) $(DOCKER_BASE_LOG)
 
@@ -517,9 +537,7 @@ SONIC_BUILD_INSTRUCTION :=  $(MAKE) \
 .PHONY: sonic-slave-build sonic-slave-bash init reset
 
 
-ifeq ($(filter clean,$(MAKECMDGOALS)),)
 COLLECT_BUILD_VERSION = { scripts/collect_build_version_files.sh \$$?; }
-endif
 
 ifdef SOURCE_FOLDER
 	DOCKER_RUN += -v $(SOURCE_FOLDER):/var/$(USER)/src

build_debian.sh

@@ -81,7 +81,10 @@ echo '[INFO] Build host debian base system...'
 TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT
 
 # Prepare buildinfo
-sudo scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
+sudo SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE} \
+    DBGOPT="${DBGOPT}" \
+    scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
+
 
 sudo chown root:root $FILESYSTEM_ROOT
 
@@ -443,10 +446,10 @@ if [[ $TARGET_BOOTLOADER == grub ]]; then
         GRUB_PKG=grub-efi-arm64-bin
     fi
 
-    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y download \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get install -d -o dir::cache=/var/cache/apt \
         $GRUB_PKG
 
-    sudo mv $FILESYSTEM_ROOT/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
+    sudo cp $FILESYSTEM_ROOT/var/cache/apt/archives/grub*.deb $FILESYSTEM_ROOT/$PLATFORM_DIR/grub
 fi
 
 ## Disable kexec supported reboot which was installed by default
@@ -638,7 +641,9 @@ if [[ $TARGET_BOOTLOADER == uboot ]]; then
 fi
 
 # Collect host image version files before cleanup
-scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
+SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}  \
+    DBGOPT="${DBGOPT}" \
+    scripts/collect_host_image_version_files.sh $CONFIGURED_ARCH $IMAGE_DISTRO $TARGET_PATH $FILESYSTEM_ROOT
 
 # Remove GCC
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc

platform/broadcom/sai.mk

@@ -18,4 +18,9 @@ $(BRCM_DNX_SAI)_URL = "$(LIBSAIBCM_DNX_URL_PREFIX)/$(BRCM_DNX_SAI)"
 SONIC_ONLINE_DEBS += $(BRCM_XGS_SAI)
 SONIC_ONLINE_DEBS += $(BRCM_DNX_SAI)
 $(BRCM_XGS_SAI_DEV)_DEPENDS += $(BRCM_XGS_SAI)
+
+$(BRCM_XGS_SAI)_SKIP_VERSION=y
+$(BRCM_XGS_SAI_DEV)_SKIP_VERSION=y
+$(BRCM_DNX_SAI)_SKIP_VERSION=y
+
 $(eval $(call add_conflict_package,$(BRCM_XGS_SAI_DEV),$(LIBSAIVS_DEV)))

rules/config

@@ -238,6 +238,15 @@ SONIC_VERSION_CONTROL_COMPONENTS ?= none
 # MIRROR_SNAPSHOT - support mirror snapshot flag
 MIRROR_SNAPSHOT ?= n
 
+# SONIC_VERSION_CACHE allows the .deb,.py, wget, git, docker and go files to be stored in the cache path. This allows the submodules to
+# cache standard installation package and restored back to avoid the package download every time.
+# SONIC_VERSION_CACHE - Method of deb package caching
+#   none    :  no caching
+#   cache   :  Use cache if exists else build the source and update the cache
+# SONIC_VERSION_CACHE_SOURCE - Defines the version cache location details
+SONIC_VERSION_CACHE_METHOD ?= none
+SONIC_VERSION_CACHE_SOURCE ?= $(SONIC_DPKG_CACHE_SOURCE)/vcache
+
 # SONiC docker registry
 #
 # Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry

scripts/build_debian_base_system.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 CONFIGURED_ARCH=$1
 IMAGE_DISTRO=$2
 FILESYSTEM_ROOT=$3

scripts/collect_build_version_files.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 RET=$1
 BLDENV=$2
 TARGET_PATH=$3
@@ -15,12 +17,17 @@ TIMESTAMP=$(date +"%Y%m%d%H%M%S")
 VERSION_BUILD_PATH=$TARGET_PATH/versions/build
 VERSION_SLAVE_PATH=$VERSION_BUILD_PATH/build-sonic-slave-${BLDENV}
 LOG_VERSION_PATH=$VERSION_BUILD_PATH/log-${TIMESTAMP}
+DEFAULT_VERSION_PATH=files/build/versions/default
+BUILD_LOG_PATH=/sonic/target/versions/log/sonic-slave-${BLDENV}/
 
 sudo chmod -R a+rw $BUILDINFO_PATH
 collect_version_files $LOG_VERSION_PATH
 ([ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls $BUILD_VERSION_PATH/)" ]) && cp -rf $BUILD_VERSION_PATH/* $LOG_VERSION_PATH/
 mkdir -p $VERSION_SLAVE_PATH
 
+mkdir -p ${BUILD_LOG_PATH}
+([ -d ${LOG_PATH} ] && [ ! -z "$(ls ${LOG_PATH})" ]) && cp ${LOG_PATH}/* ${BUILD_LOG_PATH}
+
 scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH
 
 [ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/*

scripts/collect_docker_version_files.sh

@@ -1,6 +1,16 @@
 #!/bin/bash
 
-set -x 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
+BUILDINFO_BASE=/usr/local/share/buildinfo
+
+SCRIPT_SRC_PATH=src/sonic-build-hooks
+if [ -e ${SCRIPT_SRC_PATH} ]; then
+    . ${SCRIPT_SRC_PATH}/scripts/utils.sh 
+else
+    . ${BUILDINFO_BASE}/scripts/utils.sh 
+fi
+
 DOCKER_IMAGE=$1
 TARGET_PATH=$2
 DOCKER_IMAGE_TAG=$3
@@ -13,6 +23,8 @@ DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\
 #Create the container specific to the user tag and slave tag
 DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-}
 TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME
+BUILD_LOG_PATH=target/versions/log/$DOCKER_IMAGE_NAME
+mkdir -p ${BUILD_LOG_PATH}
 
 [ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH
 mkdir -p $TARGET_VERSIONS_PATH
@@ -34,5 +46,48 @@ docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG}
 DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH}
 DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH}
 docker rmi tmp-${DOCKER_IMAGE_TAG}
+docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/
+
+
+# Save the cache contents from docker build
+LOCAL_CACHE_FILE=target/vcache/${DOCKER_IMAGE_NAME}/cache.tgz
+SRC_VERSION_PATH=files/build/versions
+
+# Version package cache
+IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
+if [[ ${DOCKER_IMAGE_NAME} == sonic-slave-* ]]; then
+    GLOBAL_CACHE_DIR=${SONIC_VERSION_CACHE_SOURCE}/${DOCKER_IMAGE_NAME}
+else
+    GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
+fi
+
+if [[ ! -z ${SONIC_VERSION_CACHE} ]]; then
+
+    # Select version files for SHA calculation
+    VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
+    DEP_FILES="${DOCKER_PATH}/Dockerfile.j2"
+    if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
+        DEP_FILES="${DEP_FILES} build_debug_docker_j2.sh"
+    fi
+
+    # Calculate the version SHA
+    VERSION_SHA="$( (echo -n "${!IMAGE_DBGS_NAME}"; cat ${DEP_FILES} ${VERSION_FILES}) | sha1sum | awk '{print substr($1,0,23);}')"
+    GLOBAL_CACHE_FILE=${GLOBAL_CACHE_DIR}/${DOCKER_IMAGE_NAME}-${VERSION_SHA}.tgz
+
+    GIT_FILE_STATUS=$(git status -s ${DEP_FILES})
+
+    # If the cache file is not exists in the global cache for the given SHA, 
+    # store the new cache file into version cache path.
+    if [ -f ${LOCAL_CACHE_FILE} ]; then
+        if [[ -z ${GIT_FILE_STATUS} && ! -e ${GLOBAL_CACHE_FILE} ]]; then
+            mkdir -p ${GLOBAL_CACHE_DIR}
+            chmod -f 777 ${GLOBAL_CACHE_DIR}
+            FLOCK ${GLOBAL_CACHE_FILE}
+            cp ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE} 
+            chmod -f 777 ${LOCAL_CACHE_FILE} ${GLOBAL_CACHE_FILE} 
+            FUNLOCK ${GLOBAL_CACHE_FILE}
+        fi
+    fi
+fi
 
 docker container rm $DOCKER_CONTAINER

scripts/collect_host_image_version_files.sh

@@ -1,7 +1,16 @@
 #!/bin/bash
 
-TARGET=$1
-FILESYSTEM_ROOT=$2
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
+SCRIPT_SRC_PATH=src/sonic-build-hooks
+if [ -e ${SCRIPT_SRC_PATH} ]; then
+    . ${SCRIPT_SRC_PATH}/scripts/utils.sh 
+fi
+
+ARCH=$1
+DISTRO=$2
+TARGET=$3
+FILESYSTEM_ROOT=$4
 VERSIONS_PATH=$TARGET/versions/host-image
 IMAGENAME="host-image"
 
@@ -13,3 +22,5 @@ sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME}
 
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/
+
+sudo LANG=C chroot $FILESYSTEM_ROOT post_run_cleanup ${IMAGENAME}

scripts/generate_buildinfo_config.sh

@@ -6,6 +6,9 @@ BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config
 
 mkdir -p $BUILDINFO_PATH/buildinfo/config
 
-echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
-echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
+echo "export PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}" >> $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CACHE_SOURCE=${SONIC_VERSION_CACHE_SOURCE}" >> $BUILDINFO_CONFIG
+echo "export DISTRO=${DISTRO}" >> $BUILDINFO_CONFIG
 echo "export MIRROR_SNAPSHOT=$MIRROR_SNAPSHOT" >> $BUILDINFO_CONFIG

scripts/prepare_debian_image_buildinfo.sh

@@ -1,9 +1,13 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 
 ARCH=$1
 DISTRO=$2
 FILESYSTEM_ROOT=$3
+HOST_IMAGE_NAME=host-image
+IMAGENAME=${HOST_IMAGE_NAME}
 
 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh
 VERSION_DEB_PREFERENCE="01-versions-deb"
@@ -26,4 +30,8 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then
 fi
 
 sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb"
-sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo"
+#sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo"
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export DISTRO=${DISTRO} >> /usr/local/share/buildinfo/config/buildinfo.config"
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export IMAGENAME=${IMAGENAME} >> /usr/local/share/buildinfo/config/buildinfo.config"
+
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo ${HOST_IMAGE_NAME}"