04.Version cache - debug framework

Makefile.work

@@ -175,11 +175,22 @@ ifeq ($(ENABLE_FIPS), y)
 endif
 endif
 
+SONIC_VERSION_CACHE := $(filter-out none,$(SONIC_VERSION_CACHE_METHOD))
+SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE)
+SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE)
+export SONIC_VERSION_CACHE SONIC_VERSION_CACHE_SOURCE
+$(shell test -d $(SONIC_VERSION_CACHE_SOURCE) || \
+    mkdir -p $(SONIC_VERSION_CACHE_SOURCE) && chmod -f 777 $(SONIC_VERSION_CACHE_SOURCE) 2>/dev/null )
+
 # Generate the version control build info
 $(shell \
 	SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
 	TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
 	PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
+	DISTRO=$(BLDENV) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
@@ -198,6 +209,8 @@ $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
 
 PREPARE_DOCKER=BUILD_SLAVE=y \
 	DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/prepare_docker_buildinfo.sh \
 		$(SLAVE_BASE_IMAGE) \
 		$(SLAVE_DIR)/Dockerfile \
@@ -221,12 +234,14 @@ SLAVE_TAG = $(shell \
 	(cat $(SLAVE_DIR)/Dockerfile.user \
 		 $(SLAVE_DIR)/Dockerfile \
 		 $(SLAVE_DIR)/buildinfo/versions/versions-* \
-		 .git/HEAD \
+		.git/HEAD 2>/dev/null \
 	&& echo $(USER)/$(PWD)/$(CONFIGURED_PLATFORM)) \
 	| sha1sum \
 	| awk '{print substr($$1,0,11);}')
 
 COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+	DBGOPT='$(DBGOPT)' \
 	scripts/collect_docker_version_files.sh \
 		$(SLAVE_BASE_IMAGE) \
 		target \
@@ -287,6 +302,10 @@ ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
 	DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache:rw"
 endif
 
+ifneq ($(SONIC_VERSION_CACHE_SOURCE),)
+	DOCKER_RUN += -v "$(SONIC_VERSION_CACHE_SOURCE):/vcache:rw"
+endif
+
 ifeq ($(SONIC_ENABLE_SECUREBOOT_SIGNATURE), y)
 ifneq ($(SIGNING_KEY),)
 	DOCKER_SIGNING_SOURCE := $(shell dirname $(SIGNING_KEY))
@@ -380,6 +399,8 @@ DOCKER_SLAVE_BASE_BUILD = docker build --no-cache \
 		    --build-arg http_proxy=$(http_proxy) \
 		    --build-arg https_proxy=$(https_proxy) \
 		    --build-arg no_proxy=$(no_proxy) \
+		    --build-arg SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
+		    --build-arg SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
 		    $(SLAVE_DIR) \
 		    $(SPLIT_LOG) $(DOCKER_BASE_LOG)
 

build_debian.sh

@@ -81,7 +81,10 @@ echo '[INFO] Build host debian base system...'
 TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT
 
 # Prepare buildinfo
-sudo scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
+sudo SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE} \
+	DBGOPT="${DBGOPT}" \
+	scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
+
 
 sudo chown root:root $FILESYSTEM_ROOT
 
@@ -635,7 +638,9 @@ if [[ $TARGET_BOOTLOADER == uboot ]]; then
 fi
 
 # Collect host image version files before cleanup
-scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
+SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}  \
+	DBGOPT="${DBGOPT}" \
+	scripts/collect_host_image_version_files.sh $CONFIGURED_ARCH $IMAGE_DISTRO $TARGET_PATH $FILESYSTEM_ROOT
 
 # Remove GCC
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc

rules/config

@@ -229,6 +229,15 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt
 #   docker: docker base images
 SONIC_VERSION_CONTROL_COMPONENTS ?= none
 
+# SONIC_VERSION_CACHE allows the .deb,.py, wget, git, docker and go files to be stored in the cache path. This allows the submodules to
+# cache standard installation package and restored back to avoid the package download every time.
+# SONIC_VERSION_CACHE - Method of deb package caching
+#	none    :  no caching
+#	cache   :  Use cache if exists else build the source and update the cache
+# SONIC_VERSION_CACHE_SOURCE - Defines the version cache location details
+SONIC_VERSION_CACHE_METHOD ?= none
+SONIC_VERSION_CACHE_SOURCE ?= $(SONIC_DPKG_CACHE_SOURCE)/vcache
+
 # SONiC docker registry
 #
 # Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry

scripts/build_debian_base_system.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 CONFIGURED_ARCH=$1
 IMAGE_DISTRO=$2
 FILESYSTEM_ROOT=$3

scripts/collect_build_version_files.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 RET=$1
 BLDENV=$2
 TARGET_PATH=$3
@@ -15,12 +17,17 @@ TIMESTAMP=$(date +"%Y%m%d%H%M%S")
 VERSION_BUILD_PATH=$TARGET_PATH/versions/build
 VERSION_SLAVE_PATH=$VERSION_BUILD_PATH/build-sonic-slave-${BLDENV}
 LOG_VERSION_PATH=$VERSION_BUILD_PATH/log-${TIMESTAMP}
+DEFAULT_VERSION_PATH=files/build/versions/default
+BUILD_LOG_PATH=/sonic/target/versions/log/sonic-slave-${BLDENV}/
 
 sudo chmod -R a+rw $BUILDINFO_PATH
 collect_version_files $LOG_VERSION_PATH
 ([ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls $BUILD_VERSION_PATH/)" ]) && cp -rf $BUILD_VERSION_PATH/* $LOG_VERSION_PATH/
 mkdir -p $VERSION_SLAVE_PATH
 
+mkdir -p ${BUILD_LOG_PATH}
+([ -d ${LOG_PATH} ] && [ ! -z "$(ls ${LOG_PATH})" ]) && cp ${LOG_PATH}/* ${BUILD_LOG_PATH}
+
 scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH
 
 [ -d $BUILD_VERSION_PATH ] && rm -rf $BUILD_VERSION_PATH/*

scripts/collect_docker_version_files.sh

@@ -1,6 +1,16 @@
 #!/bin/bash
 
-set -x 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
+BUILDINFO_BASE=/usr/local/share/buildinfo
+
+SCRIPT_SRC_PATH=src/sonic-build-hooks
+if [ -e ${SCRIPT_SRC_PATH} ]; then
+	. ${SCRIPT_SRC_PATH}/scripts/utils.sh 
+else
+	. ${BUILDINFO_BASE}/scripts/utils.sh 
+fi
+
 DOCKER_IMAGE=$1
 TARGET_PATH=$2
 DOCKER_IMAGE_TAG=$3
@@ -13,6 +23,8 @@ DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1 | sed "s/-$DOCKER_USERNAME\
 #Create the container specific to the user tag and slave tag
 DOCKER_CONTAINER=${DOCKER_IMAGE_TAG/:/-}
 TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME
+BUILD_LOG_PATH=target/versions/log/$DOCKER_IMAGE_NAME
+mkdir -p ${BUILD_LOG_PATH}
 
 [ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH
 mkdir -p $TARGET_VERSIONS_PATH
@@ -34,5 +46,7 @@ docker tag ${DOCKER_IMAGE_TAG} tmp-${DOCKER_IMAGE_TAG}
 DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --target output -o target/vcache/${DOCKER_IMAGE_NAME} ${DOCKER_PATH}
 DOCKER_BUILDKIT=1 docker build -f ${DOCKER_PATH}/Dockerfile.cleanup  --no-cache --target final --tag ${DOCKER_IMAGE_TAG} ${DOCKER_PATH}
 docker rmi tmp-${DOCKER_IMAGE_TAG}
+docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/
+
 
 docker container rm $DOCKER_CONTAINER

scripts/collect_host_image_version_files.sh

@@ -1,7 +1,16 @@
 #!/bin/bash
 
-TARGET=$1
-FILESYSTEM_ROOT=$2
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
+SCRIPT_SRC_PATH=src/sonic-build-hooks
+if [ -e ${SCRIPT_SRC_PATH} ]; then
+	. ${SCRIPT_SRC_PATH}/scripts/utils.sh 
+fi
+
+ARCH=$1
+DISTRO=$2
+TARGET=$3
+FILESYSTEM_ROOT=$4
 VERSIONS_PATH=$TARGET/versions/host-image
 IMAGENAME="host-image"
 
@@ -13,3 +22,5 @@ sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo ${IMAGENAME}
 
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/
 cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/
+
+sudo LANG=C chroot $FILESYSTEM_ROOT post_run_cleanup ${IMAGENAME}

scripts/generate_buildinfo_config.sh

@@ -6,5 +6,8 @@ BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config
 
 mkdir -p $BUILDINFO_PATH/buildinfo/config
 
-echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
-echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
+echo "export PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE}" >> $BUILDINFO_CONFIG
+echo "export SONIC_VERSION_CACHE_SOURCE=${SONIC_VERSION_CACHE_SOURCE}" >> $BUILDINFO_CONFIG
+echo "export DISTRO=${DISTRO}" >> $BUILDINFO_CONFIG

scripts/prepare_debian_image_buildinfo.sh

@@ -1,9 +1,13 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 
 ARCH=$1
 DISTRO=$2
 FILESYSTEM_ROOT=$3
+HOST_IMAGE_NAME=host-image
+IMAGENAME=${HOST_IMAGE_NAME}
 
 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh
 VERSION_DEB_PREFERENCE="01-versions-deb"
@@ -26,4 +30,8 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then
 fi
 
 sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb"
-sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo"
+#sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo"
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export DISTRO=${DISTRO} >> /usr/local/share/buildinfo/config/buildinfo.config"
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo export IMAGENAME=${IMAGENAME} >> /usr/local/share/buildinfo/config/buildinfo.config"
+
+sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo ${HOST_IMAGE_NAME}"

scripts/prepare_docker_buildinfo.sh

@@ -1,14 +1,26 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
+BUILDINFO_BASE=/usr/local/share/buildinfo
+
+SCRIPT_SRC_PATH=src/sonic-build-hooks
+if [ -e ${SCRIPT_SRC_PATH} ]; then
+	. ${SCRIPT_SRC_PATH}/scripts/utils.sh 
+else
+	. ${BUILDINFO_BASE}/scripts/utils.sh 
+fi
+
 IMAGENAME=$1
 DOCKERFILE=$2
 ARCH=$3
-DOCKERFILE_TARGE=$4
+DOCKERFILE_TARGET=$4
 DISTRO=$5
 
+
 [ -z "$BUILD_SLAVE" ] && BUILD_SLAVE=n
-[ -z "$DOCKERFILE_TARGE" ] && DOCKERFILE_TARGE=$DOCKERFILE
-DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE")
+[ -z "$DOCKERFILE_TARGET" ] && DOCKERFILE_TARGET=$DOCKERFILE
+DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGET")
 BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo"
 BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions"
 DOCKER_PATH=$(dirname $DOCKERFILE)
@@ -34,28 +46,33 @@ fi
 scripts/docker_version_control.sh $@
 
 DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo
+ARG SONIC_VERSION_CACHE
+ARG SONIC_VERSION_CONTROL_COMPONENTS
 COPY ["buildinfo", "/usr/local/share/buildinfo"]
 COPY vcache/ /sonic/target/vcache/'${IMAGENAME}'
 RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb
 ENV IMAGENAME='${IMAGENAME}'
+ENV DISTRO='${DISTRO}'
 RUN pre_run_buildinfo '${IMAGENAME}'
 '
 
 # Add the auto-generate code if it is not added in the target Dockerfile
-if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then
+if [ ! -f $DOCKERFILE_TARGET ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGET; then
     # Insert the docker build script before the RUN command
     LINE_NUMBER=$(grep -Fn -m 1 'RUN' $DOCKERFILE | cut -d: -f1)
     TEMP_FILE=$(mktemp)
     awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE
 
     # Append the docker build script at the end of the docker file
     echo -e "\nRUN post_run_buildinfo ${IMAGENAME} " >> $TEMP_FILE
+    echo -e "\nRUN post_run_cleanup ${IMAGENAME} " >> $TEMP_FILE
 
-    cat $TEMP_FILE > $DOCKERFILE_TARGE
+    cat $TEMP_FILE > $DOCKERFILE_TARGET
     rm -f $TEMP_FILE
 fi
 
 # Copy the build info config
+mkdir -p ${BUILDINFO_PATH}
 cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH
 
 # Generate the version lock files

scripts/prepare_slave_container_buildinfo.sh

@@ -1,14 +1,19 @@
 #!/bin/bash
 
+[[ ! -z "${DBGOPT}" && $0 =~ ${DBGOPT} ]] && set -x 
+
 SLAVE_DIR=$1
 ARCH=$2
 DISTRO=$3
 
 # Install the latest debian package sonic-build-hooks in the slave container
-sudo dpkg -i --force-overwrite $SLAVE_DIR/buildinfo/sonic-build-hooks_*.deb > /dev/null
+sudo dpkg -i --force-overwrite $SLAVE_DIR/buildinfo/sonic-build-hooks_*.deb &> /dev/null
 
 # Enable the build hooks
-symlink_build_hooks
+sudo symlink_build_hooks
+
+# set the global permissions
+sudo chmod -f 777 /usr/local/share/buildinfo/ -R
 
 # Build the slave running config
 cp -rf $SLAVE_DIR/buildinfo/* /usr/local/share/buildinfo/
@@ -21,8 +26,8 @@ apt-get update > /dev/null 2>&1
 # Build the slave version config
 [ -d /usr/local/share/buildinfo/versions ] && rm -rf /usr/local/share/buildinfo/versions
 scripts/versions_manager.py generate -t "/usr/local/share/buildinfo/versions" -n "build-${SLAVE_DIR}" -d "$DISTRO" -a "$ARCH"
-touch ${BUILDINFO_PATH}/versions/versions-deb
+touch ${BUILDINFO_PATH}/versions/versions-deb ${BUILDINFO_PATH}/versions/versions-web
 
-rm -f /etc/apt/preferences.d/01-versions-deb
-([ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]) && cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/
+sudo rm -f /etc/apt/preferences.d/01-versions-deb
+([ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]) && sudo cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/
 exit 0