06.Version-cache - Environment settings

- ARCH flags are not passed correctly to version cache framework.
- Added required environment variables to the cache framework.
- Fixed the uncleaned cache file copy code from version script due to merge.

Makefile.work

@@ -176,6 +176,7 @@ endif
 endif
 
 SONIC_VERSION_CACHE := $(filter-out none,$(SONIC_VERSION_CACHE_METHOD))
+SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS)
 SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE)
 SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE)
 export SONIC_VERSION_CACHE SONIC_VERSION_CACHE_SOURCE
@@ -188,6 +189,7 @@ $(shell \
 	TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
 	PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
 	DISTRO=$(BLDENV) \
+	ARCH=$(CONFIGURED_ARCH) \
 	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 	SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
 	DBGOPT='$(DBGOPT)' \
@@ -211,6 +213,8 @@ PREPARE_DOCKER=BUILD_SLAVE=y \
 	DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
 	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 	DBGOPT='$(DBGOPT)' \
+	DISTRO=$(BLDENV) \
+	ARCH=$(CONFIGURED_ARCH) \
 	scripts/prepare_docker_buildinfo.sh \
 		$(SLAVE_BASE_IMAGE) \
 		$(SLAVE_DIR)/Dockerfile \
@@ -238,8 +242,11 @@ SLAVE_TAG = $(shell \
 	| awk '{print substr($$1,0,11);}')
 
 COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
+	SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
 	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 	DBGOPT='$(DBGOPT)' \
+	DISTRO=$(BLDENV) \
+	ARCH=$(CONFIGURED_ARCH) \
 	scripts/collect_docker_version_files.sh \
 		$(SLAVE_BASE_IMAGE) \
 		target \

rules/functions

@@ -47,8 +47,23 @@ log_green = echo -e "$(GREEN)$(1)$(GRAY)"
 
 FLUSH_LOG = rm -f $@.log
 
-LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; }
-LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; test $${PIPESTATUS[-2]} -eq 0 || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; }
+LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || \
+	{ \
+		[ $$? -eq 0 ] \
+		|| pushd $(PROJECT_ROOT) > /dev/null ; \
+			./update_screen.sh -e $@ ; \
+			popd > /dev/null ; \
+			false ; \
+	}
+LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; \
+	test $${PIPESTATUS[-2]} -eq 0 || \
+	{ \
+		[ $$? -eq 0 ] \
+		|| pushd $(PROJECT_ROOT) > /dev/null ; \
+			./update_screen.sh -e $@ ; \
+			popd > /dev/null ; \
+			false ; \
+	}
 
 ###############################################################################
 ## Header and footer for each target

scripts/collect_docker_version_files.sh

@@ -39,6 +39,7 @@ docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE_TAG
 docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/
 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/
 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/
+docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/
 
 # Save the cache contents from docker build
 IMAGENAME=${DOCKER_IMAGE_TAG} j2 files/build_templates/build_docker_cache.j2 > ${DOCKER_FILE}.cleanup
@@ -51,16 +52,7 @@ docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/
 
 # Save the cache contents from docker build
 LOCAL_CACHE_FILE=target/vcache/${DOCKER_IMAGE_NAME}/cache.tgz
-CACHE_ENCODE_FILE=${DOCKER_PATH}/vcache/cache.base64
-sleep 1; sync ${CACHE_ENCODE_FILE}
-
-# Decode the cache content into gz format
 SRC_VERSION_PATH=files/build/versions
-if [[ -e ${CACHE_ENCODE_FILE} ]]; then
-
-	cat ${CACHE_ENCODE_FILE} | base64 -d  >${LOCAL_CACHE_FILE}
-	rm -f ${CACHE_ENCODE_FILE}
-fi
 
 # Version package cache
 IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
@@ -70,10 +62,10 @@ else
 	GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
 fi
 
-if [[ ! -z ${SONIC_VERSION_CACHE} && -e ${CACHE_ENCODE_FILE} ]]; then
+if [[ ! -z ${SONIC_VERSION_CACHE} ]]; then
 
 	# Select version files for SHA calculation
-	VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
+	VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME/-dbg/}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
 	DEP_FILES="${DOCKER_PATH}/Dockerfile.j2"
 	if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
 		DEP_FILES="${DEP_FILES} build_debug_docker_j2.sh"

scripts/prepare_docker_buildinfo.sh

@@ -76,7 +76,7 @@ mkdir -p ${BUILDINFO_PATH}
 cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH
 
 # Generate the version lock files
-scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH"
+scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "${IMAGENAME//-dbg/}" -d "$DISTRO" -a "$ARCH"
 
 touch $BUILDINFO_VERSION_PATH/versions-deb
 
@@ -103,7 +103,7 @@ SRC_VERSION_PATH=files/build/versions
 if [ ! -z ${SONIC_VERSION_CACHE} ]; then
 
 	# Version files for SHA calculation
-	VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
+	VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME/-dbg/}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
 	DEP_FILES="Dockerfile.j2"
 	if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
 		DEP_DBG_FILES="build_debug_docker_j2.sh"
@@ -129,7 +129,7 @@ if [ ! -z ${SONIC_VERSION_CACHE} ]; then
 	if [[  -e ${GLOBAL_CACHE_FILE} ]]; then
 		cp ${GLOBAL_CACHE_FILE} ${LOCAL_CACHE_FILE}
 		touch ${GLOBAL_CACHE_FILE}
-	else
+	elif [[ "${SONIC_VERSION_CONTROL_COMPONENTS}" != "none" ]]; then
 		# When file is modified, Global SHA is calculated with the local change.
 		# Load from the previous version of build cache if exists
 		VERSIONS=( "HEAD" "HEAD~1" "HEAD~2" )

slave.mk

@@ -387,6 +387,11 @@ $(info "INCLUDE_PDE"                     : "$(INCLUDE_PDE)")
 $(info "SONIC_DEBUGGING_ON"              : "$(SONIC_DEBUGGING_ON)")
 $(info "SONIC_PROFILING_ON"              : "$(SONIC_PROFILING_ON)")
 $(info "KERNEL_PROCURE_METHOD"           : "$(KERNEL_PROCURE_METHOD)")
+$(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)")
+$(info "SONIC_VERSION_CACHE_METHOD"      : "$(SONIC_VERSION_CACHE)")
+ifneq ($(SONIC_VERSION_CACHE),)
+$(info "SONIC_VERSION_CACHE_SOURCE"      : "$(SONIC_VERSION_CACHE_SOURCE)")
+endif
 $(info "BUILD_TIMESTAMP"                 : "$(BUILD_TIMESTAMP)")
 $(info "BUILD_LOG_TIMESTAMP"             : "$(BUILD_LOG_TIMESTAMP)")
 $(info "SONIC_IMAGE_VERSION"             : "$(SONIC_IMAGE_VERSION)")
@@ -414,7 +419,6 @@ $(info "ENABLE_NATIVE_WRITE"             : "$(ENABLE_NATIVE_WRITE)")
 $(info "ENABLE_AUTO_TECH_SUPPORT"        : "$(ENABLE_AUTO_TECH_SUPPORT)")
 $(info "PDDF_SUPPORT"                    : "$(PDDF_SUPPORT)")
 $(info "MULTIARCH_QEMU_ENVIRON"          : "$(MULTIARCH_QEMU_ENVIRON)")
-$(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)")
 $(info "ENABLE_ASAN"                     : "$(ENABLE_ASAN)")
 $(info "DEFAULT_CONTAINER_REGISTRY"      : "$(SONIC_DEFAULT_CONTAINER_REGISTRY)")
 ifeq ($(CONFIGURED_PLATFORM),vs)
@@ -920,6 +924,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
 	TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
 	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 	DBGOPT='$(DBGOPT)' \
+	DISTRO=$(BLDENV) \
+	ARCH=$(CONFIGURED_ARCH) \
 	scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo $(LOG)
 	docker info $(LOG)
 	docker build --squash --no-cache \
@@ -935,8 +941,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
 		-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
 
 	if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH} \
+	SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 		DBGOPT='$(DBGOPT)' \
+		DISTRO=$(BLDENV) ARCH=${CONFIGURED_ARCH} \
 		scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
 
 	$(call docker-image-save,$*,$@)
@@ -1044,6 +1051,8 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
 		TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
 		SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 		DBGOPT='$(DBGOPT)' \
+		DISTRO=$(BLDENV) \
+		ARCH=$(CONFIGURED_ARCH) \
 		scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(LOG)
 		docker info $(LOG)
 		docker build --squash --no-cache \
@@ -1065,8 +1074,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
 			-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)
 
 		if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-		SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH}\
+		SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 			DBGOPT='$(DBGOPT)' \
+			DISTRO=$(BLDENV) ARCH=${CONFIGURED_ARCH} \
 			scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG)
 		if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi
 
@@ -1113,6 +1123,8 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
 		TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
 		SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 		DBGOPT='$(DBGOPT)' \
+		DISTRO=$(BLDENV) \
+		ARCH=$(CONFIGURED_ARCH) \
 		scripts/prepare_docker_buildinfo.sh $*-dbg $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH) $(LOG)
 		docker info $(LOG)
 		docker build \
@@ -1129,8 +1141,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
 			-t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG)
 
 		if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
-		SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH}\
+		SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
 			DBGOPT='$(DBGOPT)' \
+			DISTRO=$(BLDENV) ARCH=$(CONFIGURED_ARCH) \
 			scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH)  $($*.gz_PATH)/Dockerfile-dbg $(LOG)
 		if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi