From 14f23faaaf803c46ef60341ce0ce2707b67838e6 Mon Sep 17 00:00:00 2001 From: Kalimuthu Velappan Date: Sun, 4 Sep 2022 09:30:58 -0700 Subject: [PATCH] 06.Version-cache - Environment settings - ARCH flags are not passed correctly to version cache framework. - Added required environment variables to the cache framework. - Fixed the uncleaned cache file copy code from version script due to merge. --- Makefile.work | 7 +++++++ rules/functions | 19 +++++++++++++++++-- scripts/collect_docker_version_files.sh | 14 +++----------- scripts/prepare_docker_buildinfo.sh | 6 +++--- slave.mk | 21 +++++++++++++++++---- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/Makefile.work b/Makefile.work index 4c73dbfc88b1..dcfb57dbdaaa 100644 --- a/Makefile.work +++ b/Makefile.work @@ -176,6 +176,7 @@ endif endif SONIC_VERSION_CACHE := $(filter-out none,$(SONIC_VERSION_CACHE_METHOD)) +SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) export SONIC_VERSION_CACHE SONIC_VERSION_CACHE_SOURCE @@ -188,6 +189,7 @@ $(shell \ TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \ DBGOPT='$(DBGOPT)' \ @@ -211,6 +213,8 @@ PREPARE_DOCKER=BUILD_SLAVE=y \ DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \ SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ DBGOPT='$(DBGOPT)' \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ scripts/prepare_docker_buildinfo.sh \ $(SLAVE_BASE_IMAGE) \ $(SLAVE_DIR)/Dockerfile \ @@ -238,8 +242,11 @@ SLAVE_TAG = $(shell \ | awk '{print substr($$1,0,11);}') COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \ + SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \ SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ DBGOPT='$(DBGOPT)' \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ scripts/collect_docker_version_files.sh \ $(SLAVE_BASE_IMAGE) \ target \ diff --git a/rules/functions b/rules/functions index 62b3c8a98375..4c5ae8b5bfb4 100644 --- a/rules/functions +++ b/rules/functions @@ -47,8 +47,23 @@ log_green = echo -e "$(GREEN)$(1)$(GRAY)" FLUSH_LOG = rm -f $@.log -LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; } -LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; test $${PIPESTATUS[-2]} -eq 0 || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; } +LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || \ + { \ + [ $$? -eq 0 ] \ + || pushd $(PROJECT_ROOT) > /dev/null ; \ + ./update_screen.sh -e $@ ; \ + popd > /dev/null ; \ + false ; \ + } +LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; \ + test $${PIPESTATUS[-2]} -eq 0 || \ + { \ + [ $$? -eq 0 ] \ + || pushd $(PROJECT_ROOT) > /dev/null ; \ + ./update_screen.sh -e $@ ; \ + popd > /dev/null ; \ + false ; \ + } ############################################################################### ## Header and footer for each target diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh index 2452a4fd09cc..d50dea57e9ea 100755 --- a/scripts/collect_docker_version_files.sh +++ b/scripts/collect_docker_version_files.sh @@ -39,6 +39,7 @@ docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE_TAG docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/ docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/ +docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/ # Save the cache contents from docker build IMAGENAME=${DOCKER_IMAGE_TAG} j2 files/build_templates/build_docker_cache.j2 > ${DOCKER_FILE}.cleanup @@ -51,16 +52,7 @@ docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/ # Save the cache contents from docker build LOCAL_CACHE_FILE=target/vcache/${DOCKER_IMAGE_NAME}/cache.tgz -CACHE_ENCODE_FILE=${DOCKER_PATH}/vcache/cache.base64 -sleep 1; sync ${CACHE_ENCODE_FILE} - -# Decode the cache content into gz format SRC_VERSION_PATH=files/build/versions -if [[ -e ${CACHE_ENCODE_FILE} ]]; then - - cat ${CACHE_ENCODE_FILE} | base64 -d >${LOCAL_CACHE_FILE} - rm -f ${CACHE_ENCODE_FILE} -fi # Version package cache IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs @@ -70,10 +62,10 @@ else GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME} fi -if [[ ! -z ${SONIC_VERSION_CACHE} && -e ${CACHE_ENCODE_FILE} ]]; then +if [[ ! -z ${SONIC_VERSION_CACHE} ]]; then # Select version files for SHA calculation - VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*" + VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME/-dbg/}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*" DEP_FILES="${DOCKER_PATH}/Dockerfile.j2" if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then DEP_FILES="${DEP_FILES} build_debug_docker_j2.sh" diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 3be1997098ef..52370ebc7bd4 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -76,7 +76,7 @@ mkdir -p ${BUILDINFO_PATH} cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH # Generate the version lock files -scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH" +scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "${IMAGENAME//-dbg/}" -d "$DISTRO" -a "$ARCH" touch $BUILDINFO_VERSION_PATH/versions-deb @@ -103,7 +103,7 @@ SRC_VERSION_PATH=files/build/versions if [ ! -z ${SONIC_VERSION_CACHE} ]; then # Version files for SHA calculation - VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*" + VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME/-dbg/}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*" DEP_FILES="Dockerfile.j2" if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then DEP_DBG_FILES="build_debug_docker_j2.sh" @@ -129,7 +129,7 @@ if [ ! -z ${SONIC_VERSION_CACHE} ]; then if [[ -e ${GLOBAL_CACHE_FILE} ]]; then cp ${GLOBAL_CACHE_FILE} ${LOCAL_CACHE_FILE} touch ${GLOBAL_CACHE_FILE} - else + elif [[ "${SONIC_VERSION_CONTROL_COMPONENTS}" != "none" ]]; then # When file is modified, Global SHA is calculated with the local change. # Load from the previous version of build cache if exists VERSIONS=( "HEAD" "HEAD~1" "HEAD~2" ) diff --git a/slave.mk b/slave.mk index ff43ba11f561..de639faa6e29 100644 --- a/slave.mk +++ b/slave.mk @@ -387,6 +387,11 @@ $(info "INCLUDE_PDE" : "$(INCLUDE_PDE)") $(info "SONIC_DEBUGGING_ON" : "$(SONIC_DEBUGGING_ON)") $(info "SONIC_PROFILING_ON" : "$(SONIC_PROFILING_ON)") $(info "KERNEL_PROCURE_METHOD" : "$(KERNEL_PROCURE_METHOD)") +$(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)") +$(info "SONIC_VERSION_CACHE_METHOD" : "$(SONIC_VERSION_CACHE)") +ifneq ($(SONIC_VERSION_CACHE),) +$(info "SONIC_VERSION_CACHE_SOURCE" : "$(SONIC_VERSION_CACHE_SOURCE)") +endif $(info "BUILD_TIMESTAMP" : "$(BUILD_TIMESTAMP)") $(info "BUILD_LOG_TIMESTAMP" : "$(BUILD_LOG_TIMESTAMP)") $(info "SONIC_IMAGE_VERSION" : "$(SONIC_IMAGE_VERSION)") @@ -414,7 +419,6 @@ $(info "ENABLE_NATIVE_WRITE" : "$(ENABLE_NATIVE_WRITE)") $(info "ENABLE_AUTO_TECH_SUPPORT" : "$(ENABLE_AUTO_TECH_SUPPORT)") $(info "PDDF_SUPPORT" : "$(PDDF_SUPPORT)") $(info "MULTIARCH_QEMU_ENVIRON" : "$(MULTIARCH_QEMU_ENVIRON)") -$(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)") $(info "ENABLE_ASAN" : "$(ENABLE_ASAN)") $(info "DEFAULT_CONTAINER_REGISTRY" : "$(SONIC_DEFAULT_CONTAINER_REGISTRY)") ifeq ($(CONFIGURED_PLATFORM),vs) @@ -920,6 +924,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ DBGOPT='$(DBGOPT)' \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo $(LOG) docker info $(LOG) docker build --squash --no-cache \ @@ -935,8 +941,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g -t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH} \ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ DBGOPT='$(DBGOPT)' \ + DISTRO=$(BLDENV) ARCH=${CONFIGURED_ARCH} \ scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) $(call docker-image-save,$*,$@) @@ -1044,6 +1051,8 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ DBGOPT='$(DBGOPT)' \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(LOG) docker info $(LOG) docker build --squash --no-cache \ @@ -1065,8 +1074,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform -t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG) if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH}\ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ DBGOPT='$(DBGOPT)' \ + DISTRO=$(BLDENV) ARCH=${CONFIGURED_ARCH} \ scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG) if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi @@ -1113,6 +1123,8 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ DBGOPT='$(DBGOPT)' \ + DISTRO=$(BLDENV) \ + ARCH=$(CONFIGURED_ARCH) \ scripts/prepare_docker_buildinfo.sh $*-dbg $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH) $(LOG) docker info $(LOG) docker build \ @@ -1129,8 +1141,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG -t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG) if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi - SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH}\ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ DBGOPT='$(DBGOPT)' \ + DISTRO=$(BLDENV) ARCH=$(CONFIGURED_ARCH) \ scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile-dbg $(LOG) if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi