Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

06.Version-cache - Environment settings #13024

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

06.Version-cache - Environment settings #13024

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Makefile.work
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ endif
endif

SONIC_VERSION_CACHE := $(filter-out none,$(SONIC_VERSION_CACHE_METHOD))
SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS)
SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE)
SONIC_OVERRIDE_BUILD_VARS += SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE)
export SONIC_VERSION_CACHE SONIC_VERSION_CACHE_SOURCE
Expand All @@ -188,6 +189,7 @@ $(shell \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
DISTRO=$(BLDENV) \
ARCH=$(CONFIGURED_ARCH) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
SONIC_VERSION_CACHE_SOURCE=$(SONIC_VERSION_CACHE_SOURCE) \
DBGOPT='$(DBGOPT)' \
Expand All @@ -211,6 +213,8 @@ PREPARE_DOCKER=BUILD_SLAVE=y \
DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
DISTRO=$(BLDENV) \
ARCH=$(CONFIGURED_ARCH) \
scripts/prepare_docker_buildinfo.sh \
$(SLAVE_BASE_IMAGE) \
$(SLAVE_DIR)/Dockerfile \
Expand Down Expand Up @@ -238,8 +242,11 @@ SLAVE_TAG = $(shell \
| awk '{print substr($$1,0,11);}')

COLLECT_DOCKER=DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
DISTRO=$(BLDENV) \
ARCH=$(CONFIGURED_ARCH) \
scripts/collect_docker_version_files.sh \
$(SLAVE_BASE_IMAGE) \
target \
Expand Down
19 changes: 17 additions & 2 deletions rules/functions
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,23 @@ log_green = echo -e "$(GREEN)$(1)$(GRAY)"

FLUSH_LOG = rm -f $@.log

LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; }
LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; test $${PIPESTATUS[-2]} -eq 0 || { [ $$? -eq 0 ] || pushd $(PROJECT_ROOT) > /dev/null ; ./update_screen.sh -e $@ ; popd > /dev/null ; false ; }
LOG_SIMPLE = &>> $(PROJECT_ROOT)/$@.log || \
{ \
[ $$? -eq 0 ] \
|| pushd $(PROJECT_ROOT) > /dev/null ; \
./update_screen.sh -e $@ ; \
popd > /dev/null ; \
false ; \
}
LOG = < /dev/null |& $(PROJECT_ROOT)/scripts/process_log.sh $(PROCESS_LOG_OPTION) &>> $(PROJECT_ROOT)/$@.log ; \
test $${PIPESTATUS[-2]} -eq 0 || \
{ \
[ $$? -eq 0 ] \
|| pushd $(PROJECT_ROOT) > /dev/null ; \
./update_screen.sh -e $@ ; \
popd > /dev/null ; \
false ; \
}

###############################################################################
## Header and footer for each target
Expand Down
14 changes: 3 additions & 11 deletions scripts/collect_docker_version_files.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE_TAG
docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/
docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/

# Save the cache contents from docker build
IMAGENAME=${DOCKER_IMAGE_TAG} j2 files/build_templates/build_docker_cache.j2 > ${DOCKER_FILE}.cleanup
Expand All @@ -51,16 +52,7 @@ docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/log ${BUILD_LOG_PATH}/

# Save the cache contents from docker build
LOCAL_CACHE_FILE=target/vcache/${DOCKER_IMAGE_NAME}/cache.tgz
CACHE_ENCODE_FILE=${DOCKER_PATH}/vcache/cache.base64
sleep 1; sync ${CACHE_ENCODE_FILE}

# Decode the cache content into gz format
SRC_VERSION_PATH=files/build/versions
if [[ -e ${CACHE_ENCODE_FILE} ]]; then

cat ${CACHE_ENCODE_FILE} | base64 -d >${LOCAL_CACHE_FILE}
rm -f ${CACHE_ENCODE_FILE}
fi

# Version package cache
IMAGE_DBGS_NAME=${DOCKER_IMAGE_NAME//-/_}_image_dbgs
Expand All @@ -70,10 +62,10 @@ else
GLOBAL_CACHE_DIR=/vcache/${DOCKER_IMAGE_NAME}
fi

if [[ ! -z ${SONIC_VERSION_CACHE} && -e ${CACHE_ENCODE_FILE} ]]; then
if [[ ! -z ${SONIC_VERSION_CACHE} ]]; then

# Select version files for SHA calculation
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME/-dbg/}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I try to build broadcom I get cat: 'files/build/versions/dockers/sonic-slave-bullseye/versions-*-bullseye-amd64': No such file or directory when calculate sha1sum hash below.
It's because version file for amd64 doesn't have arch suffix:
versions-deb-bullseye versions-deb-bullseye-arm64 versions-deb-bullseye-armhf versions-py3
Should we fix this?

DEP_FILES="${DOCKER_PATH}/Dockerfile.j2"
if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
DEP_FILES="${DEP_FILES} build_debug_docker_j2.sh"
Expand Down
6 changes: 3 additions & 3 deletions scripts/prepare_docker_buildinfo.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ mkdir -p ${BUILDINFO_PATH}
cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH

# Generate the version lock files
scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH"
scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "${IMAGENAME//-dbg/}" -d "$DISTRO" -a "$ARCH"

touch $BUILDINFO_VERSION_PATH/versions-deb

Expand All @@ -103,7 +103,7 @@ SRC_VERSION_PATH=files/build/versions
if [ ! -z ${SONIC_VERSION_CACHE} ]; then

# Version files for SHA calculation
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
VERSION_FILES="${SRC_VERSION_PATH}/dockers/${DOCKER_IMAGE_NAME/-dbg/}/versions-*-${DISTRO}-${ARCH} ${SRC_VERSION_PATH}/default/versions-*"
DEP_FILES="Dockerfile.j2"
if [[ ${DOCKER_IMAGE_NAME} =~ '-dbg' ]]; then
DEP_DBG_FILES="build_debug_docker_j2.sh"
Expand All @@ -129,7 +129,7 @@ if [ ! -z ${SONIC_VERSION_CACHE} ]; then
if [[ -e ${GLOBAL_CACHE_FILE} ]]; then
cp ${GLOBAL_CACHE_FILE} ${LOCAL_CACHE_FILE}
touch ${GLOBAL_CACHE_FILE}
else
elif [[ "${SONIC_VERSION_CONTROL_COMPONENTS}" != "none" ]]; then
# When file is modified, Global SHA is calculated with the local change.
# Load from the previous version of build cache if exists
VERSIONS=( "HEAD" "HEAD~1" "HEAD~2" )
Expand Down
21 changes: 17 additions & 4 deletions slave.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,11 @@ $(info "INCLUDE_PDE" : "$(INCLUDE_PDE)")
$(info "SONIC_DEBUGGING_ON" : "$(SONIC_DEBUGGING_ON)")
$(info "SONIC_PROFILING_ON" : "$(SONIC_PROFILING_ON)")
$(info "KERNEL_PROCURE_METHOD" : "$(KERNEL_PROCURE_METHOD)")
$(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)")
$(info "SONIC_VERSION_CACHE_METHOD" : "$(SONIC_VERSION_CACHE)")
ifneq ($(SONIC_VERSION_CACHE),)
$(info "SONIC_VERSION_CACHE_SOURCE" : "$(SONIC_VERSION_CACHE_SOURCE)")
endif
$(info "BUILD_TIMESTAMP" : "$(BUILD_TIMESTAMP)")
$(info "BUILD_LOG_TIMESTAMP" : "$(BUILD_LOG_TIMESTAMP)")
$(info "SONIC_IMAGE_VERSION" : "$(SONIC_IMAGE_VERSION)")
Expand Down Expand Up @@ -414,7 +419,6 @@ $(info "ENABLE_NATIVE_WRITE" : "$(ENABLE_NATIVE_WRITE)")
$(info "ENABLE_AUTO_TECH_SUPPORT" : "$(ENABLE_AUTO_TECH_SUPPORT)")
$(info "PDDF_SUPPORT" : "$(PDDF_SUPPORT)")
$(info "MULTIARCH_QEMU_ENVIRON" : "$(MULTIARCH_QEMU_ENVIRON)")
$(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)")
$(info "ENABLE_ASAN" : "$(ENABLE_ASAN)")
$(info "DEFAULT_CONTAINER_REGISTRY" : "$(SONIC_DEFAULT_CONTAINER_REGISTRY)")
ifeq ($(CONFIGURED_PLATFORM),vs)
Expand Down Expand Up @@ -920,6 +924,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
DISTRO=$(BLDENV) \
ARCH=$(CONFIGURED_ARCH) \
scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo $(LOG)
docker info $(LOG)
docker build --squash --no-cache \
Expand All @@ -935,8 +941,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g
-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)

if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH} \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
DISTRO=$(BLDENV) ARCH=${CONFIGURED_ARCH} \
scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)

$(call docker-image-save,$*,$@)
Expand Down Expand Up @@ -1044,6 +1051,8 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
DISTRO=$(BLDENV) \
ARCH=$(CONFIGURED_ARCH) \
scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(LOG)
docker info $(LOG)
docker build --squash --no-cache \
Expand All @@ -1065,8 +1074,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform
-t $(DOCKER_IMAGE_REF) $($*.gz_PATH) $(LOG)

if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH}\
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
DISTRO=$(BLDENV) ARCH=${CONFIGURED_ARCH} \
scripts/collect_docker_version_files.sh $* $(TARGET_PATH) $(DOCKER_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile $(LOG)
if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi

Expand Down Expand Up @@ -1113,6 +1123,8 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
DISTRO=$(BLDENV) \
ARCH=$(CONFIGURED_ARCH) \
scripts/prepare_docker_buildinfo.sh $*-dbg $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH) $(LOG)
docker info $(LOG)
docker build \
Expand All @@ -1129,8 +1141,9 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG
-t $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $(LOG)

if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then docker tag $(DOCKER_IMAGE_REF) $*; fi
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) ARCH=${CONFIGURED_ARCH}\
SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \
DBGOPT='$(DBGOPT)' \
DISTRO=$(BLDENV) ARCH=$(CONFIGURED_ARCH) \
scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) $(DOCKER_DBG_IMAGE_REF) $($*.gz_PATH) $($*.gz_PATH)/Dockerfile-dbg $(LOG)
if [ ! -z $(filter $*.gz,$(SONIC_PACKAGES_LOCAL)) ]; then docker tag $(DOCKER_IMAGE_REF) $*:$(SONIC_IMAGE_VERSION); fi

Expand Down