From 6f7a50e13d829e1cb3ea5f0836703391a066ec7b Mon Sep 17 00:00:00 2001 From: Kalimuthu-Velappan <53821802+Kalimuthu-Velappan@users.noreply.github.com> Date: Tue, 6 Jun 2023 00:30:58 +0530 Subject: [PATCH] 06.Version Cache - Support for wget (#14612) When a package is referenced from the web through wget command, it downloads the package for every build. This feature caches all the packages that are being downloaded from the web, so that subsequent build always loads the cache instead of from web. --- .../scripts/buildinfo_base.sh | 110 ++++++++++++++---- 1 file changed, 90 insertions(+), 20 deletions(-) diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh index a83a5dc67914..365a96b7ae2a 100755 --- a/src/sonic-build-hooks/scripts/buildinfo_base.sh +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -152,57 +152,127 @@ download_packages() { local parameters=("$@") declare -A filenames + declare -A SRC_FILENAMES + local url= + local real_version= + local SRC_FILENAME= + local DST_FILENAME= + for (( i=0; i<${#parameters[@]}; i++ )) do local para=${parameters[$i]} local nexti=$((i+1)) - if [[ "$para" == *://* ]]; then + if [[ "$para" =~ ^-[^-]*o.* || "$para" =~ ^-[^-]*O.* ]]; then + DST_FILENAME="${parameters[$nexti]}" + elif [[ "$para" == *://* ]]; then local url=$para local real_version= # Skip to use the proxy, if the url has already used the proxy server - if [[ $url == ${URL_PREFIX}* ]]; then + if [[ ! -z "${URL_PREFIX}" && $url == ${URL_PREFIX}* ]]; then continue fi + local result=0 + WEB_CACHE_PATH=${PKG_CACHE_PATH}/web + mkdir -p ${WEB_CACHE_PATH} + local WEB_FILENAME=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) + if [ -z "${DST_FILENAME}" ];then + DST_FILENAME="${WEB_FILENAME}" + fi + local VERSION=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}') + if [ ! -z "${VERSION}" ]; then + + if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then + if [ ! -z "$(get_version_cache_option)" ]; then + SRC_FILENAME=${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz + if [ -f "${SRC_FILENAME}" ]; then + log_info "Loading from web cache URL:${url}, SRC:${SRC_FILENAME}, DST:${DST_FILENAME}" + cp "${SRC_FILENAME}" "${DST_FILENAME}" + touch "${SRC_FILENAME}" + continue + fi + fi + fi + fi if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then local version= local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}') if [ -z "$version" ]; then - log_err "Warning: Failed to verify the package: $url, the version is not specified" - continue - fi - - local version_filename="${filename}-${version}" - local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}" - local url_exist=$(check_if_url_exist $proxy_url) - if [ "$url_exist" == y ]; then - parameters[$i]=$proxy_url - filenames[$version_filename]=$filename - real_version=$version + log_err "Warning: Failed to verify the package: $url, the version is not specified" 1>&2 + real_version=$(get_url_version $url) else - real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; } - if [ "$real_version" != "$version" ]; then - log_err "Warning: Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" - continue + + local version_filename="${filename}-${version}" + local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}" + local url_exist=$(check_if_url_exist $proxy_url) + if [ "$url_exist" == y ]; then + parameters[$i]=$proxy_url + filenames[$version_filename]=$filename + real_version=$version + else + real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; } + if [ "$real_version" != "$version" ]; then + log_err "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2 + fi fi fi else real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; } fi + # ignore md5sum for string "" # echo -n "" | md5sum == d41d8cd98f00b204e9800998ecf8427e - [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE} + [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || { + + VERSION=${real_version} + local SRC_FILENAME="${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz" + SRC_FILENAMES[${DST_FILENAME}]="${SRC_FILENAME}" + + echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE} + sort ${BUILD_WEB_VERSION_FILE} -o ${BUILD_WEB_VERSION_FILE} -u &> /dev/null + } fi done + # Skip the real command if all the files are loaded from cache + local result=0 + if [[ ! -z "$(get_version_cache_option)" && ${#SRC_FILENAMES[@]} -eq 0 ]]; then + return $result + fi + $REAL_COMMAND "${parameters[@]}" - local result=$? + result=$? + + #Return if there is any error + if [ ${result} -ne 0 ]; then + exit ${result} + fi for filename in "${!filenames[@]}" do - [ -f "$filename" ] && mv "$filename" "${filenames[$filename]}" + if [ -f "$filename" ] ; then + mv "$filename" "${filenames[$filename]}" + fi + done + + if [[ -z "$(get_version_cache_option)" ]]; then + return $result + fi + + #Save them into cache + for DST_FILENAME in "${!SRC_FILENAMES[@]}" + do + SRC_FILENAME="${SRC_FILENAMES[${DST_FILENAME}]}" + if [[ ! -e "${DST_FILENAME}" || -e "${SRC_FILENAME}" ]] ; then + continue + fi + FLOCK "${SRC_FILENAME}" + cp "${DST_FILENAME}" "${SRC_FILENAME}" + chmod -f 777 "${SRC_FILENAME}" + FUNLOCK "${SRC_FILENAME}" + log_info "Saving into web cache URL:${url}, DST:${SRC_FILENAME}, SRC:${DST_FILENAME}" done return $result