06.Version Cache - Support for wget (sonic-net#14612)

When a package is referenced from the web through wget command, it downloads the package for every build. This feature caches all the packages that are being downloaded from the web, so that subsequent build always loads the cache instead of from web.
sonic-otn · Sep 20, 2023 · 6f7a50e · 6f7a50e
1 parent bd662b0
commit 6f7a50e
Showing 1 changed file with 90 additions and 20 deletions.
diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh
@@ -152,57 +152,127 @@ download_packages()
 {
     local parameters=("$@")
     declare -A filenames
+    declare -A SRC_FILENAMES
+    local url=
+    local real_version=
+    local SRC_FILENAME=
+    local DST_FILENAME=
+
     for (( i=0; i<${#parameters[@]}; i++ ))
     do
         local para=${parameters[$i]}
         local nexti=$((i+1))
-        if [[ "$para" == *://* ]]; then
+        if [[ "$para" =~ ^-[^-]*o.* || "$para" =~ ^-[^-]*O.* ]]; then
+            DST_FILENAME="${parameters[$nexti]}"
+        elif [[ "$para" == *://* ]]; then
             local url=$para
             local real_version=
 
             # Skip to use the proxy, if the url has already used the proxy server
-            if [[ $url == ${URL_PREFIX}* ]]; then
+            if [[ ! -z "${URL_PREFIX}" && $url == ${URL_PREFIX}* ]]; then
                 continue
             fi
+            local result=0
+            WEB_CACHE_PATH=${PKG_CACHE_PATH}/web
+            mkdir -p ${WEB_CACHE_PATH}
+            local WEB_FILENAME=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
+            if [ -z "${DST_FILENAME}" ];then
+                DST_FILENAME="${WEB_FILENAME}"
+            fi
+            local VERSION=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
+            if [ ! -z "${VERSION}" ]; then
+
+                if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then
+                    if [ ! -z "$(get_version_cache_option)" ]; then
+                        SRC_FILENAME=${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz
+                        if [ -f "${SRC_FILENAME}" ]; then
+                            log_info "Loading from web cache URL:${url}, SRC:${SRC_FILENAME}, DST:${DST_FILENAME}"
+                            cp "${SRC_FILENAME}" "${DST_FILENAME}"
+                            touch "${SRC_FILENAME}"
+                            continue
+                        fi
+                    fi
+                fi
+            fi
 
             if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then
                 local version=
                 local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1)
                 [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}')
                 if [ -z "$version" ]; then
-                    log_err "Warning: Failed to verify the package: $url, the version is not specified"
-                    continue
-                fi
-
-                local version_filename="${filename}-${version}"
-                local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}"
-                local url_exist=$(check_if_url_exist $proxy_url)
-                if [ "$url_exist" == y ]; then
-                    parameters[$i]=$proxy_url
-                    filenames[$version_filename]=$filename
-                    real_version=$version
+                    log_err "Warning: Failed to verify the package: $url, the version is not specified" 1>&2
+                    real_version=$(get_url_version $url)
                 else
-                    real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
-                    if [ "$real_version" != "$version" ]; then
-                       log_err "Warning: Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename"
-                       continue
+
+                    local version_filename="${filename}-${version}"
+                    local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}"
+                    local url_exist=$(check_if_url_exist $proxy_url)
+                    if [ "$url_exist" == y ]; then
+                        parameters[$i]=$proxy_url
+                        filenames[$version_filename]=$filename
+                        real_version=$version
+                    else
+                        real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
+                        if [ "$real_version" != "$version" ]; then
+                            log_err "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2
+                        fi
                     fi
                 fi
             else
                 real_version=$(get_url_version $url) || { echo "get_url_version $url failed"; exit 1; }
             fi
+
             # ignore md5sum for string ""
             # echo -n "" | md5sum    ==   d41d8cd98f00b204e9800998ecf8427e
-            [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE}
+            [[ $real_version == "d41d8cd98f00b204e9800998ecf8427e" ]] || {
+
+				VERSION=${real_version}
+				local SRC_FILENAME="${WEB_CACHE_PATH}/${WEB_FILENAME}-${VERSION}.tgz"
+				SRC_FILENAMES[${DST_FILENAME}]="${SRC_FILENAME}"
+
+				echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE}
+				sort ${BUILD_WEB_VERSION_FILE} -o ${BUILD_WEB_VERSION_FILE} -u &> /dev/null
+			}
         fi
     done
 
+    # Skip the real command if all the files are loaded from cache
+    local result=0
+    if [[ ! -z "$(get_version_cache_option)" && ${#SRC_FILENAMES[@]} -eq 0 ]]; then
+        return $result
+    fi
+
     $REAL_COMMAND "${parameters[@]}"
-    local result=$?
+    result=$?
+
+    #Return if there is any error
+    if [ ${result} -ne 0 ]; then
+        exit ${result}
+    fi
 
     for filename in "${!filenames[@]}"
     do
-        [ -f "$filename" ] && mv "$filename" "${filenames[$filename]}"
+        if [ -f "$filename" ] ; then
+            mv "$filename" "${filenames[$filename]}"
+        fi
+    done
+
+    if [[  -z "$(get_version_cache_option)" ]]; then
+        return $result
+    fi
+
+    #Save them into cache
+    for DST_FILENAME in "${!SRC_FILENAMES[@]}"
+    do
+        SRC_FILENAME="${SRC_FILENAMES[${DST_FILENAME}]}"
+        if [[ ! -e "${DST_FILENAME}" || -e "${SRC_FILENAME}" ]] ; then
+            continue
+        fi
+        FLOCK "${SRC_FILENAME}"
+        cp "${DST_FILENAME}" "${SRC_FILENAME}"
+        chmod -f 777 "${SRC_FILENAME}"
+        FUNLOCK "${SRC_FILENAME}"
+        log_info "Saving into web cache URL:${url}, DST:${SRC_FILENAME}, SRC:${DST_FILENAME}"
     done
 
     return $result