From 32cda89f9316bc3e57993d9cb967075cf3a0a936 Mon Sep 17 00:00:00 2001 From: xumia <59720581+xumia@users.noreply.github.com> Date: Tue, 21 Jun 2022 13:03:55 +0800 Subject: [PATCH] [Build]: Support to use symbol links for lazy installation targets to reduce the image size (#10923) Why I did it Support to use symbol links in platform folder to reduce the image size. The current solution is to copy each lazy installation targets (xxx.deb files) to each of the folders in the platform folder. The size will keep growing when more and more packages added in the platform folder. For cisco-8000 as an example, the size will be up to 2G, while most of them are duplicate packages in the platform folder. How I did it Create a new folder in platform/common, all the deb packages are copied to the folder, any other folders where use the packages are the symbol links to the common folder. Why platform.tar? We have implemented a patch for it, see #10775, but the problem is the the onie use really old unzip version, cannot support the symbol links. The current solution is similar to the PR 10775, but make the platform folder into a tar package, which can be supported by onie. During the installation, the package.tar will be extracted to the original folder and removed. --- build_debian.sh | 2 +- files/Aboot/boot0.j2 | 7 ++++++- files/build_templates/sonic_debian_extension.j2 | 4 +++- installer/arm64/install.sh | 6 ++++-- installer/armhf/install.sh | 6 ++++-- installer/x86_64/install.sh | 7 +++++-- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/build_debian.sh b/build_debian.sh index 062f72926efa..405360c70a64 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -611,5 +611,5 @@ fi pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd ## Compress together with /boot, /var/lib/docker and $PLATFORM_DIR as an installer payload zip file -pushd $FILESYSTEM_ROOT && sudo zip $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ $PLATFORM_DIR/; popd +pushd $FILESYSTEM_ROOT && sudo tar czf platform.tar.gz -C $PLATFORM_DIR . && sudo zip -n .gz $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ platform.tar.gz; popd sudo zip -g -n .squashfs:.gz $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS diff --git a/files/Aboot/boot0.j2 b/files/Aboot/boot0.j2 index a2ac3debdcea..0944bc54cf6f 100644 --- a/files/Aboot/boot0.j2 +++ b/files/Aboot/boot0.j2 @@ -348,7 +348,12 @@ extract_image() { info "Extracting swi content" ## Unzip the image except boot0 and dockerfs archive - unzip -oq "$swipath" -x boot0 "$dockerfs" -d "$image_path" + unzip -oq "$swipath" -x boot0 "$dockerfs" "platform.tar.gz" -d "$image_path" + + ## Extract the platform.tar.gz + info "Extracting platform.tar.gz" + mkdir -p "$image_path/platform" + unzip -oqp "$swipath" "platform.tar.gz" | tar xzf - -C "$image_path/platform" $TAR_EXTRA_OPTION ## detect rootfs type local mountstr="$(grep " $target_path " /proc/mounts)" diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 1fc1146ef683..db0be75aa0a1 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -590,7 +590,9 @@ sudo LANG=C chroot $FILESYSTEM_ROOT depmod -a {{kversion}} sudo dpkg --root=$FILESYSTEM_ROOT -i {{deb}} || sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f --download-only sudo mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}} -sudo cp {{ deb }} $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}/ +sudo mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/common +sudo cp {{ deb }} $FILESYSTEM_ROOT/$PLATFORM_DIR/common/ +sudo ln -sf "../common/{{ debfilename }}" "$FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}/{{ debfilename }}" for f in $(find $FILESYSTEM_ROOT/var/cache/apt/archives -name "*.deb"); do sudo mv $f $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}/ done diff --git a/installer/arm64/install.sh b/installer/arm64/install.sh index c1e20b62bf42..6e61999ada97 100755 --- a/installer/arm64/install.sh +++ b/installer/arm64/install.sh @@ -133,9 +133,9 @@ fi # Decompress the file for the file system directly to the partition if [ x"$docker_inram" = x"on" ]; then # when disk is small, keep dockerfs.tar.gz in disk, expand it into ramfs during initrd - unzip -o $ONIE_INSTALLER_PAYLOAD -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "platform.tar.gz" -d $demo_mnt/$image_dir else - unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" "platform.tar.gz" -d $demo_mnt/$image_dir if [ "$install_env" = "onie" ]; then TAR_EXTRA_OPTION="--numeric-owner" @@ -146,6 +146,8 @@ else unzip -op $ONIE_INSTALLER_PAYLOAD "$FILESYSTEM_DOCKERFS" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/$DOCKERFS_DIR fi +mkdir -p $demo_mnt/$image_dir/platform +unzip -op $ONIE_INSTALLER_PAYLOAD "platform.tar.gz" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/platform if [ "$install_env" = "onie" ]; then # Store machine description in target file system diff --git a/installer/armhf/install.sh b/installer/armhf/install.sh index c85170ff256e..c0fa82ab2cc3 100755 --- a/installer/armhf/install.sh +++ b/installer/armhf/install.sh @@ -133,9 +133,9 @@ fi # Decompress the file for the file system directly to the partition if [ x"$docker_inram" = x"on" ]; then # when disk is small, keep dockerfs.tar.gz in disk, expand it into ramfs during initrd - unzip -o $ONIE_INSTALLER_PAYLOAD -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "platform.tar.gz" -d $demo_mnt/$image_dir else - unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" "platform.tar.gz" -d $demo_mnt/$image_dir if [ "$install_env" = "onie" ]; then TAR_EXTRA_OPTION="--numeric-owner" @@ -146,6 +146,8 @@ else unzip -op $ONIE_INSTALLER_PAYLOAD "$FILESYSTEM_DOCKERFS" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/$DOCKERFS_DIR fi +mkdir -p $demo_mnt/$image_dir/platform +unzip -op $ONIE_INSTALLER_PAYLOAD "platform.tar.gz" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/platform if [ "$install_env" = "onie" ]; then # Store machine description in target file system diff --git a/installer/x86_64/install.sh b/installer/x86_64/install.sh index a75e70f1f6dd..15ef85e2cf2b 100755 --- a/installer/x86_64/install.sh +++ b/installer/x86_64/install.sh @@ -493,9 +493,9 @@ fi # Decompress the file for the file system directly to the partition if [ x"$docker_inram" = x"on" ]; then # when disk is small, keep dockerfs.tar.gz in disk, expand it into ramfs during initrd - unzip -o $ONIE_INSTALLER_PAYLOAD -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "platform.tar.gz" -d $demo_mnt/$image_dir else - unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" "platform.tar.gz" -d $demo_mnt/$image_dir if [ "$install_env" = "onie" ]; then TAR_EXTRA_OPTION="--numeric-owner" @@ -506,6 +506,9 @@ else unzip -op $ONIE_INSTALLER_PAYLOAD "$FILESYSTEM_DOCKERFS" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/$DOCKERFS_DIR fi +mkdir -p $demo_mnt/$image_dir/platform +unzip -op $ONIE_INSTALLER_PAYLOAD "platform.tar.gz" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/platform + if [ "$install_env" = "onie" ]; then # Store machine description in target file system if [ -f /etc/machine-build.conf ]; then