From d6b8dc5821faf3928bd95b25559316db8c3880c1 Mon Sep 17 00:00:00 2001 From: Jeff Widman Date: Fri, 1 Sep 2023 10:44:41 -0700 Subject: [PATCH] Replace `gzip` with `zstd` for speed + size benefits Replace `gzip` with `zstd` for both speed + size benefits. I added `zstd` to the base image rather than the python-specific image because: 1. it's only 1,695 KB 2. We'll likely use it in other places down the road, such as unpacking deps for jobs 3. The Python image would have required additional steps like `apt-get update` This results in faster compression + space savings: ```shell root@ca684869d4af:/usr/local/.pyenv/versions# time tar -acf 3.11.5.tar.gz 3.11.5 real 0m5.564s user 0m5.458s sys 0m0.626s root@ca684869d4af:/usr/local/.pyenv/versions# time tar -acf 3.11.5.tar.zst 3.11.5 real 0m0.850s user 0m1.003s sys 0m0.354s root@ca684869d4af:/usr/local/.pyenv/versions# ls -lah -rw-r--r-- 1 root root 32M Sep 1 17:19 3.11.5.tar.gz -rw-r--r-- 1 root root 30M Sep 1 17:20 3.11.5.tar.zst ``` As well as faster decompression too: ```shell root@ca684869d4af:/usr/local/.pyenv/versions# ls -lah -rw-r--r-- 1 root root 32M Sep 1 17:38 3.11.5.tar.gz -rw-r--r-- 1 root root 30M Sep 1 17:36 3.11.5.tar.zst root@ca684869d4af:/usr/local/.pyenv/versions# time tar -axf $PYENV_ROOT/versions/3.11.5.tar.gz -C $PYENV_ROOT/versions real 0m1.113s user 0m0.986s sys 0m0.811s root@ca684869d4af:/usr/local/.pyenv/versions# rm -rf 3.11.5 root@ca684869d4af:/usr/local/.pyenv/versions# time tar -axf $PYENV_ROOT/versions/3.11.5.tar.zst -C $PYENV_ROOT/versions real 0m0.774s user 0m0.501s sys 0m0.695s ``` The `-a` flag to `tar` tells it to autoselect the compression format based on the file extension, so it's flipping between `gzip` and `zstd`. --- Dockerfile.updater-core | 4 +++- python/Dockerfile | 6 +++--- python/lib/dependabot/python/language_version_manager.rb | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile.updater-core b/Dockerfile.updater-core index dc9803acd883..4f42b9ceb413 100644 --- a/Dockerfile.updater-core +++ b/Dockerfile.updater-core @@ -14,9 +14,11 @@ RUN apt-get update \ # dev dependencies for CI build-essential \ curl \ - zlib1g-dev \ libgmp-dev \ + # Compression libs + zlib1g-dev \ unzip \ + zstd \ # VCS section git \ git-lfs \ diff --git a/python/Dockerfile b/python/Dockerfile index 0df12db236c7..f62127cc5263 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -48,7 +48,7 @@ RUN PYENV_VERSION=$PY_3_8 pyenv exec python --version | grep "Python $PY_3_8" || RUN bash /opt/python/helpers/build $PY_3_8 # This python environment occupies ~0.5 GB and gets used for a fraction of jobs, so store it compressed. RUN cd $PYENV_ROOT/versions \ - && tar czf $PY_3_8.tar.gz $PY_3_8 + && tar -acf $PY_3_8.tar.zst $PY_3_8 ## 3.9 # Docker doesn't support parametrizing `COPY --from:python:$PY_1_23-bookworm`, so work around it using an alias. @@ -66,7 +66,7 @@ RUN PYENV_VERSION=$PY_3_9 pyenv exec python --version | grep "Python $PY_3_9" || RUN bash /opt/python/helpers/build $PY_3_9 # This python environment occupies ~0.5 GB and gets used for a fraction of jobs, so store it compressed. RUN cd $PYENV_ROOT/versions \ - && tar czf $PY_3_9.tar.gz $PY_3_9 + && tar -acf $PY_3_9.tar.zst $PY_3_9 ## 3.10 # Docker doesn't support parametrizing `COPY --from:python:$PY_1_23-bookworm`, so work around it using an alias. @@ -84,7 +84,7 @@ RUN PYENV_VERSION=$PY_3_10 pyenv exec python --version | grep "Python $PY_3_10" RUN bash /opt/python/helpers/build $PY_3_10 # This python environment occupies ~0.5 GB and gets used for a fraction of jobs, so store it compressed. RUN cd $PYENV_ROOT/versions \ - && tar czf $PY_3_10.tar.gz $PY_3_10 + && tar -acf $PY_3_10.tar.zst $PY_3_10 ## 3.11 # Docker doesn't support parametrizing `COPY --from:python:$PY_1_23-bookworm`, so work around it using an alias. diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index c207834c58b2..a44bcdb4507c 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -23,7 +23,7 @@ def install_required_python return if SharedHelpers.run_shell_command("pyenv versions").include?(" #{python_major_minor}.") SharedHelpers.run_shell_command( - "tar xzf /usr/local/.pyenv/versions/#{python_version}.tar.gz -C /usr/local/.pyenv/versions" + "tar -axf $PYENV_ROOT/versions/#{python_version}.tar.gz -C $PYENV_ROOT/versions" ) end