Optimize binaries with UPX

.github/workflows/docker.yml

@@ -21,6 +21,9 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Install UPX
+        run: sudo apt-get update && sudo apt-get install upx -y
+
       - name: Test Formulae
         run: |
           set -e

Dockerfile

@@ -6,7 +6,7 @@ ARG BASE_IMAGE
 ENV DEBIAN_FRONTEND noninteractive
 RUN : \
     && apt-get update \
-    && apt-get install -y curl git wget libssl-dev libffi-dev llvm clang gcc g++ pkg-config build-essential jq sudo openssh-client conntrack cloud-utils qemu-utils qemu-kvm qemu-system-x86-64 qemu-system-aarch64 \
+    && apt-get install -y curl git wget libssl-dev libffi-dev llvm clang gcc g++ pkg-config build-essential jq sudo openssh-client conntrack cloud-utils qemu-utils qemu-kvm qemu-system-x86-64 qemu-system-aarch64 upx \
     && rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
 # Install Python versions with deadsnakes.
@@ -70,6 +70,7 @@ RUN --mount=type=bind,src=formulae,target=/tmp/formulae \
     # helm
     #
     && ( curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash ) \
+    && upx `which helm` \
     #
     # [cleanup]
     #
@@ -90,27 +91,27 @@ RUN --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN : \
     && export SCCACHE_REDIS_ENDPOINT=redis://redis-headless.sccache:6379 \
     && sccache --start-server \
     && export RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \
-    && time cargo install cargo-deny --version 0.14.24 \
-    && time cargo install cargo-semver-checks --version 0.33.0 \
-    && time cargo install sqlx-cli --version 0.8.0 \
-    && time cargo install cargo-llvm-cov --version 0.6.11 \
-    && time cargo install cargo-hack --version 0.6.30 \
-    && time cargo install buffrs --version 0.9.0 \
+    && time cargo install cargo-deny --version 0.14.24 --locked \
+    && time cargo install cargo-semver-checks --version 0.33.0 --locked \
+    && time cargo install sqlx-cli --version 0.8.0 --locked \
+    && time cargo install cargo-llvm-cov --version 0.6.11 --locked \
+    && time cargo install cargo-hack --version 0.6.30 --locked \
+    && time cargo install buffrs --version 0.9.0 --locked \
     && sccache --stop-server \
     && du -hd1 /root
 
 #
 # Python tools
 #
 RUN : \
-    && python -m pip install pipx==1.6.0 -v \
-    && pipx install poetry==1.8.3 \
-    && pipx install pdm==2.17.3 \
-    && pipx install slap-cli==1.14.1 \
-    && pipx install kraken-wrapper==0.38.0 \
-    && pipx install uv==0.2.33 \
+    && python -m pip install pipx==1.7.1 uv==0.3.2 -v \
+    && uv tool install poetry==1.8.3 \
+    && uv tool install pdm==2.17.3 \
+    && uv tool install slap-cli==1.14.1 \
+    && uv tool install kraken-wrapper==0.38.0 \
+    # NOTE: Uv does not support --include-deps yet, see https://github.com/astral-sh/uv/issues/6314
     && pipx install ansible==9.8.0 --include-deps \
-    && rm -rf ~/.cache/pip
+    && rm -rf ~/.cache
 
 #
 # Nix

README.md

@@ -51,6 +51,7 @@ the base image in that minor version range besides a higher minor having already
 | QEMU (kvm, x86_64, aarch64)            | apt-get                                                                                         | latest  |
 | sccache                                | [GitHub releases](https://github.com/mozilla/sccache/releases) ([formula](formulae/sccache.py)) | 0.8.1   |
 | sqlx-cli                               | cargo                                                                                           | 0.8.0   |
+| [UPX](https://upx.github.io/)          | apt-get                                                                                         | latest  |
 | wget                                   | apt-get                                                                                         | latest  |
 | [yq](https://mikefarah.gitbook.io/yq/) | [GitHub releases](https://github.com/mikefarah/yq/releases)                                     | 4.44.3  |
 

formulae/argocd.py

@@ -12,3 +12,4 @@ class ArgocdFormula(DownloadFileFormula):
     output_directory = "${install_to}"
     output_file = "${output_directory}/argocd"
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/buf.py

@@ -10,4 +10,6 @@ class BufFormula(DownloadFileFormula):
     download_url = "https://github.com/bufbuild/buf/releases/download/v${version}/buf-${platform}-${archv1}"
     chmod = 0o775
     output_directory = "${install_to}"
+    output_file = "${install_to}/buf"
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/buildkit.py

@@ -7,3 +7,4 @@ class BuildkitFormula(BinaryInstallFormula):
     archive_url = "https://github.com/moby/buildkit/releases/download/v${version}/buildkit-v${version}.linux-${archv2}.tar.gz"
     archive_members = ["bin/*"]
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/cni.py

@@ -8,3 +8,4 @@ class CniFormula(BinaryInstallFormula):
     archive_members = ["*"]
     install_to = "/opt/cni/bin"
     strip_all_components = False
+    upx_optimize = True

formulae/cri-dockerd.py

@@ -7,3 +7,4 @@ class CriDockerdFormula(BinaryInstallFormula):
     archive_url = "https://github.com/Mirantis/cri-dockerd/releases/download/v${version}/cri-dockerd-${version}.amd64.tgz"
     archive_members = ["cri-dockerd/cri-dockerd"]
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/crictl.py

@@ -7,3 +7,4 @@ class CrictlFormula(BinaryInstallFormula):
     archive_url = "https://github.com/kubernetes-sigs/cri-tools/releases/download/${version}/crictl-${version}-linux-amd64.tar.gz"
     archive_members = ["crictl"]
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/grcov.py

@@ -10,3 +10,4 @@ class GrcovFormula(BinaryInstallFormula):
     archive_url = "https://github.com/mozilla/grcov/releases/download/v${version}/grcov-${archv1}-${platform}.tar.bz2"
     archive_members = ["grcov"]
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/kubectl.py

@@ -11,3 +11,4 @@ class KubectlFormula(DownloadFileFormula):
     chmod = 0o775
     output_directory = "${install_to}"
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/manifest-tool.py

@@ -13,3 +13,4 @@ class ManifestToolFormula(BinaryInstallFormula):
     )
     archive_members = {"manifest-tool-${platform}-${archv2}": "manifest-tool"}
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/minikube.py

@@ -12,3 +12,4 @@ class MinikubeFormula(DownloadFileFormula):
     output_directory = "${install_to}"
     output_file = "${output_directory}/minikube"
     install_to = "/usr/local/bin"
+    upx_optimize = True

formulae/protobuf-compiler.py

@@ -13,3 +13,4 @@ class ProtobufCompilerFormula(UnixPackageFormula):
         "https://github.com/protocolbuffers/protobuf/releases/download/v${version}/"
         "protoc-${version}-${platform}-${arch}.zip"
     )
+    upx_optimize = True

formulae/terraform.py

@@ -10,6 +10,7 @@ class TerraformFormula(BinaryInstallFormula):
     archive_url = "https://releases.hashicorp.com/terraform/${version}/terraform_${version}_linux_${archv2}.zip"
     archive_members = ["terraform"]
     install_to = "/usr/local/bin"
+    upx_optimize = True
 
     def finalize(self) -> None:
         self.chmod("+x", "${install_to}/terraform")

src/formula.py

@@ -13,6 +13,7 @@
 import shutil
 import stat
 import string
+import subprocess
 import sys
 import tarfile
 import urllib.request
@@ -119,13 +120,18 @@ def install(self) -> None:
     def finalize(self) -> None:
         pass
 
+
 class BinaryInstallFormula(Formula):
 
     archive_type: str | None = None
     archive_url: str
     archive_members: Sequence[str] | Mapping[str, str]
     install_to: str
     strip_all_components: bool = True
+    mode: int | None = None
+    upx_optimize: bool = False
+
+    _extracted_members: list[str] = []
 
     # Formula
 
@@ -163,13 +169,17 @@ def get_output_filename(x: str) -> str:
                     src, info = archive.get_member(archive_member)
                     with src, output_path.open("wb") as dst:
                         shutil.copyfileobj(src, dst)
-                    output_path.chmod(info.mode)
+                    output_path.chmod(self.mode if self.mode is not None else info.mode)
+                    if self.upx_optimize and os.access(output_path, os.X_OK):
+                        self.log('optimizing binary "%s" with UPX', output_path)
+                        upx_optimize(output_path)
 
 
 class UnixPackageFormula(Formula):
     """ Installs a package that has been split into a typical Unix directory structure, bin/, lib/, include/, etc. """
     archive_url: str
     install_to: str = '/usr/local'
+    upx_optimize: bool = False
 
     def install(self) -> None:
         install_to = self._eval_member('install_to')
@@ -184,6 +194,10 @@ def install(self) -> None:
                 with src, output_path.open("wb") as dst:
                     shutil.copyfileobj(src, dst)
                 output_path.chmod(info.mode)
+                if self.upx_optimize and os.access(output_path, os.X_OK):
+                    self.log('optimizing binary "%s" with UPX', output_path)
+                    upx_optimize(output_path)
+
 
 @contextlib.contextmanager
 def _read_file_archive(filename: str, fp: BinaryIO) -> Iterator[Archive]:
@@ -195,6 +209,7 @@ def _read_file_archive(filename: str, fp: BinaryIO) -> Iterator[Archive]:
     else:
         raise ValueError(f"invalid archive type: {archive_type!r}")
 
+
 def _file_archive_type(filename: str) -> str:
     suffix1 = Path(filename).suffix
     suffix2 = Path(Path(filename).stem).suffix
@@ -205,13 +220,15 @@ def _file_archive_type(filename: str) -> str:
     else:
         raise RuntimeError(f"could not determine archive type from {filename!r}")
 
+
 class DownloadFileFormula(Formula):
 
     download_url: str
     output_file: str | None = None
     output_directory: str | None = None
     chmod: int | None = None
     install_to: str
+    upx_optimize: bool = False
 
     def install(self) -> None:
         download_url = self._eval_member("download_url")
@@ -224,6 +241,7 @@ def install(self) -> None:
         else:
             assert False, "output_file or output_directory must be set"
 
+        self.output_file = output_file
         self.log('fetching file at "%s"', download_url)
         try:
             with urllib.request.urlopen(download_url) as response:
@@ -237,6 +255,11 @@ def install(self) -> None:
             self.log('an unexpected error occurred fetching "%s":\n%s', download_url, exc.read().decode())
             raise
 
+        if self.upx_optimize and os.access(output_file, os.X_OK):
+            self.log('optimizing binary "%s" with UPX', output_file)
+            upx_optimize(output_file)
+
+
 @dataclass
 class ArchiveMemberInfo:
     mode: int
@@ -284,3 +307,7 @@ def get_member(self, name: str) -> tuple[BinaryIO, ArchiveMemberInfo]:
 
     def members(self) -> list[str]:
         return [entry.filename for entry in self._zip.infolist() if not entry.filename.endswith('/')] # dirs end with /
+
+
+def upx_optimize(file: str | Path) -> None:
+    subprocess.run(["upx", str(file)], check=True)