install maximum squashfuse compression libs, add opensuse builds

Signed-off-by: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com>
DrDaveD · Sep 27, 2024 · 76e8dd7 · 76e8dd7
1 parent 878ae6b
commit 76e8dd7
Show file tree

Hide file tree

Showing 11 changed files with 188 additions and 56 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -114,8 +114,8 @@ jobs:
       - name: Check License Changes
         run: git diff --exit-code -- LICENSE_DEPENDENCIES.md
 
-  debian:
-    name: debian
+  debbuild-debian11:
+    name: debbuild-debian11
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
@@ -130,7 +130,23 @@ jobs:
           GO_ARCH: linux-amd64
         run: ./scripts/ci-docker-run
 
-  ubuntu-2310:
+  debbuild-debian12:
+    name: debbuild-debian12
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      # fetch tags as checkout@v2 doesn't do that by default
+      - run: git fetch --prune --unshallow --tags --force
+
+      - name: Build and test deb under docker
+        env:
+          OS_TYPE: debian
+          OS_VERSION: 12
+          # setting GO_ARCH speeds things by using go binaries instead of source
+          GO_ARCH: linux-amd64
+        run: ./scripts/ci-docker-run
+
+  debbuild-ubuntu23:
     name: debbuild-ubuntu23
     runs-on: ubuntu-22.04
     steps:
@@ -145,7 +161,7 @@ jobs:
           GO_ARCH: linux-amd64
         run: ./scripts/ci-docker-run
 
-  ubuntu-2404:
+  debbuild-ubuntu24:
     name: debbuild-ubuntu24
     runs-on: ubuntu-22.04
     steps:
@@ -161,8 +177,8 @@ jobs:
         run: ./scripts/ci-docker-run
 
   rpmbuild-rocky8:
-    runs-on: ubuntu-22.04
     name: rpmbuild-rocky8
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
       # fetch tags as checkout@v2 doesn't do that by default
@@ -211,9 +227,16 @@ jobs:
           TEST_TYPE: unpriv
         run: ./scripts/ci-docker-run
 
+      - name: Install and test unprivileged for openSUSE tumbleweed
+        env:
+          OS_TYPE: opensuse/tumbleweed
+          OS_VERSION: latest
+          TEST_TYPE: unpriv
+        run: ./scripts/ci-docker-run
+
   rpmbuild-rocky9:
-    runs-on: ubuntu-22.04
     name: rpmbuild-rocky9
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
       # fetch tags as checkout@v2 doesn't do that by default
@@ -240,6 +263,52 @@ jobs:
           TEST_TYPE: unpriv
         run: ./scripts/ci-docker-run
 
+  rpmbuild-opensuse-leap:
+    name: rpmbuild-opensuse-leap
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      # fetch tags as checkout@v2 doesn't do that by default
+      - run: git fetch --prune --unshallow --tags --force
+
+      - name: Build and test rpm under docker
+        env:
+          OS_TYPE: opensuse/leap
+          OS_VERSION: latest
+          GO_ARCH: linux-amd64
+        run: ./scripts/ci-docker-run
+
+      - name: Install and test unprivileged for openSUSE leap
+        env:
+          OS_TYPE: opensuse/leap
+          OS_VERSION: latest
+          INS_OPTS: -o -d suse15
+          TEST_TYPE: unpriv
+        run: ./scripts/ci-docker-run
+
+  rpmbuild-opensuse-tumbleweed:
+    name: rpmbuild-opensuse-tumbleweed
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      # fetch tags as checkout@v2 doesn't do that by default
+      - run: git fetch --prune --unshallow --tags --force
+
+      - name: Build and test rpm under docker
+        env:
+          OS_TYPE: opensuse/tumbleweed
+          OS_VERSION: latest
+          GO_ARCH: linux-amd64
+        run: ./scripts/ci-docker-run
+
+      - name: Install and test unprivileged for openSUSE tumbleweed
+        env:
+          OS_TYPE: opensuse/tumbleweed
+          OS_VERSION: latest
+          INS_OPTS: -o -d opensuse-tumbleweed
+          TEST_TYPE: unpriv
+        run: ./scripts/ci-docker-run
+
   short_unit_tests:
     name: short_unit_tests
     runs-on: ubuntu-22.04

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,7 +17,11 @@ For older changes see the [archived Singularity change log](https://github.com/a
   (`$HOME/.apptainer/docker-config.json`). The commands `pull`, `push`, `run`,
   `exec`, `shell` and `instance start` can now also be passed a `--authfile
   <path>` option, to read OCI registry credentials from this custom file.
-- Add support for libsubid
+- Add support for libsubid.
+- Expand the build instructions for squashfuse and apptainer packaging to
+  include the libraries needed for maximum support of compression algorithms
+  by squashfuse_ll.
+- Add automated tests for OpenSUSE Leap and Tumbleweed and Debian Bookworm.
 
 ## Changes for v1.3.x
 

diff --git a/INSTALL.md b/INSTALL.md
@@ -36,7 +36,7 @@ Including for libsubid support (requires at least Ubuntu Noble or Debian Bookwor
 sudo apt-get install -y libsubid-dev
 ```
 
-On RHEL or its derivatives:
+On RHEL or its derivatives or Fedora:
 
 ```sh
 # Install basic tools for compiling
@@ -52,11 +52,11 @@ sudo dnf install -y \
     wget git
 ```
 
-Including for libsubid support, use --enablerepo=devel for el8 and el9 but not
+For including libsubid support, use --enablerepo=devel for el8 and el9 but not
 for fedora:
 
 ```sh
-sudo dnf --enablerepo=devel install shadow-utils-subid-devel
+sudo dnf --enablerepo=devel install -y shadow-utils-subid-devel
 ```
 
 On SLE/openSUSE
@@ -67,14 +67,20 @@ sudo zypper install -y \
   libseccomp-devel \
   libuuid-devel \
   openssl-devel \
+  squashfs fakeroot \
   cryptsetup sysuser-tools \
-  gcc go
+  wget git go
+# Install diffutils before devel tools to avoid busybox-diffutils on Tumbleweed
+sudo zypper install -y diffutils
+# Install basic tools for compiling
+# --replacefiles is needed to avoid pam conflict on Tumbleweed
+sudo zypper install -y --replacefiles -t pattern devel_basis
 ```
 
 For libsubid support (requires openSUSE Tumbleweed):
 
 ```sh
-sudo zypper -y install libsubid5  libsubid-devel
+sudo zypper install -y libsubid-devel
 ```
 
 ## Install Go
@@ -166,7 +172,7 @@ make
 sudo make install
 ```
 
-And that's it! Now you can check your Apptainer version by running:
+Now you can check your Apptainer version by running:
 
 ```sh
 apptainer --version
@@ -187,23 +193,30 @@ See the output of `./mconfig -h` for available options.
 
 ## Compiling dependent FUSE-based packages
 
-In many cases Apptainer uses FUSE-based packages in order to mount
-filesystems.  Very often these packages are available in Linux
-distributions but out of date for Apptainer needs, so they need to be
+In most cases Apptainer uses FUSE-based packages in order to mount
+filesystems.  Often some of these packages are available in Linux
+distributions but are out of date for Apptainer needs, so they need to be
 compiled and installed with Apptainer following these directions.
 
 First, make sure that additional required packages are installed.  On Debian:
 
 ```sh
-sudo apt-get install -y autoconf automake libtool pkg-config libfuse3-dev zlib1g-dev
+sudo apt-get install -y autoconf automake libtool pkg-config libfuse3-dev \
+    zlib1g-dev liblzo2-dev liblz4-dev liblzma-dev libzstd-dev
 ```
 
 On RHEL or derivatives:
 
 ```sh
-sudo dnf install -y autoconf automake libtool pkgconfig fuse3-devel zlib-devel
+sudo dnf install -y fuse3-devel lzo-devel lz4-devel
 ```
 
+On SLE/openSUSE:
+
+```sh
+sudo zypper install -y gzip fuse3-devel lzo-devel liblz4-devel \
+    xz-devel libzstd-devel
+
 To download the source code from the top level of the Apptainer source
 tree do:
 
@@ -269,10 +282,10 @@ sudo sysctl -p /etc/sysctl.d/90-disable-userns-restrictions.conf
 
 ## Building & Installing from RPM
 
-On a RHEL / Fedora machine you can build an Apptainer into rpm
+On a RHEL / Fedora /SUSE machine you can build an Apptainer into rpm
 packages, and install it from them. This is useful if you need to install
 Apptainer across multiple machines, or wish to manage all software via
-`dnf`.
+`dnf` (or `zypper` for SUSE).
 
 To build the rpms, in addition to the
 [system dependencies](#install-system-dependencies)
@@ -284,6 +297,12 @@ also install these extra packages:
 sudo dnf install -y rpm-build golang
 ```
 
+or on SLE/openSUSE:
+
+```sh
+sudo zypper install -y rpm-build binutils-gold
+```
+
 The rpm build will use the OS distribution or EPEL version of Go,
 or it will use a different installation of Go, whichever is first in $PATH.
 If the first `go` found in $PATH is too old,

diff --git a/dist/debian/control b/dist/debian/control
@@ -21,6 +21,9 @@ Build-Depends:
  pkg-config,
  libfuse3-dev,
  zlib1g-dev,
+ liblzo2-dev,
+ liblzma-dev,
+ libzstd-dev,
  dh-apparmor
 Standards-Version: 3.9.8
 Homepage: http://apptainer.org

diff --git a/dist/rpm/apptainer.spec.in b/dist/rpm/apptainer.spec.in
@@ -106,12 +106,14 @@ Provides: bundled(fuse-overlayfs) = %{fuse_overlayfs_version}
 %if "%{_target_vendor}" == "suse"
 BuildRequires: binutils-gold
 BuildRequires: go
+BuildRequires: liblz4-devel
 %if 0%{?suse_version} > 1600
 BuildRequires: libsubid-devel
 %endif
 %else
 BuildRequires: shadow-utils-subid-devel
 BuildRequires: golang
+BuildRequires: lz4-devel
 %endif
 BuildRequires: git
 BuildRequires: gcc
@@ -125,6 +127,9 @@ BuildRequires: automake
 BuildRequires: libtool
 BuildRequires: pkgconfig
 BuildRequires: zlib-devel
+BuildRequires: lzo-devel
+BuildRequires: xz-devel
+BuildRequires: libzstd-devel
 %endif
 %if "%{_target_vendor}" == "suse"
 Requires: squashfs

diff --git a/pkg/image/squashfs.go b/pkg/image/squashfs.go
@@ -26,6 +26,7 @@ const (
 	squashfsLzoComp  = 3
 	squashfsXzComp   = 4
 	squashfsLz4Comp  = 5
+	squashfsZstdComp = 6
 )
 
 // this represents the superblock of a v4 squashfs image
@@ -94,6 +95,8 @@ func CheckSquashfsHeader(b []byte) (uint64, error) {
 			compressionType = "lzo"
 		case squashfsXzComp:
 			compressionType = "xz"
+		case squashfsZstdComp:
+			compressionType = "zstd"
 		default:
 			return 0, fmt.Errorf("corrupted image: unknown compression algorithm value %d", sinfo.Compression)
 		}
@@ -124,6 +127,8 @@ func GetSquashfsComp(b []byte) (string, error) {
 			compType = "lzo"
 		case squashfsXzComp:
 			compType = "xz"
+		case squashfsZstdComp:
+			compType = "zstd"
 		}
 		return compType, nil
 	} else if sb.Major < 4 {

diff --git a/scripts/ci-deb-build-test b/scripts/ci-deb-build-test
@@ -36,11 +36,13 @@ apt-get install -y \
     dh-apparmor
 
 # for squashfuse_ll build
-apt-get install -y autoconf automake libtool pkg-config libfuse3-dev zlib1g-dev
+apt-get install -y autoconf automake libtool pkg-config libfuse3-dev \
+    zlib1g-dev liblzo2-dev liblz4-dev liblzma-dev libzstd-dev
 
-# for libsubid support in Ubuntu 24.04
-if [[ $OS_NAME = "Ubuntu" ]] && [ $OS_MAJOR -gt 23 ]; then
-    apt-get install -y libsubid4 libsubid-dev
+# for libsubid support in Ubuntu 24.04+ or Debian 12+
+if { [ $OS_NAME = "Ubuntu" ] && [ $OS_MAJOR -ge 24 ]; } || \
+	{ [ $OS_NAME = "Debian" ] && [ $OS_MAJOR -ge 12 ]; }; then
+    apt-get install -y libsubid-dev
 fi
 
 # move source code down a level because debuild writes into parent dir

diff --git a/scripts/ci-docker-run b/scripts/ci-docker-run
@@ -26,7 +26,7 @@ CONTAINER_VERS="${CONTAINER_VERS:-$DOCKER_HUB_URI}"
 
 docker run --privileged --network=host -v "$(pwd):/build:rw" \
   -e OS_TYPE=$OS_TYPE -e GO_ARCH=$GO_ARCH -e CONTAINER_VERS="$CONTAINER_VERS" \
-  -e HIDE_DIST=$HIDE_DIST \
+  -e HIDE_DIST=$HIDE_DIST -e INS_OPTS="$INS_OPTS" \
   --name "$DOCKER_CONTAINER_NAME" "$DOCKER_HUB_URI" /bin/bash -exc \
 	"cd /build && scripts/ci-${TEST_TYPE:-$PKGTYPE-build}-test"
 

diff --git a/scripts/ci-rpm-build-test b/scripts/ci-rpm-build-test
@@ -7,17 +7,31 @@
 # this script runs as root under docker --privileged
 
 # install dependencies
-dnf install -y rpm-build make yum-utils gcc binutils util-linux-ng which git
-dnf install -y libseccomp-devel cryptsetup
-dnf --enablerepo=devel install -y shadow-utils-subid-devel
-if [ $OS_TYPE != fedora ]; then
-  dnf install -y epel-release
+if [[ $OS_TYPE == *suse* ]]; then
+  zypper install -y libseccomp-devel libuuid-devel openssl-devel \
+    squashfs fakeroot cryptsetup sysuser-tools wget git go
+  zypper install -y diffutils
+  zypper install -y --replacefiles -t pattern devel_basis
+  if [[ $OS_TYPE == *tumbleweed* ]]; then
+    zypper install -y libsubid-devel
+  fi
+  zypper install -y gzip fuse3-devel lzo-devel liblz4-devel \
+    xz-devel libzstd-devel
+  zypper install -y rpm-build binutils-gold
+  zypper install -y sudo
+else
+  dnf groupinstall -y 'Development Tools'
+  if [ $OS_TYPE != fedora ]; then
+    dnf install -y epel-release
+  fi
+  dnf install -y libseccomp-devel squashfs-tools fakeroot cryptsetup wget git
+  dnf --enablerepo=devel install -y shadow-utils-subid-devel
+  dnf install -y golang
+  dnf install -y fuse3-devel lzo-devel lz4-devel
+  dnf install -y sudo
 fi
-dnf install -y golang fakeroot
-dnf install -y autoconf automake libtool pkgconfig fuse3-devel zlib-devel
 
 # switch to an unprivileged user with sudo privileges
-dnf install -y sudo
 useradd -u 1000 --create-home -s /bin/bash testuser
 echo "Defaults:testuser env_keep=DOCKER_HOST" >>/etc/sudoers
 echo "testuser ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
@@ -48,7 +62,11 @@ su testuser -c '
     echo "%dist %{nil}" >$HOME/.rpmmacros
   fi
   make -C builddir rpm
-  sudo dnf install -y $HOME/rpmbuild/RPMS/*/*.rpm
+  if [[ '$OS_TYPE' == *suse* ]]; then
+    sudo zypper install -y --allow-unsigned-rpm $HOME/rpmbuild/RPMS/*/*.rpm
+  else
+    sudo dnf install -y $HOME/rpmbuild/RPMS/*/*.rpm
+  fi
   BLD="$(echo $HOME/rpmbuild/BUILD/apptainer-*)"
   export GOPATH=$BLD/gopath
   PATH=$GOPATH/bin:$PATH