diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f4cf69f50..abc7191121 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,8 +114,8 @@ jobs: - name: Check License Changes run: git diff --exit-code -- LICENSE_DEPENDENCIES.md - debian: - name: debian + debbuild-debian11: + name: debbuild-debian11 runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 @@ -130,7 +130,23 @@ jobs: GO_ARCH: linux-amd64 run: ./scripts/ci-docker-run - ubuntu-2310: + debbuild-debian12: + name: debbuild-debian12 + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + # fetch tags as checkout@v2 doesn't do that by default + - run: git fetch --prune --unshallow --tags --force + + - name: Build and test deb under docker + env: + OS_TYPE: debian + OS_VERSION: 12 + # setting GO_ARCH speeds things by using go binaries instead of source + GO_ARCH: linux-amd64 + run: ./scripts/ci-docker-run + + debbuild-ubuntu23: name: debbuild-ubuntu23 runs-on: ubuntu-22.04 steps: @@ -145,7 +161,7 @@ jobs: GO_ARCH: linux-amd64 run: ./scripts/ci-docker-run - ubuntu-2404: + debbuild-ubuntu24: name: debbuild-ubuntu24 runs-on: ubuntu-22.04 steps: @@ -161,8 +177,8 @@ jobs: run: ./scripts/ci-docker-run rpmbuild-rocky8: - runs-on: ubuntu-22.04 name: rpmbuild-rocky8 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 # fetch tags as checkout@v2 doesn't do that by default @@ -211,9 +227,16 @@ jobs: TEST_TYPE: unpriv run: ./scripts/ci-docker-run + - name: Install and test unprivileged for openSUSE tumbleweed + env: + OS_TYPE: opensuse/tumbleweed + OS_VERSION: latest + TEST_TYPE: unpriv + run: ./scripts/ci-docker-run + rpmbuild-rocky9: - runs-on: ubuntu-22.04 name: rpmbuild-rocky9 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 # fetch tags as checkout@v2 doesn't do that by default @@ -240,6 +263,52 @@ jobs: TEST_TYPE: unpriv run: ./scripts/ci-docker-run + rpmbuild-opensuse-leap: + name: rpmbuild-opensuse-leap + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + # fetch tags as checkout@v2 doesn't do that by default + - run: git fetch --prune --unshallow --tags --force + + - name: Build and test rpm under docker + env: + OS_TYPE: opensuse/leap + OS_VERSION: latest + GO_ARCH: linux-amd64 + run: ./scripts/ci-docker-run + + - name: Install and test unprivileged for openSUSE leap + env: + OS_TYPE: opensuse/leap + OS_VERSION: latest + INS_OPTS: -o -d suse15 + TEST_TYPE: unpriv + run: ./scripts/ci-docker-run + + rpmbuild-opensuse-tumbleweed: + name: rpmbuild-opensuse-tumbleweed + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + # fetch tags as checkout@v2 doesn't do that by default + - run: git fetch --prune --unshallow --tags --force + + - name: Build and test rpm under docker + env: + OS_TYPE: opensuse/tumbleweed + OS_VERSION: latest + GO_ARCH: linux-amd64 + run: ./scripts/ci-docker-run + + - name: Install and test unprivileged for openSUSE tumbleweed + env: + OS_TYPE: opensuse/tumbleweed + OS_VERSION: latest + INS_OPTS: -o -d opensuse-tumbleweed + TEST_TYPE: unpriv + run: ./scripts/ci-docker-run + short_unit_tests: name: short_unit_tests runs-on: ubuntu-22.04 diff --git a/CHANGELOG.md b/CHANGELOG.md index 044c17b4bc..955a07b80a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,11 @@ For older changes see the [archived Singularity change log](https://github.com/a (`$HOME/.apptainer/docker-config.json`). The commands `pull`, `push`, `run`, `exec`, `shell` and `instance start` can now also be passed a `--authfile ` option, to read OCI registry credentials from this custom file. -- Add support for libsubid +- Add support for libsubid. +- Expand the build instructions for squashfuse and apptainer packaging to + include the libraries needed for maximum support of compression algorithms + by squashfuse_ll. +- Add automated tests for OpenSUSE Leap and Tumbleweed and Debian Bookworm. ## Changes for v1.3.x diff --git a/INSTALL.md b/INSTALL.md index 879cdd4801..b0b493e491 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -36,7 +36,7 @@ Including for libsubid support (requires at least Ubuntu Noble or Debian Bookwor sudo apt-get install -y libsubid-dev ``` -On RHEL or its derivatives: +On RHEL or its derivatives or Fedora: ```sh # Install basic tools for compiling @@ -52,11 +52,11 @@ sudo dnf install -y \ wget git ``` -Including for libsubid support, use --enablerepo=devel for el8 and el9 but not +For including libsubid support, use --enablerepo=devel for el8 and el9 but not for fedora: ```sh -sudo dnf --enablerepo=devel install shadow-utils-subid-devel +sudo dnf --enablerepo=devel install -y shadow-utils-subid-devel ``` On SLE/openSUSE @@ -67,14 +67,20 @@ sudo zypper install -y \ libseccomp-devel \ libuuid-devel \ openssl-devel \ + squashfs fakeroot \ cryptsetup sysuser-tools \ - gcc go + wget git go +# Install diffutils before devel tools to avoid busybox-diffutils on Tumbleweed +sudo zypper install -y diffutils +# Install basic tools for compiling +# --replacefiles is needed to avoid pam conflict on Tumbleweed +sudo zypper install -y --replacefiles -t pattern devel_basis ``` For libsubid support (requires openSUSE Tumbleweed): ```sh -sudo zypper -y install libsubid5 libsubid-devel +sudo zypper install -y libsubid-devel ``` ## Install Go @@ -166,7 +172,7 @@ make sudo make install ``` -And that's it! Now you can check your Apptainer version by running: +Now you can check your Apptainer version by running: ```sh apptainer --version @@ -187,23 +193,30 @@ See the output of `./mconfig -h` for available options. ## Compiling dependent FUSE-based packages -In many cases Apptainer uses FUSE-based packages in order to mount -filesystems. Very often these packages are available in Linux -distributions but out of date for Apptainer needs, so they need to be +In most cases Apptainer uses FUSE-based packages in order to mount +filesystems. Often some of these packages are available in Linux +distributions but are out of date for Apptainer needs, so they need to be compiled and installed with Apptainer following these directions. First, make sure that additional required packages are installed. On Debian: ```sh -sudo apt-get install -y autoconf automake libtool pkg-config libfuse3-dev zlib1g-dev +sudo apt-get install -y autoconf automake libtool pkg-config libfuse3-dev \ + zlib1g-dev liblzo2-dev liblz4-dev liblzma-dev libzstd-dev ``` On RHEL or derivatives: ```sh -sudo dnf install -y autoconf automake libtool pkgconfig fuse3-devel zlib-devel +sudo dnf install -y fuse3-devel lzo-devel lz4-devel ``` +On SLE/openSUSE: + +```sh +sudo zypper install -y gzip fuse3-devel lzo-devel liblz4-devel \ + xz-devel libzstd-devel + To download the source code from the top level of the Apptainer source tree do: @@ -269,10 +282,10 @@ sudo sysctl -p /etc/sysctl.d/90-disable-userns-restrictions.conf ## Building & Installing from RPM -On a RHEL / Fedora machine you can build an Apptainer into rpm +On a RHEL / Fedora /SUSE machine you can build an Apptainer into rpm packages, and install it from them. This is useful if you need to install Apptainer across multiple machines, or wish to manage all software via -`dnf`. +`dnf` (or `zypper` for SUSE). To build the rpms, in addition to the [system dependencies](#install-system-dependencies) @@ -284,6 +297,12 @@ also install these extra packages: sudo dnf install -y rpm-build golang ``` +or on SLE/openSUSE: + +```sh +sudo zypper install -y rpm-build binutils-gold +``` + The rpm build will use the OS distribution or EPEL version of Go, or it will use a different installation of Go, whichever is first in $PATH. If the first `go` found in $PATH is too old, diff --git a/dist/debian/control b/dist/debian/control index 289e817cf1..36c74c2976 100644 --- a/dist/debian/control +++ b/dist/debian/control @@ -21,6 +21,9 @@ Build-Depends: pkg-config, libfuse3-dev, zlib1g-dev, + liblzo2-dev, + liblzma-dev, + libzstd-dev, dh-apparmor Standards-Version: 3.9.8 Homepage: http://apptainer.org diff --git a/dist/rpm/apptainer.spec.in b/dist/rpm/apptainer.spec.in index 00eb69ab6d..cefaa39ac5 100644 --- a/dist/rpm/apptainer.spec.in +++ b/dist/rpm/apptainer.spec.in @@ -106,12 +106,14 @@ Provides: bundled(fuse-overlayfs) = %{fuse_overlayfs_version} %if "%{_target_vendor}" == "suse" BuildRequires: binutils-gold BuildRequires: go +BuildRequires: liblz4-devel %if 0%{?suse_version} > 1600 BuildRequires: libsubid-devel %endif %else BuildRequires: shadow-utils-subid-devel BuildRequires: golang +BuildRequires: lz4-devel %endif BuildRequires: git BuildRequires: gcc @@ -125,6 +127,9 @@ BuildRequires: automake BuildRequires: libtool BuildRequires: pkgconfig BuildRequires: zlib-devel +BuildRequires: lzo-devel +BuildRequires: xz-devel +BuildRequires: libzstd-devel %endif %if "%{_target_vendor}" == "suse" Requires: squashfs diff --git a/pkg/image/squashfs.go b/pkg/image/squashfs.go index 755791970e..48e73b3527 100644 --- a/pkg/image/squashfs.go +++ b/pkg/image/squashfs.go @@ -26,6 +26,7 @@ const ( squashfsLzoComp = 3 squashfsXzComp = 4 squashfsLz4Comp = 5 + squashfsZstdComp = 6 ) // this represents the superblock of a v4 squashfs image @@ -94,6 +95,8 @@ func CheckSquashfsHeader(b []byte) (uint64, error) { compressionType = "lzo" case squashfsXzComp: compressionType = "xz" + case squashfsZstdComp: + compressionType = "zstd" default: return 0, fmt.Errorf("corrupted image: unknown compression algorithm value %d", sinfo.Compression) } @@ -124,6 +127,8 @@ func GetSquashfsComp(b []byte) (string, error) { compType = "lzo" case squashfsXzComp: compType = "xz" + case squashfsZstdComp: + compType = "zstd" } return compType, nil } else if sb.Major < 4 { diff --git a/scripts/ci-deb-build-test b/scripts/ci-deb-build-test index 6c9b09daf1..a14a41b1f5 100755 --- a/scripts/ci-deb-build-test +++ b/scripts/ci-deb-build-test @@ -36,11 +36,13 @@ apt-get install -y \ dh-apparmor # for squashfuse_ll build -apt-get install -y autoconf automake libtool pkg-config libfuse3-dev zlib1g-dev +apt-get install -y autoconf automake libtool pkg-config libfuse3-dev \ + zlib1g-dev liblzo2-dev liblz4-dev liblzma-dev libzstd-dev -# for libsubid support in Ubuntu 24.04 -if [[ $OS_NAME = "Ubuntu" ]] && [ $OS_MAJOR -gt 23 ]; then - apt-get install -y libsubid4 libsubid-dev +# for libsubid support in Ubuntu 24.04+ or Debian 12+ +if { [ $OS_NAME = "Ubuntu" ] && [ $OS_MAJOR -ge 24 ]; } || \ + { [ $OS_NAME = "Debian" ] && [ $OS_MAJOR -ge 12 ]; }; then + apt-get install -y libsubid-dev fi # move source code down a level because debuild writes into parent dir diff --git a/scripts/ci-docker-run b/scripts/ci-docker-run index 4faa92cfab..951204dec0 100755 --- a/scripts/ci-docker-run +++ b/scripts/ci-docker-run @@ -26,7 +26,7 @@ CONTAINER_VERS="${CONTAINER_VERS:-$DOCKER_HUB_URI}" docker run --privileged --network=host -v "$(pwd):/build:rw" \ -e OS_TYPE=$OS_TYPE -e GO_ARCH=$GO_ARCH -e CONTAINER_VERS="$CONTAINER_VERS" \ - -e HIDE_DIST=$HIDE_DIST \ + -e HIDE_DIST=$HIDE_DIST -e INS_OPTS="$INS_OPTS" \ --name "$DOCKER_CONTAINER_NAME" "$DOCKER_HUB_URI" /bin/bash -exc \ "cd /build && scripts/ci-${TEST_TYPE:-$PKGTYPE-build}-test" diff --git a/scripts/ci-rpm-build-test b/scripts/ci-rpm-build-test index c367fdefc6..90c5b421d4 100755 --- a/scripts/ci-rpm-build-test +++ b/scripts/ci-rpm-build-test @@ -7,17 +7,31 @@ # this script runs as root under docker --privileged # install dependencies -dnf install -y rpm-build make yum-utils gcc binutils util-linux-ng which git -dnf install -y libseccomp-devel cryptsetup -dnf --enablerepo=devel install -y shadow-utils-subid-devel -if [ $OS_TYPE != fedora ]; then - dnf install -y epel-release +if [[ $OS_TYPE == *suse* ]]; then + zypper install -y libseccomp-devel libuuid-devel openssl-devel \ + squashfs fakeroot cryptsetup sysuser-tools wget git go + zypper install -y diffutils + zypper install -y --replacefiles -t pattern devel_basis + if [[ $OS_TYPE == *tumbleweed* ]]; then + zypper install -y libsubid-devel + fi + zypper install -y gzip fuse3-devel lzo-devel liblz4-devel \ + xz-devel libzstd-devel + zypper install -y rpm-build binutils-gold + zypper install -y sudo +else + dnf groupinstall -y 'Development Tools' + if [ $OS_TYPE != fedora ]; then + dnf install -y epel-release + fi + dnf install -y libseccomp-devel squashfs-tools fakeroot cryptsetup wget git + dnf --enablerepo=devel install -y shadow-utils-subid-devel + dnf install -y golang + dnf install -y fuse3-devel lzo-devel lz4-devel + dnf install -y sudo fi -dnf install -y golang fakeroot -dnf install -y autoconf automake libtool pkgconfig fuse3-devel zlib-devel # switch to an unprivileged user with sudo privileges -dnf install -y sudo useradd -u 1000 --create-home -s /bin/bash testuser echo "Defaults:testuser env_keep=DOCKER_HOST" >>/etc/sudoers echo "testuser ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers @@ -48,7 +62,11 @@ su testuser -c ' echo "%dist %{nil}" >$HOME/.rpmmacros fi make -C builddir rpm - sudo dnf install -y $HOME/rpmbuild/RPMS/*/*.rpm + if [[ '$OS_TYPE' == *suse* ]]; then + sudo zypper install -y --allow-unsigned-rpm $HOME/rpmbuild/RPMS/*/*.rpm + else + sudo dnf install -y $HOME/rpmbuild/RPMS/*/*.rpm + fi BLD="$(echo $HOME/rpmbuild/BUILD/apptainer-*)" export GOPATH=$BLD/gopath PATH=$GOPATH/bin:$PATH diff --git a/scripts/ci-unpriv-test b/scripts/ci-unpriv-test index 7a4ced8046..5f77770eaa 100755 --- a/scripts/ci-unpriv-test +++ b/scripts/ci-unpriv-test @@ -10,7 +10,7 @@ if [ "$OS_TYPE" = debian ] || [ "$OS_TYPE" = ubuntu ]; then apt-get update apt-get install -y curl rpm2cpio cpio e2fsprogs tzdata -elif [ "$OS_TYPE" = "opensuse/leap" ] || [ "$OS_TYPE" = "opensuse/tumbleweed" ]; then +elif [[ "$OS_TYPE" == *suse* ]]; then zypper install -y e2fsprogs curl cpio util-linux timezone else dnf install -y e2fsprogs cpio @@ -26,7 +26,7 @@ su testuser -c ' set -x set -e rm -rf ins image*.sif overlay.img - tools/install-unprivileged.sh -e -v apptainer-[1-9]*.$(arch).rpm ins + tools/install-unprivileged.sh -e -v apptainer-[1-9]*.$(arch).rpm '"$INS_OPTS"' ins ( echo Bootstrap: docker echo From: '"$CONTAINER_VERS"' diff --git a/tools/install-unprivileged.sh b/tools/install-unprivileged.sh index bdbcd1ff30..8add4c10a5 100755 --- a/tools/install-unprivileged.sh +++ b/tools/install-unprivileged.sh @@ -111,12 +111,12 @@ case "$DIST" in ubuntu20*|debian10*|debian11*) DIST=el8;; ubuntu*|debian1*|debian) DIST=el9;; el*|fc*);; - opensuse-tumbleweed) + suse20*|opensuse-tumbleweed) if $NOOPENSUSE; then DIST=el9 fi ;; - suse15) + suse15|opensuse-leap) if $NOOPENSUSE; then DIST=el8 fi @@ -160,16 +160,16 @@ case $1 in OSREPOURL="https://dl.fedoraproject.org/pub/fedora/linux/updates/$FC/Everything/$ARCH/Packages/" EPELREPOURL="https://dl.fedoraproject.org/pub/fedora/linux/updates$TESTING/$FC/Everything/$ARCH/Packages/" ;; - opensuse-tumbleweed) + suse20*|opensuse-tumbleweed) OSREPOURL="https://download.opensuse.org/tumbleweed/repo/oss/$ARCH" EPELREPOURL="https://download.opensuse.org/repositories/network:/cluster/openSUSE_Tumbleweed/$ARCH" EXTRASREPOURL=$OSREPOURL OSSPLIT=false ;; - suse15) - OSREPOURL="https://download.opensuse.org/distribution/leap/15.4/repo/oss/$ARCH" - EPELREPOURL="https://download.opensuse.org/repositories/network:/cluster/15.4/$ARCH" - EXTRASREPOURL="https://download.opensuse.org/repositories/filesystems/15.4/$ARCH" + suse15|opensuse-leap) + OSREPOURL="https://download.opensuse.org/distribution/leap/15.6/repo/oss/$ARCH" + EPELREPOURL="https://download.opensuse.org/repositories/network:/cluster/15.6/$ARCH" + EXTRASREPOURL="https://download.opensuse.org/repositories/filesystems/15.6/$ARCH" OSSPLIT=false ;; *) fatal "$1 distribution not supported";; @@ -199,10 +199,10 @@ latesturl() # optimization: reuse last list if it hasn't changed LASTURL="$URL" LASTPKGS="$(curl $CURLOPTS "$URL")" - elif [ $RETRY -gt 0 ]; then + elif [ -z "$LASTPKGS" ]; then LASTPKGS="$(curl $CURLOPTS "$URL")" fi - typeset LATEST="$(echo "$LASTPKGS"|sed -e 's/.*href="//;s/".*//' -e 's/\.mirrorlist//' -e 's/\-32bit//' -e 's@^\.\/@@' |grep "^$2-[0-9].*$ARCH"|tail -1)" + typeset LATEST="$(echo "$LASTPKGS"|sed -e 's/.*href="//;s/".*//' -e 's/\.mirrorlist//' -e 's/\-32bit//' -e 's@^\.\/@@' |grep "^$2-[0-9].*$ARCH\.rpm"|tail -1)" if [ -n "$LATEST" ]; then RETRY=0 echo "$URL/$LATEST" @@ -241,9 +241,10 @@ if [[ "$VERSION" == *.rpm ]]; then # take third from last field separated by dots as the rpm dist # shellcheck disable=SC2001 RPMDIST="$(echo "$VERSION"|sed 's/.*\.\(.*\)\.[^.]*\..*$/\1/')" - # but if it starts with a number, the dist is missing so just try - # to use the default DIST (which happens if RPMDIST is empty) - if [[ "$RPMDIST" == [0-9]* ]]; then + # but if it starts with a number or contains a dash, the dist is + # missing so just try to use the default DIST (which happens if + # RPMDIST is empty) + if [[ "$RPMDIST" == [0-9]* ]] || [[ "$RPMDIST" == *-* ]]; then RPMDIST="" fi elif [ -z "$VERSION" ] || ! $NOOPENSUSE; then @@ -304,12 +305,16 @@ fi OSUTILS="" NEEDSFUSE2FS=true -if [ -f libexec/apptainer/bin/fuse2fs ]; then +LIBEXEC=libexec +if [[ "$DIST" == *suse* ]]; then + LIBEXEC=lib +fi +if [ -f $LIBEXEC/apptainer/bin/fuse2fs ]; then # apptainer-1.3.0 or newer NEEDSFUSE2FS=false fi EXTRASUTILS="" -if [ ! -f libexec/apptainer/bin/fuse-overlayfs ]; then +if [ ! -f $LIBEXEC/apptainer/bin/fuse-overlayfs ]; then # older than apptainer-1.3.0 EXTRASUTILS="fuse-overlayfs" fi @@ -330,26 +335,28 @@ if [ "$DIST" = el7 ]; then EPELUTILS="$EPELUTILS fuse2fs" fi elif [ "$DIST" = el8 ]; then - OSUTILS="$OSUTILS lzo squashfs-tools libzstd fuse3-libs libsepol bzip2-libs audit-libs libcap-ng libattr libacl pcre2 libxcrypt libselinux libsemanage shadow-utils-subid" + # lz4-libs are installed by default on rhel but needed when + # these binaries are used on suse + OSUTILS="$OSUTILS lzo squashfs-tools lz4-libs libzstd fuse3-libs libsepol bzip2-libs audit-libs libcap-ng libattr libacl pcre2 libxcrypt libselinux libsemanage shadow-utils-subid" if $NEEDSFUSE2FS; then OSUTILS="$OSUTILS fuse-libs e2fsprogs-libs e2fsprogs" fi -elif [ "$DIST" = "opensuse-tumbleweed" ]; then - OSUTILS="$OSUTILS squashfs liblzo2-2 libzstd1 libfuse3-3 $EXTRASUTILS $EPELUTILS" +elif [[ "$DIST" == suse20* ]] || [ "$DIST" = "opensuse-tumbleweed" ]; then + OSUTILS="$OSUTILS squashfs liblzo2-2 liblz4-1 libzstd1 libfuse3-3 $EXTRASUTILS $EPELUTILS" if $NEEDSFUSE2FS; then OSUTILS="$OSUTILS fuse2fs" fi EXTRASUTILS="" EPELUTILS="" -elif [ "$DIST" = "suse15" ]; then - OSUTILS="$OSUTILS squashfs liblzo2-2 libzstd1 libfuse3-3 $EPELUTILS" +elif [ "$DIST" = "suse15" ] || [ "$DIST" = "opensuse-leap" ]; then + OSUTILS="$OSUTILS squashfs liblzo2-2 liblz4-1 libzstd1 libfuse3-3 $EPELUTILS" if $NEEDSFUSE2FS; then EXTRASUTILS="$EXTRASUTILS libext2fs2 libfuse2 fuse2fs" fi EPELUTILS="" else # el9 & fc* - OSUTILS="$OSUTILS lzo squashfs-tools libzstd libsepol bzip2-libs audit-libs libcap-ng libattr libacl pcre2 libxcrypt libselinux libsemanage shadow-utils-subid" + OSUTILS="$OSUTILS lzo squashfs-tools lz4-libs libzstd libsepol bzip2-libs audit-libs libcap-ng libattr libacl pcre2 libxcrypt libselinux libsemanage shadow-utils-subid" if $NEEDSFUSE2FS; then OSUTILS="$OSUTILS fuse-libs e2fsprogs-libs e2fsprogs" fi