fix(umoci)!: umoci's copy buffer set to 1 MiB (project-stacker#531)

* fix(umoci)!: umoci's copy buffer set to 1 MiB

project-stacker/umoci@89a7ea1

Use updated umoci (project-stacker's fork) to use a 1 MiB copy buffer.
This change brings stacker inline with compression parameters used in
containers/image libraries and makes them compatible.

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

* test(gzip): add unit tests for gzip

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

---------

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

Makefile

@@ -31,6 +31,8 @@ HACK_D := $(TOP_LEVEL)/hack
 TOOLS_D := $(HACK_D)/tools
 REGCLIENT := $(TOOLS_D)/bin/regctl
 REGCLIENT_VERSION := v0.5.1
+export SKOPEO = $(TOOLS_D)/bin/skopeo
+export SKOPEO_VERSION = 1.9.3
 # OCI registry
 ZOT := $(TOOLS_D)/bin/zot
 ZOT_VERSION := v2.0.0-rc6
@@ -99,6 +101,19 @@ $(REGCLIENT):
 $(ZOT):
 	$(call dlbin,$@,https://github.com/project-zot/zot/releases/download/$(ZOT_VERSION)/zot-linux-amd64-minimal)
 
+$(SKOPEO):
+	@mkdir -p "$(TOOLS_D)/bin"; \
+	tmpdir=$$(mktemp -d); \
+	cd $$tmpdir; \
+	git clone https://github.com/containers/skopeo.git; \
+	cd skopeo; \
+	git fetch --all --tags --prune; \
+	git checkout tags/v$(SKOPEO_VERSION) -b tag-$(SKOPEO_VERSION); \
+	make bin/skopeo; \
+	cp bin/skopeo $(SKOPEO); \
+	cd $(TOP_LEVEL); \
+	rm -rf $$tmpdir;
+
 TEST?=$(patsubst test/%.bats,%,$(wildcard test/*.bats))
 PRIVILEGE_LEVEL?=
 
@@ -108,7 +123,7 @@ PRIVILEGE_LEVEL?=
 check: lint test go-test
 
 .PHONY: test
-test: stacker $(REGCLIENT) $(ZOT)
+test: stacker $(REGCLIENT) $(SKOPEO) $(ZOT)
 	sudo -E PATH="$$PATH" \
 		LXC_BRANCH=$(LXC_BRANCH) \
 		LXC_CLONE_URL=$(LXC_CLONE_URL) \

go.mod

@@ -234,6 +234,6 @@ require (
 )
 
 replace (
-	github.com/opencontainers/umoci => github.com/project-stacker/umoci v0.0.0-20231019200834-3f97387412c4
+	github.com/opencontainers/umoci => github.com/project-stacker/umoci v0.0.0-20231025233220-c7bb24d57e98
 	stackerbuild.io/stacker-bom => github.com/project-stacker/stacker-bom v0.0.0-20230522080732-de2712897250
 )

go.sum

@@ -723,8 +723,8 @@ github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU
 github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0=
 github.com/project-stacker/stacker-bom v0.0.0-20230522080732-de2712897250 h1:5gSyDxGXisvvu+aMUq7WRxgq3phvdy9/1CM/TqUHLVQ=
 github.com/project-stacker/stacker-bom v0.0.0-20230522080732-de2712897250/go.mod h1:P0o0hINRm/kcAB0CRf/W9RMLBWWb2EzzhPysXipj3Cg=
-github.com/project-stacker/umoci v0.0.0-20231019200834-3f97387412c4 h1:mtCuBc3xMRcZQCPHbDsfKXkr3TJL3N4OPg+2tQnH55w=
-github.com/project-stacker/umoci v0.0.0-20231019200834-3f97387412c4/go.mod h1:XUXUpCpA/Y8aJWezK1i8o4WDR0Y/vhMcWg+FUNQkKMQ=
+github.com/project-stacker/umoci v0.0.0-20231025233220-c7bb24d57e98 h1:qfk5zDnZM5VA7Bn8uKHQlsfp1425n4bOth3nWnzBGns=
+github.com/project-stacker/umoci v0.0.0-20231025233220-c7bb24d57e98/go.mod h1:XUXUpCpA/Y8aJWezK1i8o4WDR0Y/vhMcWg+FUNQkKMQ=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=

install-build-deps.sh

@@ -3,12 +3,18 @@ set -o pipefail
 set -o errexit
 
 installdeps_fedora() {
-        sudo dnf install \
-        bats \
-        jq \
-        lxc-devel \
-        libcap-devel \
-        libacl-devel
+  sudo dnf install \
+    bats \
+    jq \
+    lxc-devel \
+    libcap-devel \
+    libacl-devel
+  # skopeo deps
+  sudo dnf install \
+    gpgme-devel \
+    libassuan-devel \
+    btrfs-progs-devel \
+    device-mapper-devel
     if ! command -v go 2>/dev/null; then
         sudo dnf install golang
         go version
@@ -35,9 +41,15 @@ installdeps_ubuntu() {
             lxc-utils \
             parallel \
             pkg-config \
-            skopeo \
             squashfs-tools \
             squashfuse
+    # skopeo deps
+    sudo apt -yy install \
+       libgpgme-dev \
+       libassuan-dev \
+       libbtrfs-dev \
+       libdevmapper-dev \
+       pkg-config
     if ! command -v go 2>/dev/null; then
         sudo apt -yy install golang-go
         go version

test/gzip.bats

@@ -0,0 +1,16 @@
+load helpers
+
+function setup() {
+    stacker_setup
+}
+
+function teardown() {
+    cleanup
+}
+
+@test "import various sizes" {
+  test_copy_buffer_size 512k tar
+  test_copy_buffer_size 2m tar
+  test_copy_buffer_size 512k tar.gz
+  test_copy_buffer_size 2m tar.gz
+}

test/helpers.bash

@@ -200,3 +200,60 @@ function zot_teardown {
   killall zot
   rm -f $TEST_TMPDIR/zot-config.json
 }
+
+function test_copy_buffer_size() {
+   local buffer_size=$1
+   local file_type=$2
+
+   # create a temporary dir
+   local tmpdir=$(mktemp -d "$BATS_TEST_TMPDIR"/copy${1:+-$1}.XXXXXX)
+   cd "$tmpdir"
+   if [ "$PRIVILEGE_LEVEL" = "priv" ]; then
+     return
+   fi
+
+   "${ROOT_DIR}/stacker" unpriv-setup
+   chown -R $SUDO_USER:$SUDO_USER .
+
+   mkdir folder1
+   truncate -s $buffer_size folder1/file1
+   if [ $file_type = "tar" ]
+   then
+     tar cvf test.$file_type folder1
+   elif [ $file_type = "tar.gz" ]
+   then
+     tar cvzf test.$file_type folder1
+   else
+    echo "unknown file type: $file_type"
+    exit 1
+   fi
+   cat > stacker.yaml <<EOF
+tar:
+  from:
+    type: tar
+    url: test.$file_type
+EOF
+  stacker build
+  cat oci/index.json | jq .
+  m1=$(cat oci/index.json | jq .manifests[0].digest | sed  's/sha256://' | tr -d \")
+  cat oci/blobs/sha256/"$m1" | jq .
+  l1=$(cat oci/blobs/sha256/"$m1" | jq .layers[0].digest | sed  's/sha256://' | tr -d \")
+  $SKOPEO --version
+  [[ "$($SKOPEO --version)" =~ "skopeo version ${SKOPEO_VERSION}" ]] || {
+    echo "$SKOPEO --version should be ${SKOPEO_VERSION}"
+    exit 1
+  }
+  $SKOPEO copy --format=oci oci:oci:tar containers-storage:test:tar
+  $SKOPEO copy --format=oci containers-storage:test:tar oci:oci:test
+  cat oci/index.json | jq .
+  m2=$(cat oci/index.json | jq .manifests[1].digest | sed  's/sha256://' | tr -d \")
+  cat oci/blobs/sha256/"$m2" | jq .
+  l2=$(cat oci/blobs/sha256/"$m2" | jq .layers[0].digest | sed  's/sha256://' | tr -d \")
+  echo "$l1"
+  echo "$l2"
+  [ "$l1" = "$l2" ]
+  stacker clean
+  rm -rf folder1
+  cd "$ROOT_DIR"
+  rm -rf "tmpdir"
+}