remove multi-arch.bash but use skopeo to get images directly

Signed-off-by: Xiaodong Liu <liuxiaodong@loongson.cn>

.travis.yml

@@ -60,8 +60,12 @@ services:
   - docker
 
 before_install:
+  - sudo su - root -c 'echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_Unstable/  /" | sudo cat > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list'
+  - wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_Unstable/Release.key -O- | sudo apt-key add -
   - sudo apt-get -qq update
-  - sudo apt-get install -y libseccomp-dev
+  - sudo apt-get install -y libseccomp-dev skopeo
+  - sudo curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/v0.4.5/umoci.amd64
+  - sudo chmod a+x /usr/local/bin/umoci
   - GO111MODULE=off go get -u golang.org/x/lint/golint
   - GO111MODULE=off go get -u github.com/vbatts/git-validation
   - env | grep TRAVIS_

Dockerfile

@@ -67,31 +67,9 @@ RUN mkdir -p /usr/src/criu \
     && cd - \
     && rm -rf /usr/src/criu
 
-# install skopeo
-RUN echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_Unstable/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list \
-    && wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/Debian_Unstable/Release.key -O- | sudo apt-key add - \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends skopeo \
-    && rm -rf /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list \
-    && apt-get clean \
-    && rm -rf /var/cache/apt /var/lib/apt/lists/*;
-
-# install umoci
-RUN curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/v0.4.5/umoci.amd64 \
-    && chmod +x /usr/local/bin/umoci
-
 COPY script/tmpmount /
 WORKDIR /go/src/github.com/opencontainers/runc
 ENTRYPOINT ["/tmpmount"]
 
-# setup a playground for us to spawn containers in
-COPY tests/integration/multi-arch.bash tests/integration/
-ENV ROOTFS /busybox
-RUN mkdir -p "${ROOTFS}"
-RUN . tests/integration/multi-arch.bash \
-    && curl -fsSL `get_busybox` | tar xfJC - "${ROOTFS}"
-
-ENV DEBIAN_ROOTFS /debian
-RUN mkdir -p "${DEBIAN_ROOTFS}"
-RUN . tests/integration/multi-arch.bash \
-    && get_and_extract_debian "$DEBIAN_ROOTFS"
+ADD tests/integration/testdata/busybox.tar /busybox
+ADD tests/integration/testdata/debian.tar /debian

Makefile

@@ -1,5 +1,6 @@
 CONTAINER_ENGINE := docker
 GO := go
+export CONTAINER_ENGINE
 
 PREFIX ?= $(DESTDIR)/usr/local
 BINDIR := $(PREFIX)/sbin
@@ -58,9 +59,13 @@ lint:
 man:
 	man/md2man-all.sh
 
-runcimage:
+runcimage: | .download_test_images
 	$(CONTAINER_ENGINE) build $(CONTAINER_ENGINE_BUILD_FLAGS) -t $(RUNC_IMAGE) .
 
+.download_test_images:
+	make -C tests/integration/testdata/
+	ln -sf tests/integration/testdata/ "$@"
+
 test: unittest integration rootlessintegration
 
 localtest: localunittest localintegration localrootlessintegration
@@ -72,7 +77,7 @@ unittest: runcimage
 		-v $(CURDIR):/go/src/$(PROJECT) \
 		$(RUNC_IMAGE) make localunittest TESTFLAGS=$(TESTFLAGS)
 
-localunittest: all
+localunittest: all | .download_test_images 
 	$(GO) test $(MOD_VENDOR) -timeout 3m -tags "$(BUILDTAGS)" $(TESTFLAGS) -v ./...
 
 integration: runcimage
@@ -82,7 +87,7 @@ integration: runcimage
 		-v $(CURDIR):/go/src/$(PROJECT) \
 		$(RUNC_IMAGE) make localintegration TESTPATH=$(TESTPATH)
 
-localintegration: all
+localintegration: all | .download_test_images
 	bats -t tests/integration$(TESTPATH)
 
 rootlessintegration: runcimage
@@ -92,7 +97,7 @@ rootlessintegration: runcimage
 		-e ROOTLESS_TESTPATH \
 		$(RUNC_IMAGE) make localrootlessintegration
 
-localrootlessintegration: all
+localrootlessintegration: all | .download_test_images
 	tests/rootless.sh
 
 shell: runcimage
@@ -116,6 +121,8 @@ clean:
 	rm -f contrib/cmd/recvtty/recvtty
 	rm -rf release
 	rm -rf man/man8
+	rm -f .download_test_images
+	make -C tests/integration/testdata/ clean
 
 validate:
 	script/validate-gofmt

Vagrantfile.centos7

@@ -51,9 +51,9 @@ EOF
     # Add a user for rootless tests
     useradd -u2000 -m -d/home/rootless -s/bin/bash rootless
 
+    make -C /vagrant .download_test_images
     # Add busybox for libcontainer/integration tests
-    . /vagrant/tests/integration/multi-arch.bash \
-        && mkdir /busybox \
-        && curl -fsSL $(get_busybox) | tar xfJC - /busybox
+    mkdir /busybox
+    tar --exclude './dev/*' -C /busybox -xf /vagrant/tests/integration/testdata/busybox.tar
   SHELL
 end

Vagrantfile.fedora32

@@ -40,11 +40,12 @@ EOF
     curl -o /usr/local/bin/umoci -fsSL https://github.com/opencontainers/umoci/releases/download/v0.4.5/umoci.amd64
     chmod +x /usr/local/bin/umoci
 
-    # Add busybox for libcontainer/integration tests
-    . /vagrant/tests/integration/multi-arch.bash \
-        && mkdir /busybox /debian \
-        && curl -fsSL $(get_busybox) | tar xfJC - /busybox \
-        && get_and_extract_debian /debian
+    make -C /vagrant .download_test_images
+    # Add busybox and debian for libcontainer/integration tests
+    for dist in busybox debian; do
+        mkdir /$dist
+        tar --exclude './dev/*' -C /${dist} -xf /vagrant/tests/integration/testdata/${dist}.tar
+    done
 
     # Delegate cgroup v2 controllers to rootless user via --systemd-cgroup
     mkdir -p /etc/systemd/system/user@.service.d

tests/integration/helpers.bash

@@ -3,8 +3,6 @@
 # Root directory of integration tests.
 INTEGRATION_ROOT=$(dirname "$(readlink -f "$BASH_SOURCE")")
 
-. ${INTEGRATION_ROOT}/multi-arch.bash
-
 RUNC="${INTEGRATION_ROOT}/../../runc"
 RECVTTY="${INTEGRATION_ROOT}/../../contrib/cmd/recvtty/recvtty"
 GOPATH="$(mktemp -d --tmpdir runc-integration-gopath.XXXXXX)"
@@ -13,15 +11,15 @@ GOPATH="$(mktemp -d --tmpdir runc-integration-gopath.XXXXXX)"
 TESTDATA="${INTEGRATION_ROOT}/testdata"
 
 # Busybox image
-BUSYBOX_IMAGE="$BATS_TMPDIR/busybox.tar"
+BUSYBOX_IMAGE="$TESTDATA/busybox.tar"
 BUSYBOX_BUNDLE="$BATS_TMPDIR/busyboxtest"
 
 # hello-world in tar format
-HELLO_FILE=`get_hello`
-HELLO_IMAGE="$TESTDATA/$HELLO_FILE"
+HELLO_IMAGE="$TESTDATA/hello-world.tar"
 HELLO_BUNDLE="$BATS_TMPDIR/hello-world"
 
 # debian image
+DEBIAN_IMAGE="$TESTDATA/debian.tar"
 DEBIAN_BUNDLE="$BATS_TMPDIR/debiantest"
 
 # CRIU PATH
@@ -420,11 +418,9 @@ function setup_busybox() {
 	setup_recvtty
 	run mkdir "$BUSYBOX_BUNDLE"
 	run mkdir "$BUSYBOX_BUNDLE"/rootfs
-	if [ -e "/testdata/busybox.tar" ]; then
-		BUSYBOX_IMAGE="/testdata/busybox.tar"
-	fi
 	if [ ! -e $BUSYBOX_IMAGE ]; then
-		curl -o $BUSYBOX_IMAGE -sSL `get_busybox`
+		echo "busybox image does not exist"
+		exit 1
 	fi
 	tar --exclude './dev/*' -C "$BUSYBOX_BUNDLE"/rootfs -xf "$BUSYBOX_IMAGE"
 	cd "$BUSYBOX_BUNDLE"
@@ -435,6 +431,10 @@ function setup_hello() {
 	setup_recvtty
 	run mkdir "$HELLO_BUNDLE"
 	run mkdir "$HELLO_BUNDLE"/rootfs
+	if [ ! -e "$HELLO_IMAGE" ]; then
+		echo "hello-world image does not exist"
+		exit 2
+	fi
 	tar --exclude './dev/*' -C "$HELLO_BUNDLE"/rootfs -xf "$HELLO_IMAGE"
 	cd "$HELLO_BUNDLE"
 	runc_spec
@@ -449,17 +449,15 @@ function setup_debian() {
 
 	setup_recvtty
 	run mkdir "$DEBIAN_BUNDLE"
-
-	if [ ! -d "$DEBIAN_ROOTFS/rootfs" ]; then
-		get_and_extract_debian "$DEBIAN_BUNDLE"
+	run mkdir "$DEBIAN_BUNDLE"/rootfs
+	if [ ! -e "$DEBIAN_IMAGE" ]; then
+		echo "debian image does not exist"
+		exit 3
 	fi
-
-	# Use the cached version
-	if [ ! -d "$DEBIAN_BUNDLE/rootfs" ]; then
-		cp -r "$DEBIAN_ROOTFS"/* "$DEBIAN_BUNDLE/"
-	fi
-
+	tar --exclude './dev/*' -C "$DEBIAN_BUNDLE"/rootfs -xf "$DEBIAN_IMAGE"
 	cd "$DEBIAN_BUNDLE"
+	runc_spec
+	update_config '.root.readonly |= false' $DEBIAN_BUNDLE
 }
 
 function teardown_running_container() {

tests/integration/testdata/Makefile

@@ -0,0 +1,9 @@
+TEST_IMAGES := debian busybox hello-world
+CONTAINER_ENGINE ?= docker
+
+download_images:
+	$(foreach image, $(TEST_IMAGES),$(shell ./download_oci_image.sh "$(image)"))
+clean:
+	-rm *.tar
+
+.PHONY: download_images clean

tests/integration/testdata/download_oci_image.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+IMAGE="$1"
+GOARCH=$(go env GOARCH)
+# according to documents: https://github.com/docker-library/official-images#architectures-other-than-amd64
+case ${GOARCH} in
+	arm)
+		skopeo copy -q docker://$(arch)/$IMAGE:latest "oci:$IMAGE:latest"
+	;;
+
+	*)
+		skopeo copy -q docker://$GOARCH/$IMAGE:latest "oci:$IMAGE:latest"
+	;;
+esac
+
+mkdir "$IMAGE"-tmp
+umoci unpack --image "$IMAGE" --rootless "$IMAGE"-tmp
+cd "$IMAGE"-tmp/rootfs
+tar -cf ../../"$IMAGE".tar .
+cd ../../
+rm -rf "$IMAGE"-tmp
+
+# add tarball read permission
+chmod a+r "$IMAGE".tar