From a4fcf09b7ab5daa30b37705702ba2c2e30e8f9c5 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Tue, 28 Jul 2020 08:36:52 -0600 Subject: [PATCH] Reenable remote system tests podman-remote is in better shape now. Let's see what needs to be done to reenable remote system tests. - logs test: skip multilog, it doesn't work remote - diff test: use -l only when local, not with remote - many other tests: skip_if_remote, with 'FIXME: pending #xxxx' where xxxx is a filed issue. Unrelated: added new helper to skip_if_remote and _if_rootless, where we check if the source message includes "remote"/"rootless" and insert it if missing. This is a minor usability enhancement to make it easier to understand at-a-glance why a skip triggers. Signed-off-by: Ed Santiago --- Makefile | 10 ++++------ test/system/030-run.bats | 14 ++++++++++++++ test/system/035-logs.bats | 2 ++ test/system/050-stop.bats | 7 +++++-- test/system/055-rm.bats | 2 ++ test/system/070-build.bats | 20 ++++++++++++-------- test/system/110-history.bats | 2 ++ test/system/120-load.bats | 6 +++++- test/system/130-kill.bats | 2 ++ test/system/140-diff.bats | 13 ++++++++++--- test/system/160-volumes.bats | 6 ++++++ test/system/200-pod.bats | 17 ++++++++++++++--- test/system/220-healthcheck.bats | 1 + test/system/300-cli-parsing.bats | 2 ++ test/system/400-unprivileged-access.bats | 7 +++++++ test/system/410-selinux.bats | 1 + test/system/helpers.bash | 22 ++++++++++++++++++++-- 17 files changed, 109 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 08a3cddac8..70e4a49c7d 100644 --- a/Makefile +++ b/Makefile @@ -353,21 +353,19 @@ remotesystem: # Start podman server using tmp socket; loop-wait for it; # test podman-remote; kill server, clean up tmp socket file. # podman server spews copious unhelpful output; ignore it. - # FIXME FIXME FIXME: remove 'exit 0' after #6538 and #6539 are fixed - exit 0;\ rc=0;\ if timeout -v 1 true; then \ SOCK_FILE=$(shell mktemp --dry-run --tmpdir podman.XXXXXX);\ export PODMAN_SOCKET=unix:$$SOCK_FILE; \ - ./bin/podman system service --timeout=0 $$PODMAN_SOCKET &> $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) & \ + ./bin/podman system service --timeout=0 $$PODMAN_SOCKET > $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) 2>&1 & \ retry=5;\ - while [[ $$retry -ge 0 ]]; do\ + while [ $$retry -ge 0 ]; do\ echo Waiting for server...;\ sleep 1;\ - ./bin/podman-remote --url $$PODMAN_SOCKET info &>/dev/null && break;\ + ./bin/podman-remote --url $$PODMAN_SOCKET info >/dev/null 2>&1 && break;\ retry=$$(expr $$retry - 1);\ done;\ - if [[ $$retry -lt 0 ]]; then\ + if [ $$retry -lt 0 ]; then\ echo "Error: ./bin/podman system service did not come up on $$SOCK_FILE" >&2;\ exit 1;\ fi;\ diff --git a/test/system/030-run.bats b/test/system/030-run.bats index b30c1103b9..e93a2efe25 100644 --- a/test/system/030-run.bats +++ b/test/system/030-run.bats @@ -96,6 +96,8 @@ echo $rand | 0 | $rand # Believe it or not, 'sh -c' resulted in different behavior run_podman 0 run --rm $IMAGE sh -c /bin/true run_podman 1 run --rm $IMAGE sh -c /bin/false + + if is_remote; then sleep 2;fi # FIXME: pending #7119 } @test "podman run --name" { @@ -202,6 +204,8 @@ echo $rand | 0 | $rand } @test "podman run docker-archive" { + skip_if_remote "FIXME: pending #7116" + # Create an image that, when run, outputs a random magic string expect=$(random_string 20) run_podman run --name myc --entrypoint="[\"/bin/echo\",\"$expect\"]" $IMAGE @@ -247,6 +251,8 @@ echo $rand | 0 | $rand # symptom only manifests on a fedora container image -- we have no # reproducer on alpine. Checking directory ownership is good enough. @test "podman run : user namespace preserved root ownership" { + skip_if_remote "FIXME: pending #7195" + for priv in "" "--privileged"; do for user in "--user=0" "--user=100"; do for keepid in "" "--userns=keep-id"; do @@ -260,10 +266,14 @@ echo $rand | 0 | $rand done done done + + if is_remote; then sleep 2;fi # FIXME: pending #7119 } # #6829 : add username to /etc/passwd inside container if --userns=keep-id @test "podman run : add username to /etc/passwd if --userns=keep-id" { + skip_if_remote "FIXME: pending #7195" + # Default: always run as root run_podman run --rm $IMAGE id -un is "$output" "root" "id -un on regular container" @@ -282,10 +292,14 @@ echo $rand | 0 | $rand run_podman run --rm --privileged --userns=keep-id --user=0 $IMAGE id -un remove_same_dev_warning # grumble is "$output" "root" "--user=0 overrides keep-id" + + if is_remote; then sleep 2;fi # FIXME: pending #7119 } # #6991 : /etc/passwd is modifiable @test "podman run : --userns=keep-id: passwd file is modifiable" { + skip_if_remote "FIXME: pending #7195" + run_podman run -d --userns=keep-id $IMAGE sh -c 'while ! test -e /stop; do sleep 0.1; done' cid="$output" diff --git a/test/system/035-logs.bats b/test/system/035-logs.bats index 055865c8d1..cbb2091e57 100644 --- a/test/system/035-logs.bats +++ b/test/system/035-logs.bats @@ -25,6 +25,8 @@ load helpers } @test "podman logs - multi" { + skip_if_remote "logs does not support multiple containers when run remotely" + # Simple helper to make the container starts, below, easier to read local -a cid doit() { diff --git a/test/system/050-stop.bats b/test/system/050-stop.bats index 093606ecea..f604ea2e28 100644 --- a/test/system/050-stop.bats +++ b/test/system/050-stop.bats @@ -12,9 +12,12 @@ load helpers run_podman stop $cid t1=$SECONDS - # Confirm that container is stopped + # Confirm that container is stopped. Podman-remote unfortunately + # cannot tell the difference between "stopped" and "exited", and + # spits them out interchangeably, so we need to recognize either. run_podman inspect --format '{{.State.Status}} {{.State.ExitCode}}' $cid - is "$output" "exited \+137" "Status and exit code of stopped container" + is "$output" "\\(stopped\|exited\\) \+137" \ + "Status and exit code of stopped container" # The initial SIGTERM is ignored, so this operation should take # exactly 10 seconds. Give it some leeway. diff --git a/test/system/055-rm.bats b/test/system/055-rm.bats index c8475c3e9d..478ba0f20a 100644 --- a/test/system/055-rm.bats +++ b/test/system/055-rm.bats @@ -44,6 +44,8 @@ load helpers # # See https://github.com/containers/podman/issues/3795 @test "podman rm -f" { + skip_if_remote "FIXME: pending #7117" + rand=$(random_string 30) ( sleep 3; run_podman rm -f $rand ) & run_podman 137 run --name $rand $IMAGE sleep 30 diff --git a/test/system/070-build.bats b/test/system/070-build.bats index a69d32a2f8..481e1759b1 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -6,9 +6,7 @@ load helpers @test "podman build - basic test" { - if is_remote && is_rootless; then - skip "unreliable with podman-remote and rootless; #2972" - fi + skip_if_remote "FIXME: pending #7136" rand_filename=$(random_string 20) rand_content=$(random_string 50) @@ -34,6 +32,7 @@ EOF # Regression from v1.5.0. This test passes fine in v1.5.0, fails in 1.6 @test "podman build - cache (#3920)" { + skip_if_remote "FIXME: pending #7136" if is_remote && is_rootless; then skip "unreliable with podman-remote and rootless; #2972" fi @@ -81,6 +80,8 @@ EOF } @test "podman build - URLs" { + skip_if_remote "FIXME: pending #7137" + tmpdir=$PODMAN_TMPDIR/build-test mkdir -p $tmpdir @@ -90,6 +91,7 @@ ADD https://github.com/containers/podman/blob/master/README.md /tmp/ EOF run_podman build -t add_url $tmpdir run_podman run --rm add_url stat /tmp/README.md + if is_remote; then sleep 2;fi # FIXME: pending #7119 run_podman rmi -f add_url # Now test COPY. That should fail. @@ -100,6 +102,8 @@ EOF @test "podman build - workdir, cmd, env, label" { + skip_if_remote "FIXME: pending #7137" + tmpdir=$PODMAN_TMPDIR/build-test mkdir -p $tmpdir @@ -234,19 +238,19 @@ Labels.$label_name | $label_value } @test "podman build - stdin test" { - if is_remote && is_rootless; then - skip "unreliable with podman-remote and rootless; #2972" - fi + skip_if_remote "FIXME: pending #7136" - # Random workdir, and multiple random strings to verify command & env + # Random workdir, and random string to verify build output workdir=/$(random_string 10) + random_echo=$(random_string 15) PODMAN_TIMEOUT=240 run_podman build -t build_test - << EOF FROM $IMAGE RUN mkdir $workdir WORKDIR $workdir -RUN /bin/echo 'Test' +RUN /bin/echo $random_echo EOF is "$output" ".*STEP 5: COMMIT" "COMMIT seen in log" + is "$output" ".*STEP .: RUN /bin/echo $random_echo" run_podman run --rm build_test pwd is "$output" "$workdir" "pwd command in container" diff --git a/test/system/110-history.bats b/test/system/110-history.bats index 5dc221d610..b83e90fe4a 100644 --- a/test/system/110-history.bats +++ b/test/system/110-history.bats @@ -3,6 +3,8 @@ load helpers @test "podman history - basic tests" { + skip_if_remote "FIXME: pending #7122" + tests=" | .*[0-9a-f]\\\{12\\\} .* CMD .* LABEL --format '{{.ID}} {{.Created}}' | .*[0-9a-f]\\\{12\\\} .* ago diff --git a/test/system/120-load.bats b/test/system/120-load.bats index afa5ab4732..ccfbc51ca0 100644 --- a/test/system/120-load.bats +++ b/test/system/120-load.bats @@ -28,6 +28,8 @@ verify_iid_and_name() { @test "podman load - by image ID" { + skip_if_remote "FIXME: pending #7123" + # FIXME: how to build a simple archive instead? get_iid_and_name @@ -74,7 +76,9 @@ verify_iid_and_name() { verify_iid_and_name $img_name } -@test "podman load - NAME and NAME:TAG arguments work (requires: #2674)" { +@test "podman load - NAME and NAME:TAG arguments work" { + skip_if_remote "FIXME: pending #7124" + get_iid_and_name run_podman save $iid -o $archive run_podman rmi $iid diff --git a/test/system/130-kill.bats b/test/system/130-kill.bats index c16e64c586..05090f852f 100644 --- a/test/system/130-kill.bats +++ b/test/system/130-kill.bats @@ -6,6 +6,8 @@ load helpers @test "podman kill - test signal handling in containers" { + skip_if_remote "FIXME: pending #7135" + # podman-remote and crun interact poorly in f31: crun seems to gobble up # some signals. # Workaround: run 'env --default-signal sh' instead of just 'sh' in diff --git a/test/system/140-diff.bats b/test/system/140-diff.bats index 9f4a2c0deb..01ec5430e6 100644 --- a/test/system/140-diff.bats +++ b/test/system/140-diff.bats @@ -6,9 +6,16 @@ load helpers @test "podman diff" { + n=$(random_string 10) # container name rand_file=$(random_string 10) - run_podman run $IMAGE sh -c "touch /$rand_file;rm /etc/services" - run_podman diff --format json -l + run_podman run --name $n $IMAGE sh -c "touch /$rand_file;rm /etc/services" + + # If running local, test `-l` (latest) option. This can't work with remote. + if ! is_remote; then + n=-l + fi + + run_podman diff --format json $n # Expected results for each type of diff declare -A expect=( @@ -22,7 +29,7 @@ load helpers is "$result" "${expect[$field]}" "$field" done - run_podman rm -l + run_podman rm $n } # vim: filetype=sh diff --git a/test/system/160-volumes.bats b/test/system/160-volumes.bats index 3233e6f04d..ef38b2a687 100644 --- a/test/system/160-volumes.bats +++ b/test/system/160-volumes.bats @@ -93,6 +93,7 @@ Labels.l | $mylabel is "$(<$mountpoint/myfile)" "$rand" "we see content created in container" # Clean up + if is_remote; then sleep 2;fi # FIXME: pending #7119 run_podman volume rm $myvolume } @@ -134,12 +135,14 @@ EOF is "$output" "got here -$rand-" "script in volume is runnable with default (exec)" # Clean up + if is_remote; then sleep 2;fi # FIXME: pending #7119 run_podman volume rm $myvolume } # Anonymous temporary volumes, and persistent autocreated named ones @test "podman volume, implicit creation with run" { + skip_if_remote "FIXME: pending #7128" # No hostdir arg: create anonymous container with random name rand=$(random_string) @@ -172,6 +175,7 @@ EOF run_podman run --rm -v $myvol:/myvol:z $IMAGE \ sh -c "cp /myvol/myfile /myvol/myfile2" + if is_remote; then sleep 2;fi # FIXME: pending #7119 run_podman volume rm $myvol # Autocreated volumes should also work with keep-id @@ -180,6 +184,7 @@ EOF run_podman run --rm -v $myvol:/myvol:z --userns=keep-id $IMAGE \ touch /myvol/myfile + if is_remote; then sleep 2;fi # FIXME: pending #7119 run_podman volume rm $myvol } @@ -187,6 +192,7 @@ EOF # Confirm that container sees the correct id @test "podman volume with --userns=keep-id" { is_rootless || skip "only meaningful when run rootless" + skip_if_remote "FIXME: pending #7195" myvoldir=${PODMAN_TMPDIR}/volume_$(random_string) mkdir $myvoldir diff --git a/test/system/200-pod.bats b/test/system/200-pod.bats index 0ad5553059..cbfd7fe030 100644 --- a/test/system/200-pod.bats +++ b/test/system/200-pod.bats @@ -18,7 +18,9 @@ function teardown() { @test "podman pod top - containers in different PID namespaces" { - skip_if_remote "podman-pod does not work with podman-remote" + if is_remote && is_rootless; then + skip "FIXME: pending #7139" + fi # With infra=false, we don't get a /pause container (we also # don't pull k8s.gcr.io/pause ) @@ -53,7 +55,9 @@ function teardown() { @test "podman pod - communicating between pods" { - skip_if_remote "podman-pod does not work with podman-remote" + if is_remote && is_rootless; then + skip "FIXME: pending #7139" + fi podname=pod$(random_string) run_podman 1 pod exists $podname @@ -77,7 +81,7 @@ function teardown() { run_podman ps --format '{{.Pod}}' newline=" " - is "$output" "${podid:0:12}${newline}${podid:0:12}" "sdfdsf" + is "$output" "${podid:0:12}${newline}${podid:0:12}" "ps shows 2 pod IDs" # Talker: send the message via common port on localhost message=$(random_string 15) @@ -89,6 +93,7 @@ function teardown() { is "$output" "$message" "message sent from one container to another" # Clean up. First the nc -l container... + if is_remote; then sleep 2;fi # FIXME: pending #7119 run_podman rm $cid1 # ...then, from pause container, find the image ID of the pause image... @@ -99,6 +104,7 @@ function teardown() { pause_iid="$output" # ...then rm the pod, then rmi the pause image so we don't leave strays. + if is_remote; then sleep 2;fi # FIXME: pending #7119 run_podman pod rm $podname run_podman rmi $pause_iid @@ -135,6 +141,10 @@ function random_ip() { } @test "podman pod create - hashtag AllTheOptions" { + if is_remote && is_rootless; then + skip "FIXME: pending #7139" + fi + mac=$(random_mac) add_host_ip=$(random_ip) add_host_n=$(random_string | tr A-Z a-z).$(random_string | tr A-Z a-z).xyz @@ -205,6 +215,7 @@ function random_ip() { is "$output" ".*options $dns_opt" "--dns-opt was added" # pod inspect + if is_remote; then sleep 2;fi # FIXME: pending #7119 run_podman pod inspect --format '{{.Name}}: {{.ID}} : {{.NumContainers}} : {{.Labels}}' mypod is "$output" "mypod: $pod_id : 1 : map\[${labelname}:${labelvalue}]" \ "pod inspect --format ..." diff --git a/test/system/220-healthcheck.bats b/test/system/220-healthcheck.bats index e649ad3d24..3405029c1f 100644 --- a/test/system/220-healthcheck.bats +++ b/test/system/220-healthcheck.bats @@ -25,6 +25,7 @@ function _check_health { @test "podman healthcheck" { + skip_if_remote "FIXME: pending #7137" # Create an image with a healthcheck script; said script will # pass until the file /uh-oh gets created (by us, via exec) diff --git a/test/system/300-cli-parsing.bats b/test/system/300-cli-parsing.bats index 92c0731023..2abc01bb79 100644 --- a/test/system/300-cli-parsing.bats +++ b/test/system/300-cli-parsing.bats @@ -10,6 +10,8 @@ load helpers # Error: invalid argument "true=\"false\"" for "-l, --label" \ # flag: parse error on line 1, column 5: bare " in non-quoted-field run_podman run --rm --label 'true="false"' $IMAGE true + + if is_remote; then sleep 2;fi # FIXME: pending #7119 } # vim: filetype=sh diff --git a/test/system/400-unprivileged-access.bats b/test/system/400-unprivileged-access.bats index 1384c0ab80..ebca75f134 100644 --- a/test/system/400-unprivileged-access.bats +++ b/test/system/400-unprivileged-access.bats @@ -101,6 +101,11 @@ EOF # #6957 - mask out /proc/acpi, /sys/dev, and other sensitive system files @test "sensitive mount points are masked without --privileged" { + # Weird error, maybe a flake? + # can only attach to created or running containers: container state improper + # https://github.com/containers/podman/pull/7111#issuecomment-666858715 + skip_if_remote "FIXME: Weird flake" + # FIXME: this should match the list in pkg/specgen/generate/config_linux.go local -a mps=( /proc/acpi @@ -160,6 +165,8 @@ EOF die "$path: Unknown file type '$type'" fi done + + if is_remote; then sleep 2;fi # FIXME: pending #7119 } # vim: filetype=sh diff --git a/test/system/410-selinux.bats b/test/system/410-selinux.bats index 497e29b3ee..c85fb25634 100644 --- a/test/system/410-selinux.bats +++ b/test/system/410-selinux.bats @@ -16,6 +16,7 @@ function check_label() { # FIXME: it'd be nice to specify the command to run, e.g. 'ls -dZ /', # but alpine ls (from busybox) doesn't support -Z run_podman run --rm $args $IMAGE cat -v /proc/self/attr/current + if is_remote; then sleep 2;fi # FIXME: pending #7119 # FIXME: on some CI systems, 'run --privileged' emits a spurious # warning line about dup devices. Ignore it. diff --git a/test/system/helpers.bash b/test/system/helpers.bash index abca917391..a6414344e5 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -240,12 +240,29 @@ function is_remote() { [[ "$PODMAN" =~ -remote ]] } +########################### +# _add_label_if_missing # make sure skip messages include rootless/remote +########################### +function _add_label_if_missing() { + local msg="$1" + local want="$2" + + if [ -z "$msg" ]; then + echo + elif expr "$msg" : ".*$want" &>/dev/null; then + echo "$msg" + else + echo "[$want] $msg" + fi +} + ###################### # skip_if_rootless # ...with an optional message ###################### function skip_if_rootless() { if is_rootless; then - skip "${1:-not applicable under rootless podman}" + local msg=$(_add_label_if_missing "$1" "rootless") + skip "${msg:-not applicable under rootless podman}" fi } @@ -254,7 +271,8 @@ function skip_if_rootless() { #################### function skip_if_remote() { if is_remote; then - skip "${1:-test does not work with podman-remote}" + local msg=$(_add_label_if_missing "$1" "remote") + skip "${msg:-test does not work with podman-remote}" fi }