Add e2e test for image verification

tests/e2e/basic_test.go

@@ -38,7 +38,7 @@ var _ = Describe("Virtlet [Basic cirros tests]", func() {
 
 	BeforeAll(func() {
 		vm = controller.VM("cirros-vm")
-		Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
+		Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		var err error
 		vmPod, err = vm.Pod()
 		Expect(err).NotTo(HaveOccurred())
@@ -182,7 +182,7 @@ var _ = Describe("Virtlet [Disruptive]", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		vm = controller.VM("cirros-vm")
-		Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
+		Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 	})
 })
 
@@ -228,3 +228,35 @@ func itShouldHaveNetworkConnectivity(podIface func() *framework.PodInterface, ss
 		}, 60*5)
 	})
 }
+
+var _ = Describe("Virtlet [Image verification]", func() {
+	var (
+		vm *framework.VMInterface
+	)
+
+	BeforeAll(func() {
+		vm = controller.VM("cirros-vm")
+	})
+
+	AfterAll(func() {
+		Expect(vm.Delete(time.Minute * 2)).To(Succeed())
+	})
+
+	It("Should fail for images with mismatching digest", func() {
+		opts := VMOptions{}.ApplyDefaults()
+		p := strings.Index(opts.Image, "@")
+		if p >= 0 {
+			opts.Image = opts.Image[:p]
+		}
+		opts.Image += "@sha256:0000000000000000000000000000000000000000000000000000000000000000"
+		Expect(vm.Create(opts, nil)).To(Succeed())
+
+		pod := vm.PodWithoutChecks()
+		Expect(pod.WaitForPodStatus([]string{
+			"ErrImagePull", "ImagePullBackOff", "ImageInspectError",
+		}, time.Minute*5)).To(Succeed())
+		events, err := pod.LoadEvents()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(events).To(ContainElement(ContainSubstring("image digest mismatch")))
+	})
+})

tests/e2e/ceph_test.go

@@ -65,7 +65,7 @@ var _ = Describe("Ceph volumes tests", func() {
 				})
 			}
 
-			Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
+			Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
 			var err error
 			_, err = vm.Pod()
 			Expect(err).NotTo(HaveOccurred())
@@ -141,7 +141,7 @@ var _ = Describe("Ceph volumes tests", func() {
 				})
 			}
 
-			Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
+			Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
 			_ = do(vm.Pod()).(*framework.PodInterface)
 		})
 

tests/e2e/cloudinit_test.go

@@ -44,7 +44,7 @@ var _ = Describe("Cloud-init related tests", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			vm = controller.VM("ssh-from-cm-impl")
-			Expect(vm.Create(VMOptions{
+			Expect(vm.CreateAndWait(VMOptions{
 				SSHKeySource: "configmap/cm-ssh-key-impl",
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		})
@@ -75,7 +75,7 @@ var _ = Describe("Cloud-init related tests", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			vm = controller.VM("ssh-from-cm-expl")
-			Expect(vm.Create(VMOptions{
+			Expect(vm.CreateAndWait(VMOptions{
 				SSHKeySource: "configmap/cm-ssh-key-expl/myKey",
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		})
@@ -106,7 +106,7 @@ var _ = Describe("Cloud-init related tests", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			vm = controller.VM("ssh-from-secret-impl")
-			Expect(vm.Create(VMOptions{
+			Expect(vm.CreateAndWait(VMOptions{
 				SSHKeySource: "secret/secret-ssh-key-impl",
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		})
@@ -137,7 +137,7 @@ var _ = Describe("Cloud-init related tests", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			vm = controller.VM("ssh-from-secret-expl")
-			Expect(vm.Create(VMOptions{
+			Expect(vm.CreateAndWait(VMOptions{
 				SSHKeySource: "secret/secret-ssh-key-expl/myKey",
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		})
@@ -171,7 +171,7 @@ var _ = Describe("Cloud-init related tests", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			vm = controller.VM("userdata-cm")
-			Expect(vm.Create(VMOptions{
+			Expect(vm.CreateAndWait(VMOptions{
 				UserDataSource: "configmap/cm-userdata",
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		})
@@ -206,7 +206,7 @@ var _ = Describe("Cloud-init related tests", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			vm = controller.VM("userdata-secret")
-			Expect(vm.Create(VMOptions{
+			Expect(vm.CreateAndWait(VMOptions{
 				UserDataSource: "secret/secret-userdata",
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		})
@@ -242,7 +242,7 @@ var _ = Describe("Cloud-init related tests", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			vm = controller.VM("userdata-cm-merge")
-			Expect(vm.Create(VMOptions{
+			Expect(vm.CreateAndWait(VMOptions{
 				UserDataSource: "configmap/cm-userdata",
 				UserData:       userData,
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())

tests/e2e/cni_test.go

@@ -35,7 +35,7 @@ var _ = Describe("Virtlet CNI", func() {
 		// if network namespace was deleted
 		It("Should delete network namespace when VM is deleted", func() {
 			vm = controller.VM("cirros-vm")
-			err := vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)
+			err := vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)
 			Expect(err).NotTo(HaveOccurred())
 
 			virtletPod, err := vm.VirtletPod()

tests/e2e/framework/pod_interface.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strings"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
@@ -73,8 +74,10 @@ func (pi *PodInterface) Delete() error {
 	return pi.controller.client.Pods(pi.Pod.Namespace).Delete(pi.Pod.Name, nil)
 }
 
-// Wait waits for pod to start and checks that it doesn't fail immediately after that
-func (pi *PodInterface) Wait(timing ...time.Duration) error {
+// WaitForPodStatus waits for the pod to reach the specified status. If expectedContainerErrors
+// is empty, the pod is expected to become Running and Ready. If it isn't, the pod is expected
+// to have one of these errors among its container statuses.
+func (pi *PodInterface) WaitForPodStatus(expectedContainerErrors []string, timing ...time.Duration) error {
 	timeout := time.Minute * 5
 	pollPeriond := time.Second
 	consistencyPeriod := time.Second * 5
@@ -95,23 +98,45 @@ func (pi *PodInterface) Wait(timing ...time.Duration) error {
 		}
 		pi.Pod = podUpdated
 
+		needErrors := len(expectedContainerErrors) > 0
 		phase := v1.PodRunning
+		if needErrors {
+			phase = v1.PodPending
+		}
 		if podUpdated.Status.Phase != phase {
 			return fmt.Errorf("pod %s is not %s phase: %s", podUpdated.Name, phase, podUpdated.Status.Phase)
 		}
 
+		gotExpectedError := false
 		for _, cs := range podUpdated.Status.ContainerStatuses {
-			if cs.State.Running == nil {
+			switch {
+			case !needErrors && cs.State.Running == nil:
 				return fmt.Errorf("container %s in pod %s is not running: %s", cs.Name, podUpdated.Name, spew.Sdump(cs.State))
-			}
-			if !cs.Ready {
+			case !needErrors && !cs.Ready:
 				return fmt.Errorf("container %s in pod %s in not ready", cs.Name, podUpdated.Name)
+			case needErrors && cs.State.Waiting == nil:
+				return fmt.Errorf("container %s in pod %s not in waiting state", cs.Name)
+			case needErrors:
+				for _, errStr := range expectedContainerErrors {
+					if cs.State.Waiting.Reason == errStr {
+						gotExpectedError = true
+						break
+					}
+				}
 			}
 		}
+		if needErrors && !gotExpectedError {
+			return fmt.Errorf("didn't get one of expected container errors: %s", strings.Join(expectedContainerErrors, " | "))
+		}
 		return nil
 	}, timeout, pollPeriond, consistencyPeriod)
 }
 
+// Wait waits for pod to start and checks that it doesn't fail immediately after that
+func (pi *PodInterface) Wait(timing ...time.Duration) error {
+	return pi.WaitForPodStatus(nil, timing...)
+}
+
 // WaitDestruction waits for the pod to be deleted
 func (pi *PodInterface) WaitDestruction(timing ...time.Duration) error {
 	timeout := time.Minute * 5
@@ -232,6 +257,20 @@ func (pi *PodInterface) DinDNodeExecutor() (Executor, error) {
 	return pi.controller.DinDNodeExecutor(pi.Pod.Spec.NodeName)
 }
 
+// LoadEvents retrieves the evnets for this pod as a list
+// of strings of the form Type:Reason:Message
+func (pi *PodInterface) LoadEvents() ([]string, error) {
+	events, err := pi.controller.client.Events(pi.controller.Namespace()).Search(scheme.Scheme, pi.Pod)
+	if err != nil {
+		return nil, err
+	}
+	var r []string
+	for _, e := range events.Items {
+		r = append(r, fmt.Sprintf("%s:%s:%s", e.Type, e.Reason, e.Message))
+	}
+	return r, nil
+}
+
 type containerInterface struct {
 	podInterface *PodInterface
 	name         string

tests/e2e/framework/vm_interface.go

@@ -97,24 +97,34 @@ func (vmi *VMInterface) Pod() (*PodInterface, error) {
 	return vmi.pod, nil
 }
 
-// Create create new virtlet VM pod in k8s
-func (vmi *VMInterface) Create(options VMOptions, waitTimeout time.Duration, beforeCreate func(*PodInterface)) error {
+// PodWithoutChecks returns the underlying pods without performing any
+// checks
+func (vmi *VMInterface) PodWithoutChecks() *PodInterface {
+	return vmi.pod
+}
+
+// Create creates a new VM pod
+func (vmi *VMInterface) Create(options VMOptions, beforeCreate func(*PodInterface)) error {
 	pod := newPodInterface(vmi.controller, vmi.buildVMPod(options))
 	if beforeCreate != nil {
 		beforeCreate(pod)
 	}
-	err := pod.Create()
-	if err != nil {
-		return err
-	}
-	err = pod.Wait(waitTimeout)
-	if err != nil {
+	if err := pod.Create(); err != nil {
 		return err
 	}
 	vmi.pod = pod
 	return nil
 }
 
+// CreateAndWait creates a new VM pod in k8s and waits for it to start
+func (vmi *VMInterface) CreateAndWait(options VMOptions, waitTimeout time.Duration, beforeCreate func(*PodInterface)) error {
+	err := vmi.Create(options, beforeCreate)
+	if err == nil {
+		err = vmi.pod.Wait(waitTimeout)
+	}
+	return err
+}
+
 // Delete deletes VM pod and waits for it to disappear from k8s
 func (vmi *VMInterface) Delete(waitTimeout time.Duration) error {
 	if vmi.pod == nil {
@@ -244,6 +254,9 @@ func (vmi *VMInterface) DomainName() (string, error) {
 	if err != nil {
 		return "", err
 	}
+	if len(pod.Pod.Status.ContainerStatuses) != 1 {
+		return "", fmt.Errorf("expected single container status, but got %d statuses", len(pod.Pod.Status.ContainerStatuses))
+	}
 	containerID := pod.Pod.Status.ContainerStatuses[0].ContainerID
 	match := regexp.MustCompile("__(.+)$").FindStringSubmatch(containerID)
 	if len(match) < 2 {

tests/e2e/hung_vm_test.go

@@ -33,7 +33,7 @@ var _ = Describe("Hung VM", func() {
 
 	BeforeAll(func() {
 		vm = controller.VM("hung-vm")
-		Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
+		Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		var err error
 		_, err = vm.Pod()
 		Expect(err).NotTo(HaveOccurred())

tests/e2e/image_name_translation_test.go

@@ -54,7 +54,7 @@ var _ = Describe("Image URL", func() {
 
 	It("Can be specified in CRD [Conformance]", func() {
 		vm := controller.VM("cirros-vm-with-remapped-image")
-		Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
+		Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		_, err := vm.Pod()
 		Expect(err).NotTo(HaveOccurred())
 		deleteVM(vm)

tests/e2e/multi_cni_test.go

@@ -100,7 +100,7 @@ func makeMultiCNIVM(name string, addCNIAnnotation bool) *multiCNIVM {
 	if addCNIAnnotation {
 		opts.MultiCNI = "calico,flannel"
 	}
-	Expect(vm.Create(opts.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
+	Expect(vm.CreateAndWait(opts.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 	vmPod, err := vm.Pod()
 	Expect(err).NotTo(HaveOccurred())
 	return &multiCNIVM{

tests/e2e/resources_test.go

@@ -36,7 +36,7 @@ var _ = Describe("VM resources", func() {
 
 	BeforeAll(func() {
 		vm = controller.VM("vm-resources")
-		Expect(vm.Create(VMOptions{
+		Expect(vm.CreateAndWait(VMOptions{
 			VCPUCount:      2,
 			RootVolumeSize: "4Gi",
 		}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())

tests/e2e/restart_virtlet_test.go

@@ -36,7 +36,7 @@ var _ = Describe("Virtlet restart [Disruptive]", func() {
 
 	BeforeAll(func() {
 		vm = controller.VM("cirros-vm")
-		vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)
+		vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)
 		var err error
 		vmPod, err = vm.Pod()
 		Expect(err).NotTo(HaveOccurred())

tests/e2e/stop_qemu_test.go

@@ -37,7 +37,7 @@ var _ = Describe("QEMU Process", func() {
 
 	BeforeAll(func() {
 		vm = controller.VM("kill-qemu-vm")
-		Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
+		Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 		var err error
 		vmPod, err := vm.Pod()
 		Expect(err).NotTo(HaveOccurred())

tests/e2e/virtletctl_test.go

@@ -66,7 +66,7 @@ var _ = Describe("virtletctl", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			vm = controller.VM("virtletctl-cirros-vm")
-			Expect(vm.Create(VMOptions{
+			Expect(vm.CreateAndWait(VMOptions{
 				SSHKeySource: "configmap/sshkey",
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
 

tests/e2e/volume_mount_test.go

@@ -177,7 +177,7 @@ var _ = Describe("Container volume mounts", func() {
 				})
 			}
 
-			Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
+			Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
 		})
 
 		AfterAll(func() {
@@ -236,7 +236,7 @@ var _ = Describe("Container volume mounts", func() {
 				})
 			}
 
-			Expect(vm.Create(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
+			Expect(vm.CreateAndWait(VMOptions{}.ApplyDefaults(), time.Minute*5, podCustomization)).To(Succeed())
 		})
 
 		AfterAll(func() {
@@ -270,7 +270,7 @@ func addFlexvolMount(pod *framework.PodInterface, name string, mountPath string,
 
 func makeVolumeMountVM(nodeName string, podCustomization func(*framework.PodInterface)) *framework.VMInterface {
 	vm := controller.VM("mount-vm")
-	Expect(vm.Create(VMOptions{
+	Expect(vm.CreateAndWait(VMOptions{
 		NodeName: nodeName,
 		// TODO: should also have an option to test using
 		// ubuntu image with volumes mounted using cloud-init

tests/longevity/vmcontoller.go

@@ -74,7 +74,7 @@ func (i *VMInstance) Create() error {
 	var err error
 
 	i.vm = i.controller.VM(i.name)
-	err = i.vm.Create(e2e.VMOptions{}.ApplyDefaults(), time.Minute*5, nil)
+	err = i.vm.CreateAndWait(e2e.VMOptions{}.ApplyDefaults(), time.Minute*5, nil)
 	if err != nil {
 		return err
 	}