Add persistent root filesystem

If volumeDevices has an entry with 'devicePath: /', it's used as a
persistent root filesystem for the VM. It gets overwritten
again if the image digest changes.
See examples/cirros-vm-persistent-rootfs.yaml

examples/cirros-vm-persistent-rootfs.yaml

@@ -0,0 +1,80 @@
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: local-block-pvc
+spec:
+  accessModes:
+  - ReadWriteOnce
+  volumeMode: Block
+  storageClassName: local-storage
+  resources:
+    requests:
+      storage: 100Mi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: local-block-pv
+spec:
+  capacity:
+    storage: 100Mi
+  accessModes:
+  - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: local-storage
+  volumeMode: Block
+  local:
+    # set up with:
+    # docker exec kube-node-1 /bin/bash -c 'dd if=/dev/zero of=/rawtest bs=1M count=1000 && losetup -f /rawtest --show'
+    path: /dev/loop0
+  claimRef:
+    name: local-block-pvc
+    namespace: default
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - kube-node-1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cirros-vm-p
+  annotations:
+    kubernetes.io/target-runtime: virtlet.cloud
+    # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason
+    VirtletDiskDriver: virtio
+    VirtletSSHKeys: |
+      ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
+spec:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: extraRuntime
+            operator: In
+            values:
+            - virtlet
+  # This is the number of seconds Virtlet gives the VM to shut down cleanly.
+  # The default value of 30 seconds is ok for containers but probably too
+  # low for VM, so overriding it here is strongly advised.
+  terminationGracePeriodSeconds: 120
+  containers:
+  - name: cirros-vm
+    image: virtlet.cloud/cirros
+    imagePullPolicy: IfNotPresent
+    # tty and stdin required for `kubectl attach -t` to work
+    tty: true
+    stdin: true
+    volumeDevices:
+    - devicePath: /
+      name: testpvc
+  volumes:
+  - name: testpvc
+    persistentVolumeClaim:
+      claimName: local-block-pvc

pkg/image/fake/store.go

@@ -119,13 +119,13 @@ func (s *FakeStore) GC() error {
 	return nil
 }
 
-// GetImagePathAndVirtualSize implements GC method of Store interface.
-func (s *FakeStore) GetImagePathAndVirtualSize(imageName string) (string, uint64, error) {
+// GetImagePathDigestAndVirtualSize implements GetImagePathDigestAndVirtualSize method of Store interface.
+func (s *FakeStore) GetImagePathDigestAndVirtualSize(imageName string) (string, digest.Digest, uint64, error) {
 	img, found := s.images[imageName]
 	if !found {
-		return "", 0, fmt.Errorf("image not found: %q", imageName)
+		return "", "", 0, fmt.Errorf("image not found: %q", imageName)
 	}
-	return img.Path, img.Size, nil
+	return img.Path, digest.Digest(img.Digest), img.Size, nil
 }
 
 // SetRefGetter implements SetRefGetter method of Store interface.

pkg/image/image.go

@@ -81,10 +81,10 @@ type Store interface {
 	// GC removes all unused or partially downloaded images.
 	GC() error
 
-	// GetImagePathAndVirtualSize returns the path to image data
-	// and virtual size for the specified image. It accepts
-	// an image reference or a digest.
-	GetImagePathAndVirtualSize(ref string) (string, uint64, error)
+	// GetImagePathDigestAndVirtualSize returns the path to image
+	// data, the digest and the virtual size for the specified
+	// image. It accepts an image reference or a digest.
+	GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error)
 
 	// SetRefGetter sets a function that will be used to determine
 	// the set of images that are currently in use.
@@ -472,30 +472,33 @@ func (s *FileStore) GC() error {
 	return nil
 }
 
-// GetImagePathAndVirtualSize implements GC method of Store interface.
-func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, error) {
+// GetImagePathDigestAndVirtualSize implements GetImagePathDigestAndVirtualSize method of Store interface.
+func (s *FileStore) GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error) {
 	s.Lock()
 	defer s.Unlock()
-	glog.V(3).Infof("GetImagePathAndVirtualSize(): %q", ref)
+	glog.V(3).Infof("GetImagePathDigestAndVirtualSize(): %q", ref)
 
 	var pathViaDigest, pathViaName string
 	// parsing digest as ref gives bad results
-	if d, err := digest.Parse(ref); err == nil {
+	d, err := digest.Parse(ref)
+	if err == nil {
 		if d.Algorithm() != digest.SHA256 {
-			return "", 0, fmt.Errorf("bad image digest (need sha256): %q", d)
+			return "", "", 0, fmt.Errorf("bad image digest (need sha256): %q", d)
 		}
 		pathViaDigest = s.dataFileName(d.Hex())
 	} else {
 		parsed, err := reference.Parse(ref)
 		if err != nil {
-			return "", 0, fmt.Errorf("bad image reference %q: %v", ref, err)
+			return "", "", 0, fmt.Errorf("bad image reference %q: %v", ref, err)
 		}
 
+		d = ""
 		if digested, ok := parsed.(reference.Digested); ok {
 			if digested.Digest().Algorithm() != digest.SHA256 {
-				return "", 0, fmt.Errorf("bad image digest (need sha256): %q", digested.Digest())
+				return "", "", 0, fmt.Errorf("bad image digest (need sha256): %q", digested.Digest())
 			}
-			pathViaDigest = s.dataFileName(digested.Digest().Hex())
+			d = digested.Digest()
+			pathViaDigest = s.dataFileName(d.Hex())
 		}
 
 		if named, ok := parsed.(reference.Named); ok && named.Name() != "" {
@@ -504,35 +507,36 @@ func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, erro
 				glog.Warningf("error reading link %q: %v", pathViaName, err)
 			} else {
 				pathViaName = filepath.Join(s.linkDir(), pathViaName)
+				d = digest.NewDigestFromHex(string(digest.SHA256), filepath.Base(pathViaName))
 			}
 		}
 	}
 
 	path := pathViaDigest
 	switch {
 	case pathViaDigest == "" && pathViaName == "":
-		return "", 0, fmt.Errorf("bad image reference %q", ref)
+		return "", "", 0, fmt.Errorf("bad image reference %q", ref)
 	case pathViaDigest == "":
 		path = pathViaName
 	case pathViaName != "":
 		fi1, err := os.Stat(pathViaName)
 		if err != nil {
-			return "", 0, err
+			return "", "", 0, err
 		}
 		fi2, err := os.Stat(pathViaDigest)
 		if err != nil {
-			return "", 0, err
+			return "", "", 0, err
 		}
 		if !os.SameFile(fi1, fi2) {
-			return "", 0, fmt.Errorf("digest / name path mismatch: %q vs %q", pathViaDigest, pathViaName)
+			return "", "", 0, fmt.Errorf("digest / name path mismatch: %q vs %q", pathViaDigest, pathViaName)
 		}
 	}
 
 	vsize, err := s.vsizeFunc(path)
 	if err != nil {
-		return "", 0, fmt.Errorf("error getting image size for %q: %v", path, err)
+		return "", "", 0, fmt.Errorf("error getting image size for %q: %v", path, err)
 	}
-	return path, vsize, nil
+	return path, d, vsize, nil
 }
 
 // SetRefGetter implements SetRefGetter method of Store interface.

pkg/image/image_test.go

@@ -187,9 +187,13 @@ func (tst *ifsTester) verifyListImages(filter string, expectedImages ...*Image)
 }
 
 func (tst *ifsTester) verifyImage(ref string, expectedContents string) {
-	if path, vsize, err := tst.store.GetImagePathAndVirtualSize(ref); err != nil {
+	if path, digest, vsize, err := tst.store.GetImagePathDigestAndVirtualSize(ref); err != nil {
 		tst.t.Errorf("GetImagePathAndVirtualSize(): %v", err)
 	} else {
+		expectedDigest := "sha256:" + sha256str(expectedContents)
+		if string(digest) != expectedDigest {
+			tst.t.Errorf("bad digest: %s instead of %s", digest, expectedDigest)
+		}
 		tst.verifyFileContents(path, expectedContents)
 		expectedVirtualSize := uint64(len(expectedContents)) + 1000
 		if vsize != expectedVirtualSize {

pkg/libvirttools/TestContainerLifecycle.out.yaml

@@ -1,6 +1,6 @@
 - name: 'domain conn: ListDomains'
   value: []
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__ceph_flexvolume.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__cloud-init.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__cloud-init_with_user_data.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__persistent_rootfs.out.yaml

@@ -0,0 +1,87 @@
+- name: GetImagePathDigestAndVirtualSize
+  value: fake/image1
+- name: CMD
+  value:
+    cmd: blockdev --getsz /fakedev/69eec606-0493-5825-73a4-c5e0c0236155/volumeDevices/kubernetes.io~local-volume/root
+    stdout: "1000"
+- name: CMD
+  value:
+    cmd: dmsetup create virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155
+    stdin: |
+      0 999 linear /fakedev/69eec606-0493-5825-73a4-c5e0c0236155/volumeDevices/kubernetes.io~local-volume/root 1
+- name: CMD
+  value:
+    cmd: qemu-img convert -O raw /fake/volume/path /dev/mapper/virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155
+- name: 'domain conn: DefineDomain'
+  value: |-
+    <domain type="kvm">
+      <name>virtlet-231700d5-c9a6-container1</name>
+      <uuid>231700d5-c9a6-5a49-738d-99a954c51550</uuid>
+      <memory unit="MiB">1024</memory>
+      <vcpu>1</vcpu>
+      <cputune>
+        <shares>0</shares>
+        <period>0</period>
+        <quota>0</quota>
+      </cputune>
+      <os>
+        <type>hvm</type>
+        <boot dev="hd"></boot>
+      </os>
+      <features>
+        <acpi></acpi>
+      </features>
+      <on_poweroff>destroy</on_poweroff>
+      <on_reboot>restart</on_reboot>
+      <on_crash>restart</on_crash>
+      <devices>
+        <emulator>/vmwrapper</emulator>
+        <disk type="block" device="disk">
+          <driver name="qemu" type="raw"></driver>
+          <source dev="/dev/mapper/virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155"></source>
+          <target dev="sda" bus="scsi"></target>
+          <address type="drive" controller="0" bus="0" target="0" unit="0"></address>
+        </disk>
+        <disk type="file" device="cdrom">
+          <driver name="qemu" type="raw"></driver>
+          <source file="/var/lib/virtlet/config/config-231700d5-c9a6-5a49-738d-99a954c51550.iso"></source>
+          <target dev="sdb" bus="scsi"></target>
+          <readonly></readonly>
+          <address type="drive" controller="0" bus="0" target="0" unit="1"></address>
+        </disk>
+        <controller type="scsi" index="0" model="virtio-scsi">
+          <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"></address>
+        </controller>
+        <controller type="pci" model="pci-root"></controller>
+        <serial type="unix">
+          <source mode="connect" path="/var/lib/libvirt/streamer.sock">
+            <reconnect enabled="yes" timeout="1"></reconnect>
+          </source>
+          <target port="0"></target>
+        </serial>
+        <input type="tablet" bus="usb"></input>
+        <graphics type="vnc" port="-1"></graphics>
+        <video>
+          <model type="cirrus"></model>
+        </video>
+      </devices>
+      <commandline xmlns="http://libvirt.org/schemas/domain/qemu/1.0">
+        <env name="VIRTLET_EMULATOR" value="/usr/bin/kvm"></env>
+        <env name="VIRTLET_NET_KEY" value="/tmp/fakenetns"></env>
+        <env name="VIRTLET_CONTAINER_ID" value="231700d5-c9a6-5a49-738d-99a954c51550"></env>
+        <env name="VIRTLET_CONTAINER_LOG_PATH" value="/var/log/pods/69eec606-0493-5825-73a4-c5e0c0236155/container1_42.log"></env>
+      </commandline>
+    </domain>
+- name: 'domain conn: virtlet-231700d5-c9a6-container1: Create'
+- name: 'domain conn: virtlet-231700d5-c9a6-container1: iso image'
+  value:
+    meta-data: '{"instance-id":"testName_0.default","local-hostname":"testName_0"}'
+    network-config: |
+      version: 1
+    user-data: |
+      #cloud-config
+- name: 'domain conn: virtlet-231700d5-c9a6-container1: Destroy'
+- name: 'domain conn: virtlet-231700d5-c9a6-container1: Undefine'
+- name: CMD
+  value:
+    cmd: dmsetup remove virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155

pkg/libvirttools/TestDomainDefinitions__plain_domain.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__raw_block_volume.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__raw_devices.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__vcpu_count.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__virtio_disk_driver.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainDefinitions__volumes.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestDomainForcedShutdown.out.yaml

@@ -1,4 +1,4 @@
-- name: GetImagePathAndVirtualSize
+- name: GetImagePathDigestAndVirtualSize
   value: fake/image1
 - name: 'storage: CreateStoragePool'
   value: |-

pkg/libvirttools/TestPersistentRootVolume__first_image_too_big.out.yaml

@@ -0,0 +1,7 @@
+- name: setup
+- name: 'image: GetImagePathDigestAndVirtualSize'
+  value: persistent/image1
+- name: CMD
+  value:
+    cmd: blockdev --getsz /dev/rootdev
+    stdout: "8"