From 37e2ee7769777856ec38c8c172a0c912a9f03783 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 14 Sep 2018 17:45:00 +0300 Subject: [PATCH] Add persistent root filesystem If volumeDevices has an entry with 'devicePath: /', it's used as a persistent root filesystem for the VM. It gets overwritten again if the image digest changes. See examples/cirros-vm-persistent-rootfs.yaml --- examples/cirros-vm-persistent-rootfs.yaml | 80 ++++++ pkg/image/fake/store.go | 8 +- pkg/image/image.go | 40 +-- pkg/image/image_test.go | 6 +- .../TestContainerLifecycle.out.yaml | 2 +- ...omainDefinitions__ceph_flexvolume.out.yaml | 2 +- ...TestDomainDefinitions__cloud-init.out.yaml | 2 +- ...itions__cloud-init_with_user_data.out.yaml | 2 +- ...ainDefinitions__persistent_rootfs.out.yaml | 87 +++++++ ...stDomainDefinitions__plain_domain.out.yaml | 2 +- ...mainDefinitions__raw_block_volume.out.yaml | 2 +- ...estDomainDefinitions__raw_devices.out.yaml | 2 +- ...TestDomainDefinitions__vcpu_count.out.yaml | 2 +- ...inDefinitions__virtio_disk_driver.out.yaml | 2 +- .../TestDomainDefinitions__volumes.out.yaml | 2 +- .../TestDomainForcedShutdown.out.yaml | 2 +- ...ntRootVolume__first_image_too_big.out.yaml | 7 + ...ersistentRootVolume__image_change.out.yaml | 52 ++++ ...istentRootVolume__image_unchanged.out.yaml | 49 ++++ ...tRootVolume__second_image_too_big.out.yaml | 33 +++ ...estPersistentRootVolume__symlinks.out.yaml | 49 ++++ .../TestRootVolumeLifeCycle.out.yaml | 4 +- ...estRootVolumeSize__default__zero_.out.yaml | 4 +- ...greater_than_fakeImageVirtualSize.out.yaml | 4 +- .../TestRootVolumeSize__negative.out.yaml | 4 +- ...ize__same_as_fakeImageVirtualSize.out.yaml | 4 +- ...smaller_than_fakeImageVirtualSize.out.yaml | 4 +- pkg/libvirttools/block_volumesource.go | 3 + pkg/libvirttools/cloudinit.go | 8 + pkg/libvirttools/diag_test.go | 2 +- pkg/libvirttools/gc_test.go | 8 +- .../persistentroot_volumesource.go | 214 +++++++++++++++ .../persistentroot_volumesource_test.go | 244 ++++++++++++++++++ pkg/libvirttools/qcow2_flexvolume_test.go | 5 +- pkg/libvirttools/root_volumesource.go | 21 +- pkg/libvirttools/root_volumesource_test.go | 71 +++-- pkg/libvirttools/virtualization.go | 8 +- pkg/libvirttools/virtualization_test.go | 59 ++++- pkg/libvirttools/volumes.go | 11 +- pkg/manager/manager.go | 2 +- pkg/manager/runtime_test.go | 4 +- pkg/utils/command.go | 70 +++++ pkg/utils/command_test.go | 64 +++++ pkg/utils/fake/command.go | 111 ++++++++ 44 files changed, 1270 insertions(+), 92 deletions(-) create mode 100644 examples/cirros-vm-persistent-rootfs.yaml create mode 100755 pkg/libvirttools/TestDomainDefinitions__persistent_rootfs.out.yaml create mode 100755 pkg/libvirttools/TestPersistentRootVolume__first_image_too_big.out.yaml create mode 100755 pkg/libvirttools/TestPersistentRootVolume__image_change.out.yaml create mode 100755 pkg/libvirttools/TestPersistentRootVolume__image_unchanged.out.yaml create mode 100755 pkg/libvirttools/TestPersistentRootVolume__second_image_too_big.out.yaml create mode 100755 pkg/libvirttools/TestPersistentRootVolume__symlinks.out.yaml create mode 100644 pkg/libvirttools/persistentroot_volumesource.go create mode 100644 pkg/libvirttools/persistentroot_volumesource_test.go create mode 100644 pkg/utils/command.go create mode 100644 pkg/utils/command_test.go create mode 100644 pkg/utils/fake/command.go diff --git a/examples/cirros-vm-persistent-rootfs.yaml b/examples/cirros-vm-persistent-rootfs.yaml new file mode 100644 index 000000000..cfc80045d --- /dev/null +++ b/examples/cirros-vm-persistent-rootfs.yaml @@ -0,0 +1,80 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: local-block-pvc +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + storageClassName: local-storage + resources: + requests: + storage: 100Mi +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: local-block-pv +spec: + capacity: + storage: 100Mi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: local-storage + volumeMode: Block + local: + # set up with: + # docker exec kube-node-1 /bin/bash -c 'dd if=/dev/zero of=/rawtest bs=1M count=1000 && losetup -f /rawtest --show' + path: /dev/loop0 + claimRef: + name: local-block-pvc + namespace: default + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - kube-node-1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: cirros-vm-p + annotations: + kubernetes.io/target-runtime: virtlet.cloud + # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason + VirtletDiskDriver: virtio + VirtletSSHKeys: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: extraRuntime + operator: In + values: + - virtlet + # This is the number of seconds Virtlet gives the VM to shut down cleanly. + # The default value of 30 seconds is ok for containers but probably too + # low for VM, so overriding it here is strongly advised. + terminationGracePeriodSeconds: 120 + containers: + - name: cirros-vm + image: virtlet.cloud/cirros + imagePullPolicy: IfNotPresent + # tty and stdin required for `kubectl attach -t` to work + tty: true + stdin: true + volumeDevices: + - devicePath: / + name: testpvc + volumes: + - name: testpvc + persistentVolumeClaim: + claimName: local-block-pvc diff --git a/pkg/image/fake/store.go b/pkg/image/fake/store.go index 0ccc1d3b0..643010e73 100644 --- a/pkg/image/fake/store.go +++ b/pkg/image/fake/store.go @@ -119,13 +119,13 @@ func (s *FakeStore) GC() error { return nil } -// GetImagePathAndVirtualSize implements GC method of Store interface. -func (s *FakeStore) GetImagePathAndVirtualSize(imageName string) (string, uint64, error) { +// GetImagePathDigestAndVirtualSize implements GetImagePathDigestAndVirtualSize method of Store interface. +func (s *FakeStore) GetImagePathDigestAndVirtualSize(imageName string) (string, digest.Digest, uint64, error) { img, found := s.images[imageName] if !found { - return "", 0, fmt.Errorf("image not found: %q", imageName) + return "", "", 0, fmt.Errorf("image not found: %q", imageName) } - return img.Path, img.Size, nil + return img.Path, digest.Digest(img.Digest), img.Size, nil } // SetRefGetter implements SetRefGetter method of Store interface. diff --git a/pkg/image/image.go b/pkg/image/image.go index 659e09dc5..3a59b62f4 100644 --- a/pkg/image/image.go +++ b/pkg/image/image.go @@ -81,10 +81,10 @@ type Store interface { // GC removes all unused or partially downloaded images. GC() error - // GetImagePathAndVirtualSize returns the path to image data - // and virtual size for the specified image. It accepts - // an image reference or a digest. - GetImagePathAndVirtualSize(ref string) (string, uint64, error) + // GetImagePathDigestAndVirtualSize returns the path to image + // data, the digest and the virtual size for the specified + // image. It accepts an image reference or a digest. + GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error) // SetRefGetter sets a function that will be used to determine // the set of images that are currently in use. @@ -472,30 +472,33 @@ func (s *FileStore) GC() error { return nil } -// GetImagePathAndVirtualSize implements GC method of Store interface. -func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, error) { +// GetImagePathDigestAndVirtualSize implements GetImagePathDigestAndVirtualSize method of Store interface. +func (s *FileStore) GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error) { s.Lock() defer s.Unlock() - glog.V(3).Infof("GetImagePathAndVirtualSize(): %q", ref) + glog.V(3).Infof("GetImagePathDigestAndVirtualSize(): %q", ref) var pathViaDigest, pathViaName string // parsing digest as ref gives bad results - if d, err := digest.Parse(ref); err == nil { + d, err := digest.Parse(ref) + if err == nil { if d.Algorithm() != digest.SHA256 { - return "", 0, fmt.Errorf("bad image digest (need sha256): %q", d) + return "", "", 0, fmt.Errorf("bad image digest (need sha256): %q", d) } pathViaDigest = s.dataFileName(d.Hex()) } else { parsed, err := reference.Parse(ref) if err != nil { - return "", 0, fmt.Errorf("bad image reference %q: %v", ref, err) + return "", "", 0, fmt.Errorf("bad image reference %q: %v", ref, err) } + d = "" if digested, ok := parsed.(reference.Digested); ok { if digested.Digest().Algorithm() != digest.SHA256 { - return "", 0, fmt.Errorf("bad image digest (need sha256): %q", digested.Digest()) + return "", "", 0, fmt.Errorf("bad image digest (need sha256): %q", digested.Digest()) } - pathViaDigest = s.dataFileName(digested.Digest().Hex()) + d = digested.Digest() + pathViaDigest = s.dataFileName(d.Hex()) } if named, ok := parsed.(reference.Named); ok && named.Name() != "" { @@ -504,6 +507,7 @@ func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, erro glog.Warningf("error reading link %q: %v", pathViaName, err) } else { pathViaName = filepath.Join(s.linkDir(), pathViaName) + d = digest.NewDigestFromHex(string(digest.SHA256), filepath.Base(pathViaName)) } } } @@ -511,28 +515,28 @@ func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, erro path := pathViaDigest switch { case pathViaDigest == "" && pathViaName == "": - return "", 0, fmt.Errorf("bad image reference %q", ref) + return "", "", 0, fmt.Errorf("bad image reference %q", ref) case pathViaDigest == "": path = pathViaName case pathViaName != "": fi1, err := os.Stat(pathViaName) if err != nil { - return "", 0, err + return "", "", 0, err } fi2, err := os.Stat(pathViaDigest) if err != nil { - return "", 0, err + return "", "", 0, err } if !os.SameFile(fi1, fi2) { - return "", 0, fmt.Errorf("digest / name path mismatch: %q vs %q", pathViaDigest, pathViaName) + return "", "", 0, fmt.Errorf("digest / name path mismatch: %q vs %q", pathViaDigest, pathViaName) } } vsize, err := s.vsizeFunc(path) if err != nil { - return "", 0, fmt.Errorf("error getting image size for %q: %v", path, err) + return "", "", 0, fmt.Errorf("error getting image size for %q: %v", path, err) } - return path, vsize, nil + return path, d, vsize, nil } // SetRefGetter implements SetRefGetter method of Store interface. diff --git a/pkg/image/image_test.go b/pkg/image/image_test.go index 4045a96a5..ce11e8f6e 100644 --- a/pkg/image/image_test.go +++ b/pkg/image/image_test.go @@ -187,9 +187,13 @@ func (tst *ifsTester) verifyListImages(filter string, expectedImages ...*Image) } func (tst *ifsTester) verifyImage(ref string, expectedContents string) { - if path, vsize, err := tst.store.GetImagePathAndVirtualSize(ref); err != nil { + if path, digest, vsize, err := tst.store.GetImagePathDigestAndVirtualSize(ref); err != nil { tst.t.Errorf("GetImagePathAndVirtualSize(): %v", err) } else { + expectedDigest := "sha256:" + sha256str(expectedContents) + if string(digest) != expectedDigest { + tst.t.Errorf("bad digest: %s instead of %s", digest, expectedDigest) + } tst.verifyFileContents(path, expectedContents) expectedVirtualSize := uint64(len(expectedContents)) + 1000 if vsize != expectedVirtualSize { diff --git a/pkg/libvirttools/TestContainerLifecycle.out.yaml b/pkg/libvirttools/TestContainerLifecycle.out.yaml index 08d072b22..72ab84eb0 100755 --- a/pkg/libvirttools/TestContainerLifecycle.out.yaml +++ b/pkg/libvirttools/TestContainerLifecycle.out.yaml @@ -1,6 +1,6 @@ - name: 'domain conn: ListDomains' value: [] -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__ceph_flexvolume.out.yaml b/pkg/libvirttools/TestDomainDefinitions__ceph_flexvolume.out.yaml index 6310c1bd8..56bde94eb 100755 --- a/pkg/libvirttools/TestDomainDefinitions__ceph_flexvolume.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__ceph_flexvolume.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__cloud-init.out.yaml b/pkg/libvirttools/TestDomainDefinitions__cloud-init.out.yaml index 50316c543..800b8c2d2 100755 --- a/pkg/libvirttools/TestDomainDefinitions__cloud-init.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__cloud-init.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__cloud-init_with_user_data.out.yaml b/pkg/libvirttools/TestDomainDefinitions__cloud-init_with_user_data.out.yaml index 0341a7835..c16c158e9 100755 --- a/pkg/libvirttools/TestDomainDefinitions__cloud-init_with_user_data.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__cloud-init_with_user_data.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__persistent_rootfs.out.yaml b/pkg/libvirttools/TestDomainDefinitions__persistent_rootfs.out.yaml new file mode 100755 index 000000000..79b594c80 --- /dev/null +++ b/pkg/libvirttools/TestDomainDefinitions__persistent_rootfs.out.yaml @@ -0,0 +1,87 @@ +- name: GetImagePathDigestAndVirtualSize + value: fake/image1 +- name: CMD + value: + cmd: blockdev --getsz /fakedev/69eec606-0493-5825-73a4-c5e0c0236155/volumeDevices/kubernetes.io~local-volume/root + stdout: "1000" +- name: CMD + value: + cmd: dmsetup create virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155 + stdin: | + 0 999 linear /fakedev/69eec606-0493-5825-73a4-c5e0c0236155/volumeDevices/kubernetes.io~local-volume/root 1 +- name: CMD + value: + cmd: qemu-img convert -O raw /fake/volume/path /dev/mapper/virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155 +- name: 'domain conn: DefineDomain' + value: |- + + virtlet-231700d5-c9a6-container1 + 231700d5-c9a6-5a49-738d-99a954c51550 + 1024 + 1 + + 0 + 0 + 0 + + + hvm + + + + + + destroy + restart + restart + + /vmwrapper + + + + +
+ + + + + + +
+ + +
+ + + + + + + + + + + + + + + + + + + +- name: 'domain conn: virtlet-231700d5-c9a6-container1: Create' +- name: 'domain conn: virtlet-231700d5-c9a6-container1: iso image' + value: + meta-data: '{"instance-id":"testName_0.default","local-hostname":"testName_0"}' + network-config: | + version: 1 + user-data: | + #cloud-config +- name: 'domain conn: virtlet-231700d5-c9a6-container1: Destroy' +- name: 'domain conn: virtlet-231700d5-c9a6-container1: Undefine' +- name: CMD + value: + cmd: dmsetup remove virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155 diff --git a/pkg/libvirttools/TestDomainDefinitions__plain_domain.out.yaml b/pkg/libvirttools/TestDomainDefinitions__plain_domain.out.yaml index 19ccad185..2d5f14752 100755 --- a/pkg/libvirttools/TestDomainDefinitions__plain_domain.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__plain_domain.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__raw_block_volume.out.yaml b/pkg/libvirttools/TestDomainDefinitions__raw_block_volume.out.yaml index 53f4cdb81..61a0abdf4 100755 --- a/pkg/libvirttools/TestDomainDefinitions__raw_block_volume.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__raw_block_volume.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__raw_devices.out.yaml b/pkg/libvirttools/TestDomainDefinitions__raw_devices.out.yaml index 9af2223ac..0f0e2f7dd 100755 --- a/pkg/libvirttools/TestDomainDefinitions__raw_devices.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__raw_devices.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__vcpu_count.out.yaml b/pkg/libvirttools/TestDomainDefinitions__vcpu_count.out.yaml index c833b4629..e8405390c 100755 --- a/pkg/libvirttools/TestDomainDefinitions__vcpu_count.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__vcpu_count.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__virtio_disk_driver.out.yaml b/pkg/libvirttools/TestDomainDefinitions__virtio_disk_driver.out.yaml index a29c38fac..d124fb640 100755 --- a/pkg/libvirttools/TestDomainDefinitions__virtio_disk_driver.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__virtio_disk_driver.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainDefinitions__volumes.out.yaml b/pkg/libvirttools/TestDomainDefinitions__volumes.out.yaml index 687e5221d..3cb2632a1 100755 --- a/pkg/libvirttools/TestDomainDefinitions__volumes.out.yaml +++ b/pkg/libvirttools/TestDomainDefinitions__volumes.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestDomainForcedShutdown.out.yaml b/pkg/libvirttools/TestDomainForcedShutdown.out.yaml index 819efd961..8fbcb25e1 100755 --- a/pkg/libvirttools/TestDomainForcedShutdown.out.yaml +++ b/pkg/libvirttools/TestDomainForcedShutdown.out.yaml @@ -1,4 +1,4 @@ -- name: GetImagePathAndVirtualSize +- name: GetImagePathDigestAndVirtualSize value: fake/image1 - name: 'storage: CreateStoragePool' value: |- diff --git a/pkg/libvirttools/TestPersistentRootVolume__first_image_too_big.out.yaml b/pkg/libvirttools/TestPersistentRootVolume__first_image_too_big.out.yaml new file mode 100755 index 000000000..8ac4313fc --- /dev/null +++ b/pkg/libvirttools/TestPersistentRootVolume__first_image_too_big.out.yaml @@ -0,0 +1,7 @@ +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image1 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "8" diff --git a/pkg/libvirttools/TestPersistentRootVolume__image_change.out.yaml b/pkg/libvirttools/TestPersistentRootVolume__image_change.out.yaml new file mode 100755 index 000000000..682dd354f --- /dev/null +++ b/pkg/libvirttools/TestPersistentRootVolume__image_change.out.yaml @@ -0,0 +1,52 @@ +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image1 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "32" +- name: CMD + value: + cmd: dmsetup create virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 + stdin: | + 0 31 linear /dev/rootdev 1 +- name: CMD + value: + cmd: qemu-img convert -O raw /fake/path1 /dev/mapper/virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end setup -- root disk + value: |- + + + + +- name: teardown +- name: CMD + value: + cmd: dmsetup remove virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end teardown +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image2 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "32" +- name: CMD + value: + cmd: dmsetup create virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 + stdin: | + 0 31 linear /dev/rootdev 1 +- name: CMD + value: + cmd: qemu-img convert -O raw /fake/path2 /dev/mapper/virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end setup -- root disk + value: |- + + + + +- name: teardown +- name: CMD + value: + cmd: dmsetup remove virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end teardown diff --git a/pkg/libvirttools/TestPersistentRootVolume__image_unchanged.out.yaml b/pkg/libvirttools/TestPersistentRootVolume__image_unchanged.out.yaml new file mode 100755 index 000000000..054a43f45 --- /dev/null +++ b/pkg/libvirttools/TestPersistentRootVolume__image_unchanged.out.yaml @@ -0,0 +1,49 @@ +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image1 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "17" +- name: CMD + value: + cmd: dmsetup create virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 + stdin: | + 0 16 linear /dev/rootdev 1 +- name: CMD + value: + cmd: qemu-img convert -O raw /fake/path1 /dev/mapper/virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end setup -- root disk + value: |- + + + + +- name: teardown +- name: CMD + value: + cmd: dmsetup remove virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end teardown +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image1 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "17" +- name: CMD + value: + cmd: dmsetup create virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 + stdin: | + 0 16 linear /dev/rootdev 1 +- name: end setup -- root disk + value: |- + + + + +- name: teardown +- name: CMD + value: + cmd: dmsetup remove virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end teardown diff --git a/pkg/libvirttools/TestPersistentRootVolume__second_image_too_big.out.yaml b/pkg/libvirttools/TestPersistentRootVolume__second_image_too_big.out.yaml new file mode 100755 index 000000000..d572b14e4 --- /dev/null +++ b/pkg/libvirttools/TestPersistentRootVolume__second_image_too_big.out.yaml @@ -0,0 +1,33 @@ +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image1 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "17" +- name: CMD + value: + cmd: dmsetup create virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 + stdin: | + 0 16 linear /dev/rootdev 1 +- name: CMD + value: + cmd: qemu-img convert -O raw /fake/path1 /dev/mapper/virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end setup -- root disk + value: |- + + + + +- name: teardown +- name: CMD + value: + cmd: dmsetup remove virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end teardown +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image2 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "17" diff --git a/pkg/libvirttools/TestPersistentRootVolume__symlinks.out.yaml b/pkg/libvirttools/TestPersistentRootVolume__symlinks.out.yaml new file mode 100755 index 000000000..054a43f45 --- /dev/null +++ b/pkg/libvirttools/TestPersistentRootVolume__symlinks.out.yaml @@ -0,0 +1,49 @@ +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image1 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "17" +- name: CMD + value: + cmd: dmsetup create virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 + stdin: | + 0 16 linear /dev/rootdev 1 +- name: CMD + value: + cmd: qemu-img convert -O raw /fake/path1 /dev/mapper/virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end setup -- root disk + value: |- + + + + +- name: teardown +- name: CMD + value: + cmd: dmsetup remove virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end teardown +- name: setup +- name: 'image: GetImagePathDigestAndVirtualSize' + value: persistent/image1 +- name: CMD + value: + cmd: blockdev --getsz /dev/rootdev + stdout: "17" +- name: CMD + value: + cmd: dmsetup create virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 + stdin: | + 0 16 linear /dev/rootdev 1 +- name: end setup -- root disk + value: |- + + + + +- name: teardown +- name: CMD + value: + cmd: dmsetup remove virtlet-dm-77f29a0e-46af-4188-a6af-9ff8b8a65224 +- name: end teardown diff --git a/pkg/libvirttools/TestRootVolumeLifeCycle.out.yaml b/pkg/libvirttools/TestRootVolumeLifeCycle.out.yaml index 37dbe2008..da4199082 100755 --- a/pkg/libvirttools/TestRootVolumeLifeCycle.out.yaml +++ b/pkg/libvirttools/TestRootVolumeLifeCycle.out.yaml @@ -1,5 +1,5 @@ -- name: 'image: GetImagePathAndVirtualSize' - value: rootfs image name +- name: 'image: GetImagePathDigestAndVirtualSize' + value: fake/image1 - name: 'volumes: CreateStorageVol' value: |- diff --git a/pkg/libvirttools/TestRootVolumeSize__default__zero_.out.yaml b/pkg/libvirttools/TestRootVolumeSize__default__zero_.out.yaml index 1b7d0fbdc..b23fffcc9 100755 --- a/pkg/libvirttools/TestRootVolumeSize__default__zero_.out.yaml +++ b/pkg/libvirttools/TestRootVolumeSize__default__zero_.out.yaml @@ -1,5 +1,5 @@ -- name: 'image: GetImagePathAndVirtualSize' - value: rootfs image name +- name: 'image: GetImagePathDigestAndVirtualSize' + value: fake/image1 - name: 'volumes: CreateStorageVol' value: |- diff --git a/pkg/libvirttools/TestRootVolumeSize__greater_than_fakeImageVirtualSize.out.yaml b/pkg/libvirttools/TestRootVolumeSize__greater_than_fakeImageVirtualSize.out.yaml index 62cb7529c..b08af1795 100755 --- a/pkg/libvirttools/TestRootVolumeSize__greater_than_fakeImageVirtualSize.out.yaml +++ b/pkg/libvirttools/TestRootVolumeSize__greater_than_fakeImageVirtualSize.out.yaml @@ -1,5 +1,5 @@ -- name: 'image: GetImagePathAndVirtualSize' - value: rootfs image name +- name: 'image: GetImagePathDigestAndVirtualSize' + value: fake/image1 - name: 'volumes: CreateStorageVol' value: |- diff --git a/pkg/libvirttools/TestRootVolumeSize__negative.out.yaml b/pkg/libvirttools/TestRootVolumeSize__negative.out.yaml index 1b7d0fbdc..b23fffcc9 100755 --- a/pkg/libvirttools/TestRootVolumeSize__negative.out.yaml +++ b/pkg/libvirttools/TestRootVolumeSize__negative.out.yaml @@ -1,5 +1,5 @@ -- name: 'image: GetImagePathAndVirtualSize' - value: rootfs image name +- name: 'image: GetImagePathDigestAndVirtualSize' + value: fake/image1 - name: 'volumes: CreateStorageVol' value: |- diff --git a/pkg/libvirttools/TestRootVolumeSize__same_as_fakeImageVirtualSize.out.yaml b/pkg/libvirttools/TestRootVolumeSize__same_as_fakeImageVirtualSize.out.yaml index 1b7d0fbdc..b23fffcc9 100755 --- a/pkg/libvirttools/TestRootVolumeSize__same_as_fakeImageVirtualSize.out.yaml +++ b/pkg/libvirttools/TestRootVolumeSize__same_as_fakeImageVirtualSize.out.yaml @@ -1,5 +1,5 @@ -- name: 'image: GetImagePathAndVirtualSize' - value: rootfs image name +- name: 'image: GetImagePathDigestAndVirtualSize' + value: fake/image1 - name: 'volumes: CreateStorageVol' value: |- diff --git a/pkg/libvirttools/TestRootVolumeSize__smaller_than_fakeImageVirtualSize.out.yaml b/pkg/libvirttools/TestRootVolumeSize__smaller_than_fakeImageVirtualSize.out.yaml index 1b7d0fbdc..b23fffcc9 100755 --- a/pkg/libvirttools/TestRootVolumeSize__smaller_than_fakeImageVirtualSize.out.yaml +++ b/pkg/libvirttools/TestRootVolumeSize__smaller_than_fakeImageVirtualSize.out.yaml @@ -1,5 +1,5 @@ -- name: 'image: GetImagePathAndVirtualSize' - value: rootfs image name +- name: 'image: GetImagePathDigestAndVirtualSize' + value: fake/image1 - name: 'volumes: CreateStorageVol' value: |- diff --git a/pkg/libvirttools/block_volumesource.go b/pkg/libvirttools/block_volumesource.go index 269fd84b0..be5ed4c75 100644 --- a/pkg/libvirttools/block_volumesource.go +++ b/pkg/libvirttools/block_volumesource.go @@ -59,6 +59,9 @@ func (v *blockVolume) Teardown() error { func GetBlockVolumes(config *types.VMConfig, owner volumeOwner) ([]VMVolume, error) { var vols []VMVolume for _, dev := range config.VolumeDevices { + if dev.DevicePath == "/" { + continue + } vols = append(vols, &blockVolume{ volumeBase: volumeBase{config, owner}, dev: dev, diff --git a/pkg/libvirttools/cloudinit.go b/pkg/libvirttools/cloudinit.go index b52763b6c..de833d1bf 100644 --- a/pkg/libvirttools/cloudinit.go +++ b/pkg/libvirttools/cloudinit.go @@ -506,6 +506,10 @@ func isRegularFile(path string) bool { func (g *CloudInitGenerator) generateSymlinkScript(volumeMap diskPathMap) string { var symlinkLines []string for _, dev := range g.config.VolumeDevices { + if dev.DevicePath == "/" { + // special case for the persistent rootfs + continue + } dpath, found := volumeMap[dev.UUID()] if !found { glog.Warningf("Couldn't determine the path for device %q inside the VM (target path inside the VM: %q)", dev.HostPath, dev.DevicePath) @@ -523,6 +527,10 @@ func (g *CloudInitGenerator) generateSymlinkScript(volumeMap diskPathMap) string func (g *CloudInitGenerator) fixMounts(volumeMap diskPathMap, mounts []interface{}) []interface{} { devMap := make(map[string]string) for _, dev := range g.config.VolumeDevices { + if dev.DevicePath == "/" { + // special case for the persistent rootfs + continue + } dpath, found := volumeMap[dev.UUID()] if !found { glog.Warningf("Couldn't determine the path for device %q inside the VM (target path inside the VM: %q)", dev.HostPath, dev.DevicePath) diff --git a/pkg/libvirttools/diag_test.go b/pkg/libvirttools/diag_test.go index 399eff4cc..f478d9524 100644 --- a/pkg/libvirttools/diag_test.go +++ b/pkg/libvirttools/diag_test.go @@ -25,7 +25,7 @@ import ( ) func TestDump(t *testing.T) { - ct := newContainerTester(t, testutils.NewToplevelRecorder()) + ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil) defer ct.teardown() sandbox := fakemeta.GetSandboxes(1)[0] diff --git a/pkg/libvirttools/gc_test.go b/pkg/libvirttools/gc_test.go index deee99770..d2a93fc84 100644 --- a/pkg/libvirttools/gc_test.go +++ b/pkg/libvirttools/gc_test.go @@ -37,7 +37,7 @@ var ( ) func TestDomainCleanup(t *testing.T) { - ct := newContainerTester(t, testutils.NewToplevelRecorder()) + ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil) defer ct.teardown() for _, uuid := range randomUUIDs { @@ -74,7 +74,7 @@ func TestDomainCleanup(t *testing.T) { } func TestRootVolumesCleanup(t *testing.T) { - ct := newContainerTester(t, testutils.NewToplevelRecorder()) + ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil) defer ct.teardown() pool, err := ct.virtTool.StoragePool() @@ -116,7 +116,7 @@ func TestRootVolumesCleanup(t *testing.T) { } func TestQcow2VolumesCleanup(t *testing.T) { - ct := newContainerTester(t, testutils.NewToplevelRecorder()) + ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil) defer ct.teardown() pool, err := ct.virtTool.StoragePool() @@ -158,7 +158,7 @@ func TestQcow2VolumesCleanup(t *testing.T) { } func TestConfigISOsCleanup(t *testing.T) { - ct := newContainerTester(t, testutils.NewToplevelRecorder()) + ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil) defer ct.teardown() directory, err := ioutil.TempDir("", "virtlet-tests-") diff --git a/pkg/libvirttools/persistentroot_volumesource.go b/pkg/libvirttools/persistentroot_volumesource.go new file mode 100644 index 000000000..214139a2b --- /dev/null +++ b/pkg/libvirttools/persistentroot_volumesource.go @@ -0,0 +1,214 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libvirttools + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + libvirtxml "github.com/libvirt/libvirt-go-xml" + digest "github.com/opencontainers/go-digest" + + "github.com/Mirantis/virtlet/pkg/metadata/types" +) + +const ( + virtletRootfsMagic = 0x263dbe52ba576702 + virtletRootfsMetadataVersion = 1 + sectorSize = 512 +) + +type virtletRootfsHeader struct { + Magic uint64 + MetadataVersion uint16 + ImageHash [sha256.Size]byte +} + +// persistentRootVolume represents a root volume that can survive the +// deletion of its pod +type persistentRootVolume struct { + volumeBase + dev types.VMVolumeDevice +} + +var _ VMVolume = &persistentRootVolume{} + +func (v *persistentRootVolume) UUID() string { + return v.dev.UUID() +} + +func (v *persistentRootVolume) dmName() string { + return "virtlet-dm-" + v.config.PodSandboxID +} + +func (v *persistentRootVolume) dmPath() string { + return "/dev/mapper/" + v.dmName() +} + +func (v *persistentRootVolume) ensureDevHeaderMatches(imageHash [sha256.Size]byte) (bool, error) { + f, err := os.OpenFile(v.dev.HostPath, os.O_RDWR|os.O_SYNC, 0) + if err != nil { + return false, fmt.Errorf("open %q: %v", v.dev.HostPath, err) + } + defer func() { + if f != nil { + f.Close() + } + }() + + var hdr virtletRootfsHeader + if err := binary.Read(f, binary.BigEndian, &hdr); err != nil { + return false, fmt.Errorf("reading rootfs header: %v", err) + } + + headerMatch := true + switch { + case hdr.Magic != virtletRootfsMagic || hdr.ImageHash != imageHash: + headerMatch = false + if _, err := f.Seek(0, os.SEEK_SET); err != nil { + return false, fmt.Errorf("seek: %v", err) + } + if err := binary.Write(f, binary.BigEndian, virtletRootfsHeader{ + Magic: virtletRootfsMagic, + MetadataVersion: virtletRootfsMetadataVersion, + ImageHash: imageHash, + }); err != nil { + return false, fmt.Errorf("writing rootfs header: %v", err) + } + case hdr.MetadataVersion != virtletRootfsMetadataVersion: + // NOTE: we should handle earlier metadata versions + // after we introduce new ones. But we can't handle + // future metadata versions and any non-matching + // metadata versions are future ones currently, so we + // don't want to lose any data here. + return false, fmt.Errorf("unsupported virtlet root device metadata version %v", hdr.MetadataVersion) + } + + if err := f.Close(); err != nil { + return false, fmt.Errorf("error closing rootfs device: %v", err) + } + f = nil + return headerMatch, nil +} + +func (v *persistentRootVolume) blockDevSizeInSectors() (uint64, error) { + // NOTE: this is also doable via ioctl but this way it's + // shorter (no need for fake non-linux impl, extra interface, + // extra fake impl for it). Some links that may help if we + // decide to use the ioctl later on: + // https://github.com/karelzak/util-linux/blob/master/disk-utils/blockdev.c + // https://github.com/aicodix/smr/blob/24aa589f378827a69a07d220f114c169693dacec/smr.go#L29 + out, err := v.owner.Commander().Command("blockdev", "--getsz", v.dev.HostPath).Run(nil) + if err != nil { + return 0, err + } + nSectors, err := strconv.ParseUint(strings.TrimSpace(string(out)), 10, 64) + if err != nil { + return 0, fmt.Errorf("bad size value returned by blockdev: %q: %v", out, err) + } + return nSectors, nil +} + +func (v *persistentRootVolume) dmCmd(cmd []string, stdin string) error { + dmCmd := v.owner.Commander().Command(cmd[0], cmd[1:]...) + var stdinBytes []byte + if stdin != "" { + stdinBytes = []byte(stdin) + } + _, err := dmCmd.Run(stdinBytes) + return err +} + +func (v *persistentRootVolume) dmSetup(imageSize uint64) error { + nSectors, err := v.blockDevSizeInSectors() + if err != nil { + return err + } + // sector 0 is reserved for the Virtlet metadata + minSectors := (imageSize+sectorSize-1)/sectorSize + 1 + if nSectors < minSectors { + return fmt.Errorf("block device too small for the image: need at least %d bytes (%d sectors) but got %d bytes (%d sectors)", + minSectors*sectorSize, + minSectors, + nSectors*sectorSize, + nSectors) + } + hostPath, err := filepath.EvalSymlinks(v.dev.HostPath) + if err != nil { + return err + } + dmTable := fmt.Sprintf("0 %d linear %s 1\n", nSectors-1, hostPath) + dmCmd := v.owner.Commander().Command("dmsetup", "create", v.dmName()) + _, err = dmCmd.Run([]byte(dmTable)) + return err +} + +func (v *persistentRootVolume) copyImageToDev(imagePath string) error { + _, err := v.owner.Commander().Command("qemu-img", "convert", "-O", "raw", imagePath, v.dmPath()).Run(nil) + return err +} + +func (v *persistentRootVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) { + imagePath, imageDigest, imageSize, err := v.owner.ImageManager().GetImagePathDigestAndVirtualSize(v.config.Image) + if err != nil { + return nil, nil, err + } + + if imageDigest.Algorithm() != digest.SHA256 { + return nil, nil, fmt.Errorf("unsupported digest algorithm %q", imageDigest.Algorithm()) + } + imageHash, err := hex.DecodeString(imageDigest.Hex()) + if err != nil { + return nil, nil, fmt.Errorf("bad digest hex: %q", imageDigest.Hex()) + } + if len(imageHash) != sha256.Size { + return nil, nil, fmt.Errorf("bad digest size: %q", imageDigest.Hex()) + } + + var hash [sha256.Size]byte + copy(hash[:], imageHash) + headerMatches, err := v.ensureDevHeaderMatches(hash) + + if err == nil { + err = v.dmSetup(imageSize) + } + + if err == nil && !headerMatches { + err = v.copyImageToDev(imagePath) + } + + if err != nil { + return nil, nil, err + } + + return &libvirtxml.DomainDisk{ + Device: "disk", + Source: &libvirtxml.DomainDiskSource{Block: &libvirtxml.DomainDiskSourceBlock{Dev: v.dmPath()}}, //hostPath}}, + Driver: &libvirtxml.DomainDiskDriver{Name: "qemu", Type: "raw"}, + }, nil, nil +} + +func (v *persistentRootVolume) Teardown() error { + _, err := v.owner.Commander().Command("dmsetup", "remove", v.dmName()).Run(nil) + return err +} diff --git a/pkg/libvirttools/persistentroot_volumesource_test.go b/pkg/libvirttools/persistentroot_volumesource_test.go new file mode 100644 index 000000000..df48a61ff --- /dev/null +++ b/pkg/libvirttools/persistentroot_volumesource_test.go @@ -0,0 +1,244 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libvirttools + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/Mirantis/virtlet/tests/gm" + digest "github.com/opencontainers/go-digest" + + "github.com/Mirantis/virtlet/pkg/metadata/types" + fakeutils "github.com/Mirantis/virtlet/pkg/utils/fake" + testutils "github.com/Mirantis/virtlet/pkg/utils/testing" +) + +func TestPersistentRootVolume(t *testing.T) { + fakeImages := []fakeImageSpec{ + { + name: "persistent/image1", + path: "/fake/path1", + digest: digest.Digest("sha256:12b05d23a781e4aae1ab9a7de27721cbd1f1d666cfb4e21ab31338eb96eb1e3f"), + size: 8192, + }, + { + name: "persistent/image2", + path: "/fake/path2", + digest: digest.Digest("sha256:d66ab8e0ea2931d41e27ba4f1d9c007a1d43ab883158a8a22f90872a8d9bb0e3"), + size: 10000, + }, + } + for _, tc := range []struct { + name string + imageName string + secondImageName string + dmPath string + fileSize uint64 + imageWrittenAgain bool + useSymlink bool + errors [2]string + }{ + { + name: "image unchanged", + imageName: "persistent/image1", + secondImageName: "persistent/image1", + fileSize: 8704, // just added a sector + }, + { + name: "image change", + imageName: "persistent/image1", + secondImageName: "persistent/image2", + fileSize: 16384, + imageWrittenAgain: true, + }, + { + name: "first image too big", + imageName: "persistent/image1", + fileSize: 4096, + errors: [2]string{ + "too small", + "", + }, + }, + { + name: "second image too big", + imageName: "persistent/image1", + secondImageName: "persistent/image2", + fileSize: 8704, + errors: [2]string{ + "", + "too small", + }, + }, + { + name: "symlinks", + imageName: "persistent/image1", + secondImageName: "persistent/image1", + fileSize: 8704, + useSymlink: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + rec := testutils.NewToplevelRecorder() + im := newFakeImageManager(rec.Child("image"), fakeImages...) + if tc.fileSize%512 != 0 { + t.Fatalf("block device size must be a multiple of 512") + } + + tmpDir, err := ioutil.TempDir("", "fake-persistent-rootfs") + if err != nil { + t.Fatalf("TempDir(): %v", err) + } + defer os.RemoveAll(tmpDir) + fakeDevDir := filepath.Join(tmpDir, "__dev__") + if err := os.Mkdir(fakeDevDir, 0777); err != nil { + t.Fatalf("Mkdir(): %v", err) + } + + devPath := filepath.Join(fakeDevDir, "rootdev") + devFile, err := os.Create(devPath) + if err != nil { + t.Fatalf("Create(): %v", err) + } + if _, err := devFile.Write(make([]byte, tc.fileSize)); err != nil { + devFile.Close() + t.Fatalf("Write(): %v", err) + } + if err := devFile.Close(); err != nil { + t.Fatalf("devFile.Close()") + } + devPathToUse := devPath + if tc.useSymlink { + devPathToUse := filepath.Join(fakeDevDir, "rootdevlink") + if err := os.Symlink(devPath, devPathToUse); err != nil { + t.Fatalf("Symlink(): %v", err) + } + } + + for n, imageName := range []string{tc.imageName, tc.secondImageName} { + if imageName == "" { + continue + } + cmdSpecs := []fakeutils.CmdSpec{ + { + Match: "blockdev --getsz", + Stdout: strconv.Itoa(int(tc.fileSize / 512)), + }, + { + Match: "dmsetup create", + }, + { + Match: "dmsetup remove", + }, + } + if n == 0 || tc.imageWrittenAgain { + // qemu-img convert is used to write the image to the block device. + // It should only be called if the image changes. + cmdSpecs = append(cmdSpecs, fakeutils.CmdSpec{ + Match: "qemu-img convert", + }) + } + cmd := fakeutils.NewCommander(rec, cmdSpecs) + cmd.ReplaceTempPath("__dev__", "/dev") + owner := newFakeVolumeOwner(nil, im, cmd) + rootVol := getPersistentRootVolume(t, imageName, devPathToUse, owner) + verifyRootVolumeSetup(t, rec, rootVol, tc.errors[n]) + if tc.errors[n] == "" { + verifyRootVolumeTeardown(t, rec, rootVol) + } + } + gm.Verify(t, gm.NewYamlVerifier(rec.Content())) + }) + } +} + +func verifyRootVolumeSetup(t *testing.T, rec testutils.Recorder, rootVol *persistentRootVolume, expectedError string) { + rec.Rec("setup", nil) + vol, fs, err := rootVol.Setup() + if expectedError == "" { + if err != nil { + t.Fatalf("Setup returned an unexpected error: %v", err) + } + } else { + switch { + case err == nil: + t.Errorf("Setup didn't return the expected error") + case !strings.Contains(err.Error(), expectedError): + t.Errorf("Setup returned a wrong error message %q (must contain %q)", err, expectedError) + } + return + } + + if fs != nil { + t.Errorf("Didn't expect a filesystem") + } + + if vol.Source.Block == nil { + t.Errorf("Expected 'block' volume type") + } + + if vol.Device != "disk" { + t.Errorf("Expected 'disk' as volume device, received: %s", vol.Device) + } + + expectedDmPath := "/dev/mapper/virtlet-dm-" + testUUID + if vol.Source.Block.Dev != expectedDmPath { + t.Errorf("Expected '%s' as root block device path, received: %s", expectedDmPath, vol.Source.Block.Dev) + } + + out, err := vol.Marshal() + if err != nil { + t.Fatalf("error marshalling the volume: %v", err) + } + rec.Rec("end setup -- root disk", out) +} + +func verifyRootVolumeTeardown(t *testing.T, rec testutils.Recorder, rootVol *persistentRootVolume) { + rec.Rec("teardown", nil) + if err := rootVol.Teardown(); err != nil { + t.Fatalf("Teardown(): %v", err) + } + rec.Rec("end teardown", nil) +} + +func getPersistentRootVolume(t *testing.T, imageName, devHostPath string, owner volumeOwner) *persistentRootVolume { + volumes, err := GetRootVolume( + &types.VMConfig{ + PodSandboxID: testUUID, + Image: imageName, + VolumeDevices: []types.VMVolumeDevice{ + { + DevicePath: "/", + HostPath: devHostPath, + }, + }, + }, owner) + if err != nil { + t.Fatalf("GetRootVolume returned an error: %v", err) + } + + if len(volumes) != 1 { + t.Fatalf("GetRootVolumes returned non single number of volumes: %d", len(volumes)) + } + + return volumes[0].(*persistentRootVolume) +} diff --git a/pkg/libvirttools/qcow2_flexvolume_test.go b/pkg/libvirttools/qcow2_flexvolume_test.go index 413411d8d..f210cce09 100644 --- a/pkg/libvirttools/qcow2_flexvolume_test.go +++ b/pkg/libvirttools/qcow2_flexvolume_test.go @@ -24,6 +24,7 @@ import ( libvirtxml "github.com/libvirt/libvirt-go-xml" "github.com/Mirantis/virtlet/pkg/metadata/types" + fakeutils "github.com/Mirantis/virtlet/pkg/utils/fake" testutils "github.com/Mirantis/virtlet/pkg/utils/testing" "github.com/Mirantis/virtlet/pkg/virt/fake" "github.com/Mirantis/virtlet/tests/gm" @@ -73,7 +74,7 @@ func TestQCOW2VolumeLifeCycle(t *testing.T) { Target: &libvirtxml.StoragePoolTarget{Path: volumesPoolPath}, }) - im := NewFakeImageManager(rec.Child("image")) + im := newFakeImageManager(rec.Child("image")) optsFilePath, err := prepareOptsFileForQcow2Volume() if err != nil { @@ -85,7 +86,7 @@ func TestQCOW2VolumeLifeCycle(t *testing.T) { TestVolumeName, optsFilePath, &types.VMConfig{DomainUUID: testUUID, Image: "rootfs image name"}, - newFakeVolumeOwner(spool, im), + newFakeVolumeOwner(spool, im, fakeutils.NewCommander(nil, nil)), ) if err != nil { t.Fatalf("newQCOW2Volume returned an error: %v", err) diff --git a/pkg/libvirttools/root_volumesource.go b/pkg/libvirttools/root_volumesource.go index 99a55b370..de379df5b 100644 --- a/pkg/libvirttools/root_volumesource.go +++ b/pkg/libvirttools/root_volumesource.go @@ -34,11 +34,22 @@ var _ VMVolume = &rootVolume{} // GetRootVolume returns volume source for root volume clone. func GetRootVolume(config *types.VMConfig, owner volumeOwner) ([]VMVolume, error) { - return []VMVolume{ - &rootVolume{ + var vol VMVolume + for _, dev := range config.VolumeDevices { + if dev.DevicePath == "/" { + vol = &persistentRootVolume{ + volumeBase: volumeBase{config, owner}, + dev: dev, + } + break + } + } + if vol == nil { + vol = &rootVolume{ volumeBase{config, owner}, - }, - }, nil + } + } + return []VMVolume{vol}, nil } func (v *rootVolume) volumeName() string { @@ -46,7 +57,7 @@ func (v *rootVolume) volumeName() string { } func (v *rootVolume) createVolume() (virt.StorageVolume, error) { - imagePath, virtualSize, err := v.owner.ImageManager().GetImagePathAndVirtualSize(v.config.Image) + imagePath, _, virtualSize, err := v.owner.ImageManager().GetImagePathDigestAndVirtualSize(v.config.Image) if err != nil { return nil, err } diff --git a/pkg/libvirttools/root_volumesource_test.go b/pkg/libvirttools/root_volumesource_test.go index 4f03aec31..d663ff522 100644 --- a/pkg/libvirttools/root_volumesource_test.go +++ b/pkg/libvirttools/root_volumesource_test.go @@ -17,12 +17,15 @@ limitations under the License. package libvirttools import ( + "fmt" "testing" libvirtxml "github.com/libvirt/libvirt-go-xml" + digest "github.com/opencontainers/go-digest" "github.com/Mirantis/virtlet/pkg/metadata/types" "github.com/Mirantis/virtlet/pkg/utils" + fakeutils "github.com/Mirantis/virtlet/pkg/utils/fake" testutils "github.com/Mirantis/virtlet/pkg/utils/testing" "github.com/Mirantis/virtlet/pkg/virt" "github.com/Mirantis/virtlet/pkg/virt/fake" @@ -34,26 +37,49 @@ const ( fakeImageVirtualSize = 424242 fakeImageStoreUsedBytes = 424242 fakeImageStoreUsedInodes = 424242 + fakeImageDigest = digest.Digest("sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2") ) -type FakeImageManager struct { - rec testutils.Recorder +type fakeImageSpec struct { + name string + path string + digest digest.Digest + size uint64 } -var _ ImageManager = &FakeImageManager{} +type fakeImageManager struct { + rec testutils.Recorder + imageMap map[string]fakeImageSpec +} -func NewFakeImageManager(rec testutils.Recorder) *FakeImageManager { - return &FakeImageManager{ - rec: rec, +var _ ImageManager = &fakeImageManager{} + +func newFakeImageManager(rec testutils.Recorder, extraImages ...fakeImageSpec) *fakeImageManager { + m := make(map[string]fakeImageSpec) + for _, img := range append(extraImages, fakeImageSpec{ + name: "fake/image1", + path: "/fake/volume/path", + digest: fakeImageDigest, + size: fakeImageVirtualSize, + }) { + m[img.name] = img + } + return &fakeImageManager{ + rec: rec, + imageMap: m, } } -func (im *FakeImageManager) GetImagePathAndVirtualSize(imageName string) (string, uint64, error) { - im.rec.Rec("GetImagePathAndVirtualSize", imageName) - return "/fake/volume/path", fakeImageVirtualSize, nil +func (im *fakeImageManager) GetImagePathDigestAndVirtualSize(imageName string) (string, digest.Digest, uint64, error) { + im.rec.Rec("GetImagePathDigestAndVirtualSize", imageName) + spec, found := im.imageMap[imageName] + if !found { + return "", "", 0, fmt.Errorf("image %q not found", imageName) + } + return spec.path, spec.digest, spec.size, nil } -func (im *FakeImageManager) FilesystemStats() (*types.FilesystemStats, error) { +func (im *fakeImageManager) FilesystemStats() (*types.FilesystemStats, error) { return &types.FilesystemStats{ Mountpoint: "/some/dir", UsedBytes: fakeImageStoreUsedBytes, @@ -61,7 +87,7 @@ func (im *FakeImageManager) FilesystemStats() (*types.FilesystemStats, error) { }, nil } -func (im *FakeImageManager) BytesUsedBy(path string) (uint64, error) { +func (im *fakeImageManager) BytesUsedBy(path string) (uint64, error) { im.rec.Rec("BytesUsedBy", path) return fakeImageVirtualSize, nil } @@ -87,12 +113,11 @@ func getRootVolumeForTest(t *testing.T, vmConfig *types.VMConfig) (*rootVolume, Name: "volumes", Target: &libvirtxml.StoragePoolTarget{Path: volumesPoolPath}, }) - im := NewFakeImageManager(rec.Child("image")) + im := newFakeImageManager(rec.Child("image")) volumes, err := GetRootVolume( vmConfig, - newFakeVolumeOwner(spool, im), - ) + newFakeVolumeOwner(spool, im, fakeutils.NewCommander(nil, nil))) if err != nil { t.Fatalf("GetRootVolume returned an error: %v", err) } @@ -139,7 +164,7 @@ func TestRootVolumeSize(t *testing.T) { t.Run(tc.name, func(t *testing.T) { rootVol, rec, spool := getRootVolumeForTest(t, &types.VMConfig{ DomainUUID: testUUID, - Image: "rootfs image name", + Image: "fake/image1", ParsedAnnotations: &types.VirtletAnnotations{ RootVolumeSize: tc.specifiedRootVolumeSize, }, @@ -172,14 +197,18 @@ func TestRootVolumeLifeCycle(t *testing.T) { expectedRootVolumePath := "/fake/volumes/pool/virtlet_root_" + testUUID rootVol, rec, _ := getRootVolumeForTest(t, &types.VMConfig{ DomainUUID: testUUID, - Image: "rootfs image name", + Image: "fake/image1", }) - vol, _, err := rootVol.Setup() + vol, fs, err := rootVol.Setup() if err != nil { t.Fatalf("Setup returned an error: %v", err) } + if fs != nil { + t.Errorf("Didn't expect a filesystem") + } + if vol.Source.File == nil { t.Errorf("Expected 'file' volume type") } @@ -207,15 +236,17 @@ func TestRootVolumeLifeCycle(t *testing.T) { type fakeVolumeOwner struct { storagePool *fake.FakeStoragePool - imageManager *FakeImageManager + imageManager *fakeImageManager + commander *fakeutils.Commander } var _ volumeOwner = fakeVolumeOwner{} -func newFakeVolumeOwner(storagePool *fake.FakeStoragePool, imageManager *FakeImageManager) *fakeVolumeOwner { +func newFakeVolumeOwner(storagePool *fake.FakeStoragePool, imageManager *fakeImageManager, commander *fakeutils.Commander) *fakeVolumeOwner { return &fakeVolumeOwner{ storagePool: storagePool, imageManager: imageManager, + commander: commander, } } @@ -240,3 +271,5 @@ func (vo fakeVolumeOwner) VolumePoolName() string { return "" } func (vo fakeVolumeOwner) Mounter() utils.Mounter { return utils.NullMounter } func (vo fakeVolumeOwner) SharedFilesystemPath() string { return "/var/lib/virtlet/fs" } + +func (vo fakeVolumeOwner) Commander() utils.Commander { return vo.commander } diff --git a/pkg/libvirttools/virtualization.go b/pkg/libvirttools/virtualization.go index 4c3f59d07..d5803e210 100644 --- a/pkg/libvirttools/virtualization.go +++ b/pkg/libvirttools/virtualization.go @@ -225,6 +225,7 @@ type VirtualizationTool struct { volumeSource VMVolumeSource config VirtualizationConfig mounter utils.Mounter + commander utils.Commander } var _ volumeOwner = &VirtualizationTool{} @@ -234,7 +235,8 @@ var _ volumeOwner = &VirtualizationTool{} func NewVirtualizationTool(domainConn virt.DomainConnection, storageConn virt.StorageConnection, imageManager ImageManager, metadataStore metadata.Store, volumeSource VMVolumeSource, - config VirtualizationConfig, mounter utils.Mounter) *VirtualizationTool { + config VirtualizationConfig, mounter utils.Mounter, + commander utils.Commander) *VirtualizationTool { return &VirtualizationTool{ domainConn: domainConn, storageConn: storageConn, @@ -244,6 +246,7 @@ func NewVirtualizationTool(domainConn virt.DomainConnection, volumeSource: volumeSource, config: config, mounter: mounter, + commander: commander, } } @@ -927,6 +930,9 @@ func (v *VirtualizationTool) Mounter() utils.Mounter { return v.mounter } // SharedFilesystemPath implements volumeOwner SharedFilesystemPath method func (v *VirtualizationTool) SharedFilesystemPath() string { return v.config.SharedFilesystemPath } +// Commander implements volumeOwner Commander method +func (v *VirtualizationTool) Commander() utils.Commander { return v.commander } + func filterContainer(containerInfo *types.ContainerInfo, filter types.ContainerFilter) bool { if filter.Id != "" && containerInfo.Id != filter.Id { return false diff --git a/pkg/libvirttools/virtualization_test.go b/pkg/libvirttools/virtualization_test.go index 213840b60..dbd6f375e 100644 --- a/pkg/libvirttools/virtualization_test.go +++ b/pkg/libvirttools/virtualization_test.go @@ -33,6 +33,7 @@ import ( fakemeta "github.com/Mirantis/virtlet/pkg/metadata/fake" "github.com/Mirantis/virtlet/pkg/metadata/types" "github.com/Mirantis/virtlet/pkg/utils" + fakeutils "github.com/Mirantis/virtlet/pkg/utils/fake" testutils "github.com/Mirantis/virtlet/pkg/utils/testing" "github.com/Mirantis/virtlet/pkg/virt/fake" "github.com/Mirantis/virtlet/tests/gm" @@ -59,7 +60,7 @@ type containerTester struct { metadataStore metadata.Store } -func newContainerTester(t *testing.T, rec *testutils.TopLevelRecorder) *containerTester { +func newContainerTester(t *testing.T, rec *testutils.TopLevelRecorder, cmds []fakeutils.CmdSpec) *containerTester { ct := &containerTester{ t: t, clock: clockwork.NewFakeClockAt(time.Date(2017, 5, 30, 20, 19, 0, 0, time.UTC)), @@ -83,7 +84,7 @@ func newContainerTester(t *testing.T, rec *testutils.TopLevelRecorder) *containe t.Fatalf("Failed to create fake bolt client: %v", err) } - imageManager := NewFakeImageManager(ct.rec) + imageManager := newFakeImageManager(ct.rec) ct.kubeletRootDir = filepath.Join(ct.tmpDir, "kubelet-root") virtConfig := VirtualizationConfig{ VolumePoolName: "volumes", @@ -91,7 +92,11 @@ func newContainerTester(t *testing.T, rec *testutils.TopLevelRecorder) *containe KubeletRootDir: ct.kubeletRootDir, StreamerSocketPath: "/var/lib/libvirt/streamer.sock", } - ct.virtTool = NewVirtualizationTool(ct.domainConn, ct.storageConn, imageManager, ct.metadataStore, GetDefaultVolumeSource(), virtConfig, utils.NullMounter) + fakeCommander := fakeutils.NewCommander(rec, cmds) + fakeCommander.ReplaceTempPath("__pods__", "/fakedev") + ct.virtTool = NewVirtualizationTool( + ct.domainConn, ct.storageConn, imageManager, ct.metadataStore, + GetDefaultVolumeSource(), virtConfig, utils.NullMounter, fakeCommander) ct.virtTool.SetClock(ct.clock) return ct @@ -181,7 +186,7 @@ func (ct *containerTester) verifyContainerRootfsExists(containerInfo *types.Cont } func TestContainerLifecycle(t *testing.T) { - ct := newContainerTester(t, testutils.NewToplevelRecorder()) + ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil) defer ct.teardown() sandbox := fakemeta.GetSandboxes(1)[0] @@ -263,7 +268,7 @@ func TestContainerLifecycle(t *testing.T) { } func TestDomainForcedShutdown(t *testing.T) { - ct := newContainerTester(t, testutils.NewToplevelRecorder()) + ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil) defer ct.teardown() sandbox := fakemeta.GetSandboxes(1)[0] @@ -298,7 +303,7 @@ func TestDomainForcedShutdown(t *testing.T) { } func TestDoubleStartError(t *testing.T) { - ct := newContainerTester(t, testutils.NewToplevelRecorder()) + ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil) defer ct.teardown() sandbox := fakemeta.GetSandboxes(1)[0] @@ -320,6 +325,7 @@ type volMount struct { type volDevice struct { name string devicePath string + size int } func TestDomainDefinitions(t *testing.T) { @@ -333,6 +339,7 @@ func TestDomainDefinitions(t *testing.T) { flexVolumes map[string]map[string]interface{} mounts []volMount volDevs []volDevice + cmds []fakeutils.CmdSpec }{ { name: "plain domain", @@ -418,11 +425,36 @@ func TestDomainDefinitions(t *testing.T) { "VirtletDiskDriver": "virtio", }, }, + { + name: "persistent rootfs", + volDevs: []volDevice{ + { + name: "root", + devicePath: "/", + size: 512000, + }, + }, + cmds: []fakeutils.CmdSpec{ + { + Match: "blockdev --getsz", + Stdout: "1000", + }, + { + Match: "qemu-img convert", + }, + { + Match: "dmsetup create", + }, + { + Match: "dmsetup remove", + }, + }, + }, } { t.Run(tc.name, func(t *testing.T) { rec := testutils.NewToplevelRecorder() - ct := newContainerTester(t, rec) + ct := newContainerTester(t, rec, tc.cmds) defer ct.teardown() sandbox := fakemeta.GetSandboxes(1)[0] @@ -458,10 +490,17 @@ func TestDomainDefinitions(t *testing.T) { t.Fatal(err) } hostPath := filepath.Join(baseDir, d.name) - if f, err := os.OpenFile(hostPath, os.O_RDONLY|os.O_CREATE, 0666); err != nil { + if f, err := os.Create(hostPath); err != nil { t.Fatal(err) } else { - f.Close() + if d.size != 0 { + if _, err := f.Write(make([]byte, d.size)); err != nil { + t.Fatal(err) + } + } + if err := f.Close(); err != nil { + t.Fatal(err) + } } volDevs = append(volDevs, types.VMVolumeDevice{ DevicePath: d.devicePath, @@ -489,7 +528,7 @@ func TestDomainResourceConstraints(t *testing.T) { rec := testutils.NewToplevelRecorder() rec.AddFilter("DefineDomain") - ct := newContainerTester(t, rec) + ct := newContainerTester(t, rec, nil) defer ct.teardown() sandbox := fakemeta.GetSandboxes(1)[0] sandbox.Annotations = map[string]string{ diff --git a/pkg/libvirttools/volumes.go b/pkg/libvirttools/volumes.go index 41ef7e3b1..d9d7c395f 100644 --- a/pkg/libvirttools/volumes.go +++ b/pkg/libvirttools/volumes.go @@ -18,16 +18,22 @@ package libvirttools import ( libvirtxml "github.com/libvirt/libvirt-go-xml" + digest "github.com/opencontainers/go-digest" "github.com/Mirantis/virtlet/pkg/metadata/types" "github.com/Mirantis/virtlet/pkg/utils" "github.com/Mirantis/virtlet/pkg/virt" ) -// ImageManager describes a images info provider. +// ImageManager describes an image info provider. type ImageManager interface { - GetImagePathAndVirtualSize(ref string) (string, uint64, error) + // GetImagePathDigestAndVirtualSize returns the path, image + // digest ("sha256:...") and the size in bytes for the + // specified image. + GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error) + // FilesystemStats returns filesystem statistics for the specified image. FilesystemStats() (*types.FilesystemStats, error) + // BytesUsedBy returns the size of the specified file. BytesUsedBy(path string) (uint64, error) } @@ -40,6 +46,7 @@ type volumeOwner interface { VolumePoolName() string Mounter() utils.Mounter SharedFilesystemPath() string + Commander() utils.Commander } // VMVolumeSource is a function that provides `VMVolume`s for VMs diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 57d8bc405..3dbfa5d2c 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -135,7 +135,7 @@ func (v *VirtletManager) Run() error { } volSrc := libvirttools.GetDefaultVolumeSource() - v.virtTool = libvirttools.NewVirtualizationTool(conn, conn, v.imageStore, v.metadataStore, volSrc, virtConfig, utils.NewMounter()) + v.virtTool = libvirttools.NewVirtualizationTool(conn, conn, v.imageStore, v.metadataStore, volSrc, virtConfig, utils.NewMounter(), utils.DefaultCommander) runtimeService := NewVirtletRuntimeService(v.virtTool, v.metadataStore, v.fdManager, streamServer, v.imageStore, nil) imageService := NewVirtletImageService(v.imageStore, translator, nil) diff --git a/pkg/manager/runtime_test.go b/pkg/manager/runtime_test.go index e11918365..4bcd77c40 100644 --- a/pkg/manager/runtime_test.go +++ b/pkg/manager/runtime_test.go @@ -45,6 +45,7 @@ import ( "github.com/Mirantis/virtlet/pkg/network" "github.com/Mirantis/virtlet/pkg/tapmanager" "github.com/Mirantis/virtlet/pkg/utils" + fakeutils "github.com/Mirantis/virtlet/pkg/utils/fake" testutils "github.com/Mirantis/virtlet/pkg/utils/testing" fakevirt "github.com/Mirantis/virtlet/pkg/virt/fake" "github.com/Mirantis/virtlet/tests/criapi" @@ -236,7 +237,8 @@ func makeVirtletCRITester(t *testing.T) *virtletCRITester { KubeletRootDir: kubeletRootDir, StreamerSocketPath: streamerSocketPath, } - virtTool := libvirttools.NewVirtualizationTool(domainConn, storageConn, imageStore, metadataStore, libvirttools.GetDefaultVolumeSource(), virtConfig, utils.NullMounter) + commander := fakeutils.NewCommander(rec, nil) + virtTool := libvirttools.NewVirtualizationTool(domainConn, storageConn, imageStore, metadataStore, libvirttools.GetDefaultVolumeSource(), virtConfig, utils.NullMounter, commander) virtTool.SetClock(clock) streamServer := newFakeStreamServer(rec.Child("streamServer")) criHandler := &criHandler{ diff --git a/pkg/utils/command.go b/pkg/utils/command.go new file mode 100644 index 000000000..d781a75da --- /dev/null +++ b/pkg/utils/command.go @@ -0,0 +1,70 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "bytes" + "fmt" + "os/exec" + "strings" +) + +// Command represents an external command being prepared or run. +type Command interface { + // Output runs the command and returns its standard output. + // If stdin is non-nil, it's passed to the command's standed + // input. Any returned error will usually be of type + // *ExitError with ExitError.Stderr containing the stderr + // output of the command. + Run(stdin []byte) ([]byte, error) +} + +// Commander is used to run external commands. +type Commander interface { + // Command returns an isntance of Command to execute the named + // program with the given arguments. + Command(name string, arg ...string) Command +} + +type realCommand struct { + *exec.Cmd +} + +func (c realCommand) Run(stdin []byte) ([]byte, error) { + if stdin != nil { + c.Stdin = bytes.NewBuffer(stdin) + } + out, err := c.Cmd.Output() + if err == nil { + return out, nil + } + fullCmd := strings.Join(append([]string{c.Path}, c.Args...), " ") + if exitErr, ok := err.(*exec.ExitError); ok { + return out, fmt.Errorf("command %s: %v: %s", fullCmd, exitErr, exitErr.Stderr) + } + return out, fmt.Errorf("command %s: %v", fullCmd, err) +} + +type defaultCommander struct{} + +func (c *defaultCommander) Command(name string, arg ...string) Command { + return realCommand{exec.Command(name, arg...)} +} + +// DefaultCommander is an implementation of Commander that runs the +// commands using exec.Command. +var DefaultCommander Commander = &defaultCommander{} diff --git a/pkg/utils/command_test.go b/pkg/utils/command_test.go new file mode 100644 index 000000000..8cdab4252 --- /dev/null +++ b/pkg/utils/command_test.go @@ -0,0 +1,64 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "strings" + "testing" +) + +func TestCommand(t *testing.T) { + for _, tc := range []struct { + cmd []string + stdin []byte + expectedOutput string + expectedError string + }{ + { + cmd: []string{"echo", "-n", "foobar"}, + stdin: nil, + expectedOutput: "foobar", + }, + { + cmd: []string{"/bin/bash", "-c", "echo -n >&2 'stderr here'; echo -n 'stdout here'; exit 1"}, + stdin: nil, + expectedOutput: "stdout here", + expectedError: "stderr here", + }, + { + cmd: []string{"sed", "s/foo/bar/g"}, + stdin: []byte("here is foo; foo."), + expectedOutput: "here is bar; bar.", + }, + } { + t.Run(strings.Join(tc.cmd, " "), func(t *testing.T) { + c := DefaultCommander.Command(tc.cmd[0], tc.cmd[1:]...) + switch out, err := c.Run(tc.stdin); { + case tc.expectedError == "" && err != nil: + t.Errorf("Command error: %v", err) + case tc.expectedError != "" && err == nil: + t.Errorf("Didn't get the expected error") + case tc.expectedError != "" && !strings.Contains(err.Error(), tc.expectedError): + t.Errorf("Bad error message %q (no substring %q)", err.Error(), tc.expectedError) + case tc.expectedError != "" && !strings.Contains(err.Error(), tc.cmd[0]): + t.Errorf("Bad error message %q (no substring %q)", err.Error(), tc.cmd[0]) + case string(out) != tc.expectedOutput: + t.Errorf("Command output mismatch: %q instead of %q", out, tc.expectedOutput) + } + }) + } +} diff --git a/pkg/utils/fake/command.go b/pkg/utils/fake/command.go new file mode 100644 index 000000000..12468eecf --- /dev/null +++ b/pkg/utils/fake/command.go @@ -0,0 +1,111 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "fmt" + "regexp" + "strings" + + "github.com/Mirantis/virtlet/pkg/utils" + testutils "github.com/Mirantis/virtlet/pkg/utils/testing" +) + +// CmdSpec specifies a command for the fake Commander +type CmdSpec struct { + // Regular expression to match against the " "-joined command + Match string + // Stdout to return from the command + Stdout string +} + +type fakeCommand struct { + rec testutils.Recorder + cmd []string + commander *Commander +} + +var _ utils.Command = &fakeCommand{} + +func (c *fakeCommand) subst(text string) string { + if c.commander.replacePath == nil { + return text + } + return c.commander.replacePath.ReplaceAllString(text, c.commander.replacement) +} + +func (c *fakeCommand) Run(stdin []byte) ([]byte, error) { + fullCmd := c.subst(strings.Join(c.cmd, " ")) + r := map[string]string{ + "cmd": fullCmd, + } + defer c.rec.Rec("CMD", r) + for _, spec := range c.commander.specs { + matched, err := regexp.MatchString(spec.Match, fullCmd) + if err != nil { + return nil, fmt.Errorf("failed to match regexp %q: %v", spec.Match, err) + } + if matched { + if c.rec != nil { + if stdin != nil { + r["stdin"] = c.subst(string(stdin)) + } + if spec.Stdout != "" { + r["stdout"] = spec.Stdout + } + } + return []byte(spec.Stdout), nil + } + } + return nil, fmt.Errorf("unexpected command %q", fullCmd) +} + +// Commander records the commands instead of executing them. It +// also provides stdout based on a table of (cmd_regexp, stdout) +// pairs. The regexp is matched against the command and its arguments +// joined with " ". +type Commander struct { + rec testutils.Recorder + specs []CmdSpec + replacePath *regexp.Regexp + replacement string +} + +var _ utils.Commander = &Commander{} + +// NewCommander creates a new Commander. +// If rec is nil, all the commands will not be recorded +func NewCommander(rec testutils.Recorder, specs []CmdSpec) *Commander { + return &Commander{rec: rec, specs: specs} +} + +// ReplaceTempPath makes the commander replace the path with specified +// suffix with the specified string. The replacement is done on the +// word boundary. +func (c *Commander) ReplaceTempPath(pathSuffix, replacement string) { + c.replacePath = regexp.MustCompile(`\S*` + regexp.QuoteMeta(pathSuffix)) + c.replacement = replacement +} + +// Command implements the Command method of Commander interface. +func (c *Commander) Command(name string, arg ...string) utils.Command { + return &fakeCommand{ + rec: c.rec, + cmd: append([]string{name}, arg...), + commander: c, + } +}