Skip to content

Commit

Permalink
Add persistent root filesystem
Browse files Browse the repository at this point in the history
If volumeDevices had an entry with 'devicePath: /', it's used as a
persistent root filesystem for the VM. It gets overwritten
again if the image digest changes.
See examples/cirros-vm-persistent-rootfs.yaml
  • Loading branch information
Ivan Shvedunov committed Sep 19, 2018
1 parent cfb5e53 commit d835a5a
Show file tree
Hide file tree
Showing 45 changed files with 1,258 additions and 88 deletions.
80 changes: 80 additions & 0 deletions examples/cirros-vm-persistent-rootfs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: local-block-pvc
spec:
accessModes:
- ReadWriteOnce
volumeMode: Block
storageClassName: local-storage
resources:
requests:
storage: 100Mi
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: local-block-pv
spec:
capacity:
storage: 100Mi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: local-storage
volumeMode: Block
local:
# set up with:
# docker exec kube-node-1 /bin/bash -c 'dd if=/dev/zero of=/rawtest bs=1M count=1000 && losetup -f /rawtest --show'
path: /dev/loop0
claimRef:
name: local-block-pvc
namespace: default
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- kube-node-1
---
apiVersion: v1
kind: Pod
metadata:
name: cirros-vm-p
annotations:
kubernetes.io/target-runtime: virtlet.cloud
# CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason
VirtletDiskDriver: virtio
VirtletSSHKeys: |
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: extraRuntime
operator: In
values:
- virtlet
# This is the number of seconds Virtlet gives the VM to shut down cleanly.
# The default value of 30 seconds is ok for containers but probably too
# low for VM, so overriding it here is strongly advised.
terminationGracePeriodSeconds: 120
containers:
- name: cirros-vm
image: virtlet.cloud/cirros
imagePullPolicy: IfNotPresent
# tty and stdin required for `kubectl attach -t` to work
tty: true
stdin: true
volumeDevices:
- devicePath: /
name: testpvc
volumes:
- name: testpvc
persistentVolumeClaim:
claimName: local-block-pvc
2 changes: 1 addition & 1 deletion pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func configFieldSet(c *virtlet_v1.VirtletConfig) *fieldSet {
fs.addStringField("kubeletRootDir", "kubelet-root-dir", "", "Pod's root dir in kubelet", kubeletRootDirEnv, kubeletRootDir, &c.KubeletRootDir)
// this field duplicates glog's --v, so no option for it, which is signified
// by "+" here (it's only for doc)
fs.addIntField("logLevel", "+v", "", "Log level to use", logLevelEnv, 1, 0, math.MaxInt32, &c.LogLevel)
fs.addIntField("logLevel", "+v", "", "Log level to use", logLevelEnv, 6, 0, math.MaxInt32, &c.LogLevel)
return &fs
}

Expand Down
6 changes: 3 additions & 3 deletions pkg/image/fake/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,12 @@ func (s *FakeStore) GC() error {
}

// GetImagePathAndVirtualSize implements GC method of Store interface.
func (s *FakeStore) GetImagePathAndVirtualSize(imageName string) (string, uint64, error) {
func (s *FakeStore) GetImagePathDigestAndVirtualSize(imageName string) (string, digest.Digest, uint64, error) {
img, found := s.images[imageName]
if !found {
return "", 0, fmt.Errorf("image not found: %q", imageName)
return "", "", 0, fmt.Errorf("image not found: %q", imageName)
}
return img.Path, img.Size, nil
return img.Path, digest.Digest(img.Digest), img.Size, nil
}

// SetRefGetter implements SetRefGetter method of Store interface.
Expand Down
38 changes: 21 additions & 17 deletions pkg/image/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ type Store interface {
// GC removes all unused or partially downloaded images.
GC() error

// GetImagePathAndVirtualSize returns the path to image data
// and virtual size for the specified image. It accepts
// an image reference or a digest.
GetImagePathAndVirtualSize(ref string) (string, uint64, error)
// GetImagePathDigestAndVirtualSize returns the path to image
// data, the digest and the virtual size for the specified
// image. It accepts an image reference or a digest.
GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error)

// SetRefGetter sets a function that will be used to determine
// the set of images that are currently in use.
Expand Down Expand Up @@ -480,29 +480,32 @@ func (s *FileStore) GC() error {
}

// GetImagePathAndVirtualSize implements GC method of Store interface.
func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, error) {
func (s *FileStore) GetImagePathDigestAndVirtualSize(ref string) (string, digest.Digest, uint64, error) {
s.Lock()
defer s.Unlock()
glog.V(3).Infof("GetImagePathAndVirtualSize(): %q", ref)
glog.V(3).Infof("GetImagePathDigestAndVirtualSize(): %q", ref)

var pathViaDigest, pathViaName string
// parsing digest as ref gives bad results
if d, err := digest.Parse(ref); err == nil {
d, err := digest.Parse(ref)
if err == nil {
if d.Algorithm() != digest.SHA256 {
return "", 0, fmt.Errorf("bad image digest (need sha256): %q", d)
return "", "", 0, fmt.Errorf("bad image digest (need sha256): %q", d)
}
pathViaDigest = s.dataFileName(d.Hex())
} else {
parsed, err := reference.Parse(ref)
if err != nil {
return "", 0, fmt.Errorf("bad image reference %q: %v", ref, err)
return "", "", 0, fmt.Errorf("bad image reference %q: %v", ref, err)
}

d = ""
if digested, ok := parsed.(reference.Digested); ok {
if digested.Digest().Algorithm() != digest.SHA256 {
return "", 0, fmt.Errorf("bad image digest (need sha256): %q", digested.Digest())
return "", "", 0, fmt.Errorf("bad image digest (need sha256): %q", digested.Digest())
}
pathViaDigest = s.dataFileName(digested.Digest().Hex())
d = digested.Digest()
pathViaDigest = s.dataFileName(d.Hex())
}

if named, ok := parsed.(reference.Named); ok && named.Name() != "" {
Expand All @@ -511,35 +514,36 @@ func (s *FileStore) GetImagePathAndVirtualSize(ref string) (string, uint64, erro
glog.Warningf("error reading link %q: %v", pathViaName, err)
} else {
pathViaName = filepath.Join(s.linkDir(), pathViaName)
d = digest.NewDigestFromHex(string(digest.SHA256), filepath.Base(pathViaName))
}
}
}

path := pathViaDigest
switch {
case pathViaDigest == "" && pathViaName == "":
return "", 0, fmt.Errorf("bad image reference %q", ref)
return "", "", 0, fmt.Errorf("bad image reference %q", ref)
case pathViaDigest == "":
path = pathViaName
case pathViaName != "":
fi1, err := os.Stat(pathViaName)
if err != nil {
return "", 0, err
return "", "", 0, err
}
fi2, err := os.Stat(pathViaDigest)
if err != nil {
return "", 0, err
return "", "", 0, err
}
if !os.SameFile(fi1, fi2) {
return "", 0, fmt.Errorf("digest / name path mismatch: %q vs %q", pathViaDigest, pathViaName)
return "", "", 0, fmt.Errorf("digest / name path mismatch: %q vs %q", pathViaDigest, pathViaName)
}
}

vsize, err := s.vsizeFunc(path)
if err != nil {
return "", 0, fmt.Errorf("error getting image size for %q: %v", path, err)
return "", "", 0, fmt.Errorf("error getting image size for %q: %v", path, err)
}
return path, vsize, nil
return path, d, vsize, nil
}

// SetRefGetter implements SetRefGetter method of Store interface.
Expand Down
6 changes: 5 additions & 1 deletion pkg/image/image_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,13 @@ func (tst *ifsTester) verifyListImages(filter string, expectedImages ...*Image)
}

func (tst *ifsTester) verifyImage(ref string, expectedContents string) {
if path, vsize, err := tst.store.GetImagePathAndVirtualSize(ref); err != nil {
if path, digest, vsize, err := tst.store.GetImagePathDigestAndVirtualSize(ref); err != nil {
tst.t.Errorf("GetImagePathAndVirtualSize(): %v", err)
} else {
expectedDigest := "sha256:" + sha256str(expectedContents)
if string(digest) != expectedDigest {
tst.t.Errorf("bad digest: %s instead of %s", digest, expectedDigest)
}
tst.verifyFileContents(path, expectedContents)
expectedVirtualSize := uint64(len(expectedContents)) + 1000
if vsize != expectedVirtualSize {
Expand Down
2 changes: 1 addition & 1 deletion pkg/libvirttools/TestContainerLifecycle.out.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
- name: 'domain conn: ListDomains'
value: []
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
87 changes: 87 additions & 0 deletions pkg/libvirttools/TestDomainDefinitions__persistent_rootfs.out.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: CMD
value:
cmd: blockdev --getsz /fakedev/69eec606-0493-5825-73a4-c5e0c0236155/volumeDevices/kubernetes.io~local-volume/root
stdout: "1000"
- name: CMD
value:
cmd: dmsetup create virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155
stdin: |
0 999 linear /fakedev/69eec606-0493-5825-73a4-c5e0c0236155/volumeDevices/kubernetes.io~local-volume/root 1
- name: CMD
value:
cmd: qemu-img convert -O raw /fake/volume/path /dev/mapper/virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155
- name: 'domain conn: DefineDomain'
value: |-
<domain type="kvm">
<name>virtlet-231700d5-c9a6-container1</name>
<uuid>231700d5-c9a6-5a49-738d-99a954c51550</uuid>
<memory unit="MiB">1024</memory>
<vcpu>1</vcpu>
<cputune>
<shares>0</shares>
<period>0</period>
<quota>0</quota>
</cputune>
<os>
<type>hvm</type>
<boot dev="hd"></boot>
</os>
<features>
<acpi></acpi>
</features>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/vmwrapper</emulator>
<disk type="block" device="disk">
<driver name="qemu" type="raw"></driver>
<source dev="/dev/mapper/virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155"></source>
<target dev="sda" bus="scsi"></target>
<address type="drive" controller="0" bus="0" target="0" unit="0"></address>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu" type="raw"></driver>
<source file="/var/lib/virtlet/config/config-231700d5-c9a6-5a49-738d-99a954c51550.iso"></source>
<target dev="sdb" bus="scsi"></target>
<readonly></readonly>
<address type="drive" controller="0" bus="0" target="0" unit="1"></address>
</disk>
<controller type="scsi" index="0" model="virtio-scsi">
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"></address>
</controller>
<controller type="pci" model="pci-root"></controller>
<serial type="unix">
<source mode="connect" path="/var/lib/libvirt/streamer.sock">
<reconnect enabled="yes" timeout="1"></reconnect>
</source>
<target port="0"></target>
</serial>
<input type="tablet" bus="usb"></input>
<graphics type="vnc" port="-1"></graphics>
<video>
<model type="cirrus"></model>
</video>
</devices>
<commandline xmlns="http://libvirt.org/schemas/domain/qemu/1.0">
<env name="VIRTLET_EMULATOR" value="/usr/bin/kvm"></env>
<env name="VIRTLET_NET_KEY" value="/tmp/fakenetns"></env>
<env name="VIRTLET_CONTAINER_ID" value="231700d5-c9a6-5a49-738d-99a954c51550"></env>
<env name="VIRTLET_CONTAINER_LOG_PATH" value="/var/log/pods/69eec606-0493-5825-73a4-c5e0c0236155/container1_42.log"></env>
</commandline>
</domain>
- name: 'domain conn: virtlet-231700d5-c9a6-container1: Create'
- name: 'domain conn: virtlet-231700d5-c9a6-container1: iso image'
value:
meta-data: '{"instance-id":"testName_0.default","local-hostname":"testName_0"}'
network-config: |
version: 1
user-data: |
#cloud-config
- name: 'domain conn: virtlet-231700d5-c9a6-container1: Destroy'
- name: 'domain conn: virtlet-231700d5-c9a6-container1: Undefine'
- name: CMD
value:
cmd: dmsetup remove virtlet-dm-69eec606-0493-5825-73a4-c5e0c0236155
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
2 changes: 1 addition & 1 deletion pkg/libvirttools/TestDomainDefinitions__volumes.out.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
2 changes: 1 addition & 1 deletion pkg/libvirttools/TestDomainForcedShutdown.out.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
- name: GetImagePathAndVirtualSize
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
- name: setup
- name: 'image: GetImagePathDigestAndVirtualSize'
value: persistent/image1
- name: CMD
value:
cmd: blockdev --getsz /dev/rootdev
stdout: "8"
Loading

0 comments on commit d835a5a

Please sign in to comment.