CRI metrics - runtime part

pkg/image/fake/store.go

@@ -25,6 +25,7 @@ import (
 	digest "github.com/opencontainers/go-digest"
 
 	"github.com/Mirantis/virtlet/pkg/image"
+	"github.com/Mirantis/virtlet/pkg/metadata/types"
 	testutils "github.com/Mirantis/virtlet/pkg/utils/testing"
 )
 
@@ -133,10 +134,15 @@ func (s *FakeStore) SetRefGetter(imageRefGetter image.RefGetter) {
 }
 
 // FilesystemStats implements FilesystemStats method from Store interface.
-func (s *FakeStore) FilesystemStats() (*image.FilesystemStats, error) {
-	return &image.FilesystemStats{
+func (s *FakeStore) FilesystemStats() (*types.FilesystemStats, error) {
+	return &types.FilesystemStats{
 		Mountpoint: fakeStoreMountpoint,
 		UsedBytes:  fakeUsedBytes,
 		UsedInodes: fakeUsedInodes,
 	}, nil
 }
+
+// BytesUsedBy implements BytesUsedBy method from Store interface.
+func (s *FakeStore) BytesUsedBy(path string) (uint64, error) {
+	return fakeUsedBytes, nil
+}

pkg/image/image.go

@@ -30,6 +30,7 @@ import (
 	"github.com/golang/glog"
 	digest "github.com/opencontainers/go-digest"
 
+	"github.com/Mirantis/virtlet/pkg/metadata/types"
 	"github.com/Mirantis/virtlet/pkg/utils"
 )
 
@@ -57,17 +58,6 @@ type Translator func(context.Context, string) Endpoint
 // that are currently in use.
 type RefGetter func() (map[string]bool, error)
 
-// FilesystemStats contains info about filesystem mountpoint and
-// space/inodes used by images on it
-type FilesystemStats struct {
-	// Mountpoint denotes the filesystem mount point
-	Mountpoint string
-	// UsedBytes is the number of bytes used by images
-	UsedBytes uint64
-	// UsedInodes is the number of inodes used by images
-	UsedInodes uint64
-}
-
 // Store is an interface for the image store.
 type Store interface {
 	// ListImage returns the list of images in the store.
@@ -101,7 +91,10 @@ type Store interface {
 	SetRefGetter(imageRefGetter RefGetter)
 
 	// FilesystemStats returns disk space and inode usage info for this store.
-	FilesystemStats() (*FilesystemStats, error)
+	FilesystemStats() (*types.FilesystemStats, error)
+
+	// BytesUsedBy returns disk usage of the file in this store.
+	BytesUsedBy(path string) (uint64, error)
 }
 
 // VirtualSizeFunc specifies a function that returns the virtual
@@ -594,7 +587,7 @@ func GetHexDigest(imageSpec string) string {
 // TODO: instead of returning data from filesystem we should retrieve from
 // metadata store sizes of images and sum them, or even retrieve precalculated
 // sum. That's because same filesystem could be used by other things than images.
-func (s *FileStore) FilesystemStats() (*FilesystemStats, error) {
+func (s *FileStore) FilesystemStats() (*types.FilesystemStats, error) {
 	occupiedBytes, occupiedInodes, err := utils.GetFsStatsForPath(s.dir)
 	if err != nil {
 		return nil, err
@@ -607,9 +600,18 @@ func (s *FileStore) FilesystemStats() (*FilesystemStats, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &FilesystemStats{
+	return &types.FilesystemStats{
 		Mountpoint: mount.FSRoot,
 		UsedBytes:  occupiedBytes,
 		UsedInodes: occupiedInodes,
 	}, nil
 }
+
+// BytesUsedBy return disk usage of provided file as seen in store
+func (s *FileStore) BytesUsedBy(path string) (uint64, error) {
+	fstat, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+	return uint64(fstat.Size()), nil
+}

pkg/libvirttools/libvirt_domain.go

@@ -211,6 +211,36 @@ func (domain *libvirtDomain) XML() (*libvirtxml.Domain, error) {
 	return &d, nil
 }
 
+// GetRSS returns RSS used by VM in bytes
+func (domain *libvirtDomain) GetRSS() (uint64, error) {
+	stats, err := domain.d.MemoryStats(uint32(libvirt.DOMAIN_MEMORY_STAT_LAST), 0)
+	if err != nil {
+		return 0, err
+	}
+	for _, stat := range stats {
+		if stat.Tag == int32(libvirt.DOMAIN_MEMORY_STAT_RSS) {
+			return stat.Val * 1024, nil
+		}
+	}
+	return 0, fmt.Errorf("rss not found in memory stats")
+}
+
+// GetCPUTime returns cpu time used by VM in nanoseconds per core
+func (domain *libvirtDomain) GetCPUTime() (uint64, error) {
+	// all vcpus as a single value
+	stats, err := domain.d.GetCPUStats(-1, 1, 0)
+	if err != nil {
+		return 0, err
+	}
+	if len(stats) != 1 {
+		return 0, fmt.Errorf("domain.GetCPUStats returned %d values while single one was expected", len(stats))
+	}
+	if !stats[0].CpuTimeSet {
+		return 0, fmt.Errorf("domain.CpuTime not found in memory stats")
+	}
+	return stats[0].CpuTime, nil
+}
+
 type libvirtSecret struct {
 	s *libvirt.Secret
 }

pkg/libvirttools/root_volumesource_test.go

@@ -30,8 +30,10 @@ import (
 )
 
 const (
-	testUUID             = "77f29a0e-46af-4188-a6af-9ff8b8a65224"
-	fakeImageVirtualSize = 424242
+	testUUID                 = "77f29a0e-46af-4188-a6af-9ff8b8a65224"
+	fakeImageVirtualSize     = 424242
+	fakeImageStoreUsedBytes  = 424242
+	fakeImageStoreUsedInodes = 424242
 )
 
 type FakeImageManager struct {
@@ -51,6 +53,19 @@ func (im *FakeImageManager) GetImagePathAndVirtualSize(imageName string) (string
 	return "/fake/volume/path", fakeImageVirtualSize, nil
 }
 
+func (im *FakeImageManager) FilesystemStats() (*types.FilesystemStats, error) {
+	return &types.FilesystemStats{
+		Mountpoint: "/some/dir",
+		UsedBytes:  fakeImageStoreUsedBytes,
+		UsedInodes: fakeImageStoreUsedInodes,
+	}, nil
+}
+
+func (im *FakeImageManager) BytesUsedBy(path string) (uint64, error) {
+	im.rec.Rec("BytesUsedBy", path)
+	return fakeImageVirtualSize, nil
+}
+
 func TestRootVolumeNaming(t *testing.T) {
 	v := rootVolume{
 		volumeBase{

pkg/libvirttools/virtualization.go

@@ -19,6 +19,7 @@ package libvirttools
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/golang/glog"
@@ -809,9 +810,98 @@ func (v *VirtualizationTool) ContainerInfo(containerID string) (*types.Container
 	return containerInfo, nil
 }
 
+// VMStats returns current cpu/memory/disk usage for VM
+func (v *VirtualizationTool) VMStats(containerID string, name string) (*types.VMStats, error) {
+	domain, err := v.domainConn.LookupDomainByUUIDString(containerID)
+	if err != nil {
+		return nil, err
+	}
+	vs := types.VMStats{
+		Timestamp:   v.clock.Now().UnixNano(),
+		ContainerID: containerID,
+		Name:        name,
+	}
+
+	rss, err := domain.GetRSS()
+	if err != nil {
+		return nil, err
+	}
+	vs.MemoryUsage = rss
+
+	cpuTime, err := domain.GetCPUTime()
+	if err != nil {
+		return nil, err
+	}
+	vs.CpuUsage = cpuTime
+
+	domainxml, err := domain.XML()
+	if err != nil {
+		return nil, err
+	}
+
+	rootDiskLocation := ""
+	for _, disk := range domainxml.Devices.Disks {
+		if disk.Source == nil || disk.Source.File == nil {
+			continue
+		}
+		fname := disk.Source.File.File
+		// TODO: split file name and use HasPrefix on last part
+		// instead of Contains
+		if strings.Contains(fname, "virtlet_root_") {
+			rootDiskLocation = fname
+		}
+	}
+	if rootDiskLocation == "" {
+		return nil, fmt.Errorf("cannot locate root disk in domain definition")
+	}
+
+	rootDiskSize, err := v.ImageManager().BytesUsedBy(rootDiskLocation)
+	if err != nil {
+		return nil, err
+	}
+	vs.FsBytes = rootDiskSize
+
+	glog.V(4).Infof("VMStats - cpu: %d, mem: %d, disk: %d, timestamp: %d", vs.CpuUsage, vs.MemoryUsage, vs.FsBytes, vs.Timestamp)
+
+	return &vs, nil
+}
+
+// ListVMStats returns statistics (same as VMStats) for all containers matching
+// provided filter (id AND podstandboxid AND labels)
+func (v *VirtualizationTool) ListVMStats(filter *types.VMStatsFilter) ([]types.VMStats, error) {
+	var containersFilter *types.ContainerFilter
+	if filter != nil {
+		containersFilter = &types.ContainerFilter{}
+		if filter.Id != "" {
+			containersFilter.Id = filter.Id
+		}
+		if filter.PodSandboxID != "" {
+			containersFilter.PodSandboxID = filter.PodSandboxID
+		}
+		if filter.LabelSelector != nil {
+			containersFilter.LabelSelector = filter.LabelSelector
+		}
+	}
+
+	infos, err := v.ListContainers(containersFilter)
+	if err != nil {
+		return nil, err
+	}
+
+	var statsList []types.VMStats
+	for _, info := range infos {
+		stats, err := v.VMStats(info.Id, info.Name)
+		if err != nil {
+			return nil, err
+		}
+		statsList = append(statsList, *stats)
+	}
+	return statsList, nil
+}
+
 // volumeOwner implementation follows
 
-// StoragePool returns StoragePool for volumes
+// StoragePool implements volumeOwner StoragePool method
 func (v *VirtualizationTool) StoragePool() (virt.StoragePool, error) {
 	return ensureStoragePool(v.storageConn, v.config.VolumePoolName)
 }

pkg/libvirttools/volumes.go

@@ -27,6 +27,8 @@ import (
 // ImageManager describes a images info provider.
 type ImageManager interface {
 	GetImagePathAndVirtualSize(ref string) (string, uint64, error)
+	FilesystemStats() (*types.FilesystemStats, error)
+	BytesUsedBy(path string) (uint64, error)
 }
 
 type volumeOwner interface {

pkg/manager/TestCRIPods.out.yaml

@@ -301,6 +301,56 @@
         name: container-for-testName_0
       started_at: 1524648266720331175
       state: 1
+- name: 'enter: ContainerStats'
+  value:
+    container_id: 231700d5-c9a6-5a49-738d-99a954c51550
+- name: 'leave: ContainerStats'
+  value:
+    stats:
+      attributes:
+        id: 231700d5-c9a6-5a49-738d-99a954c51550
+        metadata:
+          name: 231700d5-c9a6-5a49-738d-99a954c51550
+      cpu:
+        timestamp: 1524648266720331175
+        usage_core_nano_seconds: {}
+      memory:
+        timestamp: 1524648266720331175
+        working_set_bytes: {}
+      writable_layer:
+        fs_id:
+          mountpoint: /var/lib/virtlet
+        inodes_used:
+          value: 1
+        timestamp: 1524648266720331175
+        used_bytes:
+          value: 1073741824
+- name: 'enter: ListContainerStats'
+  value: {}
+- name: 'domain conn: ListDomains'
+  value:
+  - virtlet-231700d5-c9a6-container-for-testName_0
+- name: 'leave: ListContainerStats'
+  value:
+    stats:
+    - attributes:
+        id: 231700d5-c9a6-5a49-738d-99a954c51550
+        metadata:
+          name: 231700d5-c9a6-5a49-738d-99a954c51550
+      cpu:
+        timestamp: 1524648266720331175
+        usage_core_nano_seconds: {}
+      memory:
+        timestamp: 1524648266720331175
+        working_set_bytes: {}
+      writable_layer:
+        fs_id:
+          mountpoint: /var/lib/virtlet
+        inodes_used:
+          value: 1
+        timestamp: 1524648266720331175
+        used_bytes:
+          value: 1073741824
 - name: 'enter: PullImage'
   value:
     image: