Add virtio-fs support (alternative to virtio-9p)

Gopkg.toml

@@ -48,7 +48,7 @@
 
 [[constraint]]
   name = "github.com/intel/govmm"
-  revision = "35a8fd3ca9a36461b7dcf24e3b292f6e1ea4e71a"
+  revision = "b3e7a9e78463a10f2a19e1a966c76a3afb215781"
 
 [[constraint]]
   name = "github.com/kata-containers/agent"

Makefile

@@ -156,6 +156,11 @@ DEFAULTEXPFEATURES := []
 DEFENTROPYSOURCE := /dev/urandom
 
 DEFDISABLEBLOCK := false
+DEFSHAREDFS := virtio-9p
+DEFVIRTIOFSDAEMON :=
+# Default DAX mapping cache size in MiB
+DEFVIRTIOFSCACHESIZE := 8192
+DEFVIRTIOFSCACHE := always
 DEFENABLEIOTHREADS := false
 DEFENABLEMEMPREALLOC := false
 DEFENABLEHUGEPAGES := false
@@ -320,6 +325,10 @@ USER_VARS += DEFAULTEXPFEATURES
 USER_VARS += DEFDISABLEBLOCK
 USER_VARS += DEFBLOCKSTORAGEDRIVER_FC
 USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
+USER_VARS += DEFSHAREDFS
+USER_VARS += DEFVIRTIOFSDAEMON
+USER_VARS += DEFVIRTIOFSCACHESIZE
+USER_VARS += DEFVIRTIOFSCACHE
 USER_VARS += DEFENABLEIOTHREADS
 USER_VARS += DEFENABLEMEMPREALLOC
 USER_VARS += DEFENABLEHUGEPAGES
@@ -456,6 +465,10 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 		-e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \
 		-e "s|@DEFBLOCKSTORAGEDRIVER_FC@|$(DEFBLOCKSTORAGEDRIVER_FC)|g" \
 		-e "s|@DEFBLOCKSTORAGEDRIVER_QEMU@|$(DEFBLOCKSTORAGEDRIVER_QEMU)|g" \
+		-e "s|@DEFSHAREDFS@|$(DEFSHAREDFS)|g" \
+		-e "s|@DEFVIRTIOFSDAEMON@|$(DEFVIRTIOFSDAEMON)|g" \
+		-e "s|@DEFVIRTIOFSCACHESIZE@|$(DEFVIRTIOFSCACHESIZE)|g" \
+		-e "s|@DEFVIRTIOFSCACHE@|$(DEFVIRTIOFSCACHE)|g" \
 		-e "s|@DEFENABLEIOTHREADS@|$(DEFENABLEIOTHREADS)|g" \
 		-e "s|@DEFENABLEMEMPREALLOC@|$(DEFENABLEMEMPREALLOC)|g" \
 		-e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \

cli/config/configuration-qemu.toml.in

@@ -97,6 +97,32 @@ default_memory = @DEFMEMSZ@
 # 9pfs is used instead to pass the rootfs.
 disable_block_device_use = @DEFDISABLEBLOCK@
 
+# Shared file system type:
+#   - virtio-9p (default)
+#   - virtio-fs
+shared_fs = "@DEFSHAREDFS@"
+
+# Path to vhost-user-fs daemon.
+virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
+
+# Default size of DAX cache in MiB
+virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
+
+# Cache mode:
+#
+#  - none
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
 # Block storage driver to be used for the hypervisor in case the container
 # rootfs is backed by a block device. This is virtio-scsi, virtio-blk
 # or nvdimm.

cli/kata-env.go

@@ -92,6 +92,7 @@ type HypervisorInfo struct {
 	MemorySlots       uint32
 	Debug             bool
 	UseVSock          bool
+	SharedFS          string
 }
 
 // ProxyInfo stores proxy details
@@ -352,6 +353,7 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 		UseVSock:          config.HypervisorConfig.UseVSock,
 		MemorySlots:       config.HypervisorConfig.MemSlots,
 		EntropySource:     config.HypervisorConfig.EntropySource,
+		SharedFS:          config.HypervisorConfig.SharedFS,
 	}
 }
 

cli/kata-env_test.go

@@ -37,6 +37,7 @@ const testHypervisorVersion = "QEMU emulator version 2.7.0+git.741f430a96-6.1, C
 
 var (
 	hypervisorDebug = false
+	enableVirtioFS  = false
 	proxyDebug      = false
 	runtimeDebug    = false
 	runtimeTrace    = false
@@ -91,6 +92,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	disableNewNetNs := false
+	sharedFS := "virtio-9p"
 
 	filesToCreate := []string{
 		hypervisorPath,
@@ -126,6 +128,10 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		return "", oci.RuntimeConfig{}, err
 	}
 
+	if enableVirtioFS {
+		sharedFS = "virtio-fs"
+	}
+
 	hypConfig := katautils.GetDefaultHypervisorConfig()
 
 	configFileOptions := katatestutils.RuntimeConfigOptions{
@@ -157,6 +163,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		NetmonDebug:          netmonDebug,
 		AgentDebug:           agentDebug,
 		AgentTrace:           agentTrace,
+		SharedFS:             sharedFS,
 	}
 
 	runtimeConfig := katatestutils.MakeRuntimeConfigFileData(configFileOptions)
@@ -321,6 +328,7 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 		MemorySlots:       config.HypervisorConfig.MemSlots,
 		Debug:             config.HypervisorConfig.Debug,
 		EntropySource:     config.HypervisorConfig.EntropySource,
+		SharedFS:          config.HypervisorConfig.SharedFS,
 	}
 }
 
@@ -498,6 +506,7 @@ func TestEnvGetEnvInfo(t *testing.T) {
 	// options are tested.
 	for _, toggle := range []bool{false, true} {
 		hypervisorDebug = toggle
+		enableVirtioFS = toggle
 		proxyDebug = toggle
 		runtimeDebug = toggle
 		runtimeTrace = toggle

pkg/katatestutils/utils.go

@@ -27,6 +27,7 @@ type RuntimeConfigOptions struct {
 	BlockDeviceDriver    string
 	AgentTraceMode       string
 	AgentTraceType       string
+	SharedFS             string
 	DisableBlock         bool
 	EnableIOThreads      bool
 	HotplugVFIOOnRootBus bool
@@ -61,6 +62,8 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
 	enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `
 	guest_hook_path = "` + config.DefaultGuestHookPath + `"
+	shared_fs = "` + config.SharedFS + `"
+	virtio_fs_daemon = "/path/to/virtiofsd"
 
 	[proxy.kata]
 	enable_debug = ` + strconv.FormatBool(config.ProxyDebug) + `

pkg/katautils/config.go

@@ -92,6 +92,10 @@ type hypervisor struct {
 	MachineType             string `toml:"machine_type"`
 	BlockDeviceDriver       string `toml:"block_device_driver"`
 	EntropySource           string `toml:"entropy_source"`
+	SharedFS                string `toml:"shared_fs"`
+	VirtioFSDaemon          string `toml:"virtio_fs_daemon"`
+	VirtioFSCache           string `toml:"virtio_fs_cache"`
+	VirtioFSCacheSize       uint32 `toml:"virtio_fs_cache_size"`
 	BlockDeviceCacheSet     bool   `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect  bool   `toml:"block_device_cache_direct"`
 	BlockDeviceCacheNoflush bool   `toml:"block_device_cache_noflush"`
@@ -326,6 +330,22 @@ func (h hypervisor) blockDeviceDriver() (string, error) {
 	return "", fmt.Errorf("Invalid hypervisor block storage driver %v specified (supported drivers: %v)", h.BlockDeviceDriver, supportedBlockDrivers)
 }
 
+func (h hypervisor) sharedFS() (string, error) {
+	supportedSharedFS := []string{config.Virtio9P, config.VirtioFS}
+
+	if h.SharedFS == "" {
+		return config.Virtio9P, nil
+	}
+
+	for _, fs := range supportedSharedFS {
+		if fs == h.SharedFS {
+			return h.SharedFS, nil
+		}
+	}
+
+	return "", fmt.Errorf("Invalid hypervisor shared file system %v specified (supported file systems: %v)", h.SharedFS, supportedSharedFS)
+}
+
 func (h hypervisor) msize9p() uint32 {
 	if h.Msize9p == 0 {
 		return defaultMsize9p
@@ -521,6 +541,16 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		return vc.HypervisorConfig{}, err
 	}
 
+	sharedFS, err := h.sharedFS()
+	if err != nil {
+		return vc.HypervisorConfig{}, err
+	}
+
+	if sharedFS == config.VirtioFS && h.VirtioFSDaemon == "" {
+		return vc.HypervisorConfig{},
+			errors.New("cannot enable virtio-fs without daemon path in configuration file")
+	}
+
 	useVSock := false
 	if h.useVSock() {
 		if utils.SupportsVsocks() {
@@ -548,6 +578,10 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EntropySource:           h.GetEntropySource(),
 		DefaultBridges:          h.defaultBridges(),
 		DisableBlockDeviceUse:   h.DisableBlockDeviceUse,
+		SharedFS:                sharedFS,
+		VirtioFSDaemon:          h.VirtioFSDaemon,
+		VirtioFSCacheSize:       h.VirtioFSCacheSize,
+		VirtioFSCache:           h.VirtioFSCache,
 		MemPrealloc:             h.MemPrealloc,
 		HugePages:               h.HugePages,
 		Mlock:                   !h.Swap,

pkg/katautils/config_test.go

@@ -83,6 +83,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	disableNewNetNs := false
+	sharedFS := "virtio-9p"
 
 	configFileOptions := katatestutils.RuntimeConfigOptions{
 		Hypervisor:           "qemu",
@@ -113,6 +114,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		NetmonDebug:          netmonDebug,
 		AgentDebug:           agentDebug,
 		AgentTrace:           agentTrace,
+		SharedFS:             sharedFS,
 	}
 
 	runtimeConfigFileData := katatestutils.MakeRuntimeConfigFileData(configFileOptions)
@@ -160,6 +162,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		MemSlots:              defaultMemSlots,
 		EntropySource:         defaultEntropySource,
 		GuestHookPath:         defaultGuestHookPath,
+		SharedFS:              sharedFS,
+		VirtioFSDaemon:        "/path/to/virtiofsd",
 	}
 
 	agentConfig := vc.KataAgentConfig{}

virtcontainers/device/config/config.go

@@ -36,6 +36,9 @@ const (
 
 	//VhostUserBlk represents a block vhostuser device type
 	VhostUserBlk = "vhost-user-blk-pci"
+
+	//VhostUserFS represents a virtio-fs vhostuser device type
+	VhostUserFS = "vhost-user-fs-pci"
 )
 
 const (
@@ -52,6 +55,14 @@ const (
 	Nvdimm = "nvdimm"
 )
 
+const (
+	// Virtio9P means use virtio-9p for the shared file system
+	Virtio9P = "virtio-9p"
+
+	// VirtioFS means use virtio-fs for the shared file system
+	VirtioFS = "virtio-fs"
+)
+
 // Defining these as a variable instead of a const, to allow
 // overriding this in the tests.
 
@@ -174,6 +185,11 @@ type VhostUserDeviceAttrs struct {
 
 	// MacAddress is only meaningful for vhost user net device
 	MacAddress string
+
+	// These are only meaningful for vhost user fs devices
+	Tag       string
+	CacheSize uint32
+	Cache     string
 }
 
 // GetHostPathFunc is function pointer used to mock GetHostPath in tests.

virtcontainers/device/drivers/vhost_user_fs.go

@@ -0,0 +1,65 @@
+// Copyright (C) 2019 Red Hat, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package drivers
+
+import (
+	"encoding/hex"
+
+	"github.com/kata-containers/runtime/virtcontainers/device/api"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
+)
+
+// VhostUserFSDevice is a virtio-fs vhost-user device
+type VhostUserFSDevice struct {
+	*GenericDevice
+	config.VhostUserDeviceAttrs
+}
+
+// Device interface
+
+func (device *VhostUserFSDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
+	skip, err := device.bumpAttachCount(true)
+	if err != nil {
+		return err
+	}
+	if skip {
+		return nil
+	}
+
+	defer func() {
+		if err != nil {
+			device.bumpAttachCount(false)
+		}
+	}()
+
+	// generate a unique ID to be used for hypervisor commandline fields
+	randBytes, err := utils.GenerateRandomBytes(8)
+	if err != nil {
+		return err
+	}
+	id := hex.EncodeToString(randBytes)
+
+	device.DevID = id
+	device.Type = device.DeviceType()
+
+	return devReceiver.AppendDevice(device)
+}
+
+func (device *VhostUserFSDevice) Detach(devReceiver api.DeviceReceiver) error {
+	_, err := device.bumpAttachCount(false)
+	return err
+}
+
+func (device *VhostUserFSDevice) DeviceType() config.DeviceType {
+	return config.VhostUserFS
+}
+
+// GetDeviceInfo returns device information that the device is created based on
+func (device *VhostUserFSDevice) GetDeviceInfo() interface{} {
+	device.Type = device.DeviceType()
+	return &device.VhostUserDeviceAttrs
+}