overlay: add overlay implementation

With help of newly introduced Overlay FileSystem in `fuse-backend-rs`
library, now we can create writable rootfs in Nydus. Implementation of
writable rootfs is based on one passthrough FS(as upper layer) over one
readonly rafs(as lower layer).

To do so, configuration is extended with some Overlay options.

Signed-off-by: Wei Zhang <weizhang555.zw@gmail.com>

Cargo.toml

@@ -35,7 +35,7 @@ path = "src/lib.rs"
 anyhow = "1"
 clap = { version = "4.0.18", features = ["derive", "cargo"] }
 flexi_logger = { version = "0.25", features = ["compress"] }
-fuse-backend-rs = "^0.11.0"
+fuse-backend-rs = "^0.12.0"
 hex = "0.4.3"
 hyper = "0.14.11"
 hyperlocal = "0.8.0"

api/src/config.rs

@@ -29,6 +29,8 @@ pub struct ConfigV2 {
     pub cache: Option<CacheConfigV2>,
     /// Configuration information for RAFS filesystem.
     pub rafs: Option<RafsConfigV2>,
+    /// Overlay configuration information for the instance.
+    pub overlay: Option<OverlayConfig>,
     /// Internal runtime configuration.
     #[serde(skip)]
     pub internal: ConfigV2Internal,
@@ -42,6 +44,7 @@ impl Default for ConfigV2 {
             backend: None,
             cache: None,
             rafs: None,
+            overlay: None,
             internal: ConfigV2Internal::default(),
         }
     }
@@ -56,6 +59,7 @@ impl ConfigV2 {
             backend: None,
             cache: None,
             rafs: None,
+            overlay: None,
             internal: ConfigV2Internal::default(),
         }
     }
@@ -1024,6 +1028,7 @@ impl From<&BlobCacheEntryConfigV2> for ConfigV2 {
             backend: Some(c.backend.clone()),
             cache: Some(c.cache.clone()),
             rafs: None,
+            overlay: None,
             internal: ConfigV2Internal::default(),
         }
     }
@@ -1395,6 +1400,7 @@ impl TryFrom<RafsConfig> for ConfigV2 {
             backend: Some(backend),
             cache: Some(cache),
             rafs: Some(rafs),
+            overlay: None,
             internal: ConfigV2Internal::default(),
         })
     }
@@ -1523,6 +1529,15 @@ impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 {
     }
 }
 
+/// Configuration information for Overlay filesystem.
+/// OverlayConfig is used to configure the writable layer(upper layer),
+/// The filesystem will be writable when OverlayConfig is set.
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+pub struct OverlayConfig {
+    pub upper_dir: String,
+    pub work_dir: String,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

builder/src/core/context.rs

@@ -1516,6 +1516,7 @@ mod tests {
             id: "id".to_owned(),
             cache: None,
             rafs: None,
+            overlay: None,
             internal: ConfigV2Internal {
                 blob_accessible: Arc::new(AtomicBool::new(true)),
             },

clib/Cargo.toml

@@ -15,7 +15,7 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 libc = "0.2.137"
 log = "0.4.17"
-fuse-backend-rs = "^0.11.0"
+fuse-backend-rs = "^0.12.0"
 nydus-api = { version = "0.3", path = "../api" }
 nydus-rafs = { version = "0.3.1", path = "../rafs" }
 nydus-storage = { version = "0.6.3", path = "../storage" }

docs/nydus-overlayfs.md

@@ -1,5 +1,7 @@
 # NydusOverlayfs
 
+## Enable with `nydus-overlayfs` helper
+
 `nydus-overlayfs` is a FUSE(Filesystem in UserSpacE) mount helper command for containerd to use with Nydus. The document explains in a nutshell on how it works.
 
 When the `--enable-nydus-overlayfs` option is specified, `nydus-snapshotter`'s `Mount()` method returns a mount slice like
@@ -39,3 +41,50 @@ mount -t overlay overlay ./foo/merged -o lowerdir=./foo/lower2:./foo/lower1,uppe
 Meanwhile, when containerd passes the `nydus-snapshotter` mount slice to `containerd-shim-kata-v2`, it can parse the mount slice and pass the `extraoption` to `nydusd`, to support nydus image format natively.
 
 So in summary, `containerd` and `containerd-shim-runc-v2` rely on the `nydus-overlay` mount helper to handle the mount slice returned by `nydus-snapshotter`, while `containerd-shim-kata-v2` can parse and handle it on its own.
+
+## Enable with Nydusd native implementation
+
+`Nydusd` itself has a native userspace Overlay FS implementation, which can be enabled with several extra configurations. 
+
+An example configuration `/etc/nydus/nydusd-config.overlay.json` is as follows:
+
+```json
+{
+	"version": 2,
+	"backend": {
+		"type": "localfs",
+		"localfs": "/var/lib/nydus/blobs"
+	},
+	"cache": {
+		"type": "blobcache",
+		"filecache": {
+			"work_dir": "/var/lib/nydus/cache"
+		}
+	},
+	"rafs": {
+		"mode": "direct",
+		"enable_xattr": true
+	},
+	"overlay": {
+		"upper_dir": "/path/to/upperdir",
+		"work_dir": "/path/to/workdir",
+	}
+}
+```
+
+An extra field `overlay` is added to the Nydusd configuration, which specifies the `upper_dir` and `work_dir` for the overlay filesystem.
+
+You can start `nydusd` with the above configuration and such command:
+
+```bash
+sudo nydusd \
+    --config /etc/nydus/nydusd-config.overlay.json \
+    --mountpoint /path/to/mnt/ \
+    --bootstrap /path/to/bootstrap \
+    --log-level info \
+    --writable
+```
+
+This will create a FUSE overlay mountpoint at `/path/to/mnt/`, with one `Nydus` image as readonly lower layer and the `/path/to/upperdir` as writable upper layer, so that it can take over whole rootfs of a container, any contents writen from container will be stored in `/path/to/upperdir`.
+
+Removing `--writable` flag will make the overlay filesystem readonly if you wish.

rafs/Cargo.toml

@@ -19,7 +19,7 @@ nix = "0.24"
 serde = { version = "1.0.110", features = ["serde_derive", "rc"] }
 serde_json = "1.0.53"
 vm-memory = "0.10"
-fuse-backend-rs = "^0.11.0"
+fuse-backend-rs = "^0.12.0"
 thiserror = "1"
 
 nydus-api = { version = "0.3", path = "../api" }

rafs/src/fs.rs

@@ -291,7 +291,8 @@ impl Rafs {
         // since nydusify gives root directory permission of 0o750 and fuse mount
         // options `rootmode=` does not affect root directory's permission bits, ending
         // up with preventing other users from accessing the container rootfs.
-        if attr.ino == self.root_ino() {
+        let root_ino = self.root_ino();
+        if attr.ino == root_ino {
             attr.mode = attr.mode & !0o777 | 0o755;
         }
 
@@ -684,9 +685,9 @@ impl FileSystem for Rafs {
         _inode: Self::Inode,
         _flags: u32,
         _fuse_flags: u32,
-    ) -> Result<(Option<Self::Handle>, OpenOptions)> {
+    ) -> Result<(Option<Self::Handle>, OpenOptions, Option<u32>)> {
         // Keep cache since we are readonly
-        Ok((None, OpenOptions::KEEP_CACHE))
+        Ok((None, OpenOptions::KEEP_CACHE, None))
     }
 
     fn release(
@@ -886,6 +887,14 @@ impl FileSystem for Rafs {
     }
 }
 
+#[cfg(target_os = "linux")]
+// Let Rafs works as an OverlayFs layer.
+impl Layer for Rafs {
+    fn root_inode(&self) -> Self::Inode {
+        self.root_ino()
+    }
+}
+
 #[cfg(all(test, feature = "backend-oss"))]
 pub(crate) mod tests {
     use super::*;

service/Cargo.toml

@@ -12,7 +12,7 @@ resolver = "2"
 [dependencies]
 bytes = { version = "1", optional = true }
 dbs-allocator = { version = "0.1.1", optional = true }
-fuse-backend-rs = { version = "^0.11.0", features = ["persist"] }
+fuse-backend-rs = { version = "^0.12.0", features = ["persist"] }
 libc = "0.2"
 log = "0.4.8"
 mio = { version = "0.8", features = ["os-poll", "os-ext"] }

service/src/fs_service.rs

@@ -13,10 +13,14 @@ use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::sync::{Arc, MutexGuard};
 
+#[cfg(target_os = "linux")]
+use fuse_backend_rs::api::filesystem::{FileSystem, FsOptions, Layer};
 use fuse_backend_rs::api::vfs::VfsError;
 use fuse_backend_rs::api::{BackFileSystem, Vfs};
 #[cfg(target_os = "linux")]
-use fuse_backend_rs::passthrough::{Config, PassthroughFs};
+use fuse_backend_rs::overlayfs::{config::Config as overlay_config, OverlayFs};
+#[cfg(target_os = "linux")]
+use fuse_backend_rs::passthrough::{CachePolicy, Config as passthrough_config, PassthroughFs};
 use nydus_api::ConfigV2;
 use nydus_rafs::fs::Rafs;
 use nydus_rafs::{RafsError, RafsIoRead};
@@ -244,8 +248,77 @@ fn fs_backend_factory(cmd: &FsBackendMountCmd) -> Result<BackFileSystem> {
             let config = Arc::new(config);
             let (mut rafs, reader) = Rafs::new(&config, &cmd.mountpoint, Path::new(&cmd.source))?;
             rafs.import(reader, prefetch_files)?;
-            info!("RAFS filesystem imported");
-            Ok(Box::new(rafs))
+
+            // Put a writable upper layer above the rafs to create an OverlayFS with two layers.
+            match &config.overlay {
+                Some(ovl_conf) => {
+                    // check workdir and upperdir params.
+                    if ovl_conf.work_dir.is_empty() || ovl_conf.upper_dir.is_empty() {
+                        return Err(Error::InvalidArguments(String::from(
+                            "workdir and upperdir must be specified for overlayfs",
+                        )));
+                    }
+
+                    // Create an overlay upper layer with passthroughfs.
+                    #[cfg(target_os = "macos")]
+                    return Err(Error::InvalidArguments(String::from(
+                        "not support OverlayFs since passthroughfs isn't supported on MacOS",
+                    )));
+                    #[cfg(target_os = "linux")]
+                    {
+                        let fs_cfg = passthrough_config {
+                            // Use upper_dir as root_dir as rw layer.
+                            root_dir: ovl_conf.upper_dir.clone(),
+                            do_import: true,
+                            writeback: true,
+                            no_open: true,
+                            no_opendir: true,
+                            xattr: true,
+                            cache_policy: CachePolicy::Always,
+                            ..Default::default()
+                        };
+                        let fsopts = FsOptions::WRITEBACK_CACHE
+                            | FsOptions::ZERO_MESSAGE_OPEN
+                            | FsOptions::ZERO_MESSAGE_OPENDIR;
+
+                        let passthrough_fs = PassthroughFs::<()>::new(fs_cfg)
+                            .map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
+                        passthrough_fs.init(fsopts).map_err(Error::PassthroughFs)?;
+
+                        type BoxedLayer = Box<dyn Layer<Inode = u64, Handle = u64> + Send + Sync>;
+                        let upper_layer = Arc::new(Box::new(passthrough_fs) as BoxedLayer);
+
+                        // Create overlay lower layer with rafs, use lower_dir as root_dir of rafs.
+                        let lower_layers = vec![Arc::new(Box::new(rafs) as BoxedLayer)];
+
+                        let overlay_config = overlay_config {
+                            work: ovl_conf.work_dir.clone(),
+                            mountpoint: cmd.mountpoint.clone(),
+                            do_import: false,
+                            no_open: true,
+                            no_opendir: true,
+                            ..Default::default()
+                        };
+                        let overlayfs =
+                            OverlayFs::new(Some(upper_layer), lower_layers, overlay_config)
+                                .map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
+                        info!(
+                            "init overlay fs inode, upper {}, work {}\n",
+                            ovl_conf.upper_dir.clone(),
+                            ovl_conf.work_dir.clone()
+                        );
+                        overlayfs
+                            .import()
+                            .map_err(|e| Error::InvalidConfig(format!("{}", e)))?;
+                        info!("Overlay filesystem imported");
+                        Ok(Box::new(overlayfs))
+                    }
+                }
+                None => {
+                    info!("RAFS filesystem imported");
+                    Ok(Box::new(rafs))
+                }
+            }
         }
         FsBackendType::PassthroughFs => {
             #[cfg(target_os = "macos")]
@@ -257,7 +330,7 @@ fn fs_backend_factory(cmd: &FsBackendMountCmd) -> Result<BackFileSystem> {
                 // Vfs by default enables no_open and writeback, passthroughfs
                 // needs to specify them explicitly.
                 // TODO(liubo): enable no_open_dir.
-                let fs_cfg = Config {
+                let fs_cfg = passthrough_config {
                     root_dir: cmd.source.to_string(),
                     do_import: false,
                     writeback: true,