feat: copy dir with globs (#189)

Cargo.toml

@@ -54,6 +54,8 @@ chrono = "0.4.31"
 sha1 = "0.10.6"
 git2 = { version = "0.18.1", features = ["vendored-openssl"] }
 fs_extra = "1.3.0"
+ignore = "0.4.20"
+globset = "0.4.13"
 
 [dev-dependencies]
 insta = {version = "1.33.0", features = ["yaml"] }

src/source/mod.rs

@@ -4,8 +4,8 @@ use std::{
     process::Command,
 };
 
-use fs_extra::dir::{copy, create_all, remove, CopyOptions};
-use fs_extra::error::ErrorKind::PermissionDenied;
+use fs_extra::dir::{create_all, CopyOptions};
+use ignore::WalkBuilder;
 
 use crate::metadata::Source;
 
@@ -41,6 +41,12 @@ pub enum SourceError {
 
     #[error("Failed to run git command: {0}")]
     GitError(#[from] git2::Error),
+
+    #[error("Could not walk dir")]
+    IgnoreError(#[from] ignore::Error),
+
+    #[error("Failed to parse glob pattern")]
+    Glob(#[from] globset::Error),
 }
 
 /// Fetches all sources in a list of sources and applies specified patches
@@ -66,7 +72,7 @@ pub async fn fetch_sources(
                 } else {
                     work_dir.to_path_buf()
                 };
-                copy_dir(&result, &dest_dir)?;
+                copy_dir(&result, &dest_dir, &[], &[], false)?;
 
                 if let Some(patches) = &src.patches {
                     patch::apply_patches(patches, work_dir, recipe_dir)?;
@@ -96,7 +102,7 @@ pub async fn fetch_sources(
                 } else {
                     work_dir.to_path_buf()
                 };
-                copy_dir(&src_path, &dest_dir)?;
+                copy_dir(&src_path, &dest_dir, &[], &[], true)?;
 
                 if let Some(patches) = &src.patches {
                     patch::apply_patches(patches, work_dir, recipe_dir)?;
@@ -124,7 +130,18 @@ fn extract(
     output
 }
 
-fn copy_dir(from: &PathBuf, to: &PathBuf) -> Result<(), SourceError> {
+/// The copy_dir function accepts additionally a list of globs to ignore or include in the copy process.
+/// It uses the `ignore` crate to read the `.gitignore` file in the source directory and uses the globs
+/// to filter the files and directories to copy.
+///
+/// The copy process also ignores hidden files and directories by default.
+fn copy_dir(
+    from: &PathBuf,
+    to: &PathBuf,
+    include_globs: &[&str],
+    exclude_globs: &[&str],
+    use_gitignore: bool,
+) -> Result<(), SourceError> {
     // Create the to path because we're going to copy the contents only
     create_all(to, true).unwrap();
 
@@ -133,30 +150,92 @@ fn copy_dir(from: &PathBuf, to: &PathBuf) -> Result<(), SourceError> {
     options.overwrite = true;
     options.content_only = true;
 
-    match copy(from, to, &options) {
-        Ok(_) => tracing::info!(
-            "Copied {} to {}",
-            from.to_string_lossy(),
-            to.to_string_lossy()
-        ),
-        // Use matches as the ErrorKind does not support `==`
-        Err(e) if matches!(e.kind, PermissionDenied) => {
-            tracing::debug!("Permission error in cache, this often happens when the previous run was exited in a faulty way. Removing the cache and retrying the copy.");
-            if let Err(remove_error) = remove(to) {
-                tracing::error!("Failed to remove cache directory: {}", remove_error);
-                return Err(SourceError::FileSystemError(e));
-            } else if let Err(retry_error) = copy(from, to, &options) {
-                tracing::error!("Failed to retry the copy operation: {}", retry_error);
-                return Err(SourceError::FileSystemError(e));
+    // We need an Arc for the glob lists bcause WalkBuilder::filter_entry does not
+    // catch its environment, so we need to move the globs in there.
+    // Because it also needs `Send` (because it uses some Arc machinery internally)
+    // we cannot use a normal Rc here, so we use an Arc
+    fn mkglobset(globs: &[&str]) -> Result<std::sync::Arc<globset::GlobSet>, globset::Error> {
+        let mut globset = globset::GlobSetBuilder::new();
+        for glob in globs {
+            globset.add(globset::Glob::new(glob)?);
+        }
+        globset.build().map(std::sync::Arc::new)
+    }
+
+    let include_globs = mkglobset(include_globs)?;
+    let exclude_globs = mkglobset(exclude_globs)?;
+
+    WalkBuilder::new(from)
+        // disregard global gitignore
+        .git_global(false)
+        .git_ignore(use_gitignore)
+        .hidden(true)
+        .filter_entry(move |entry| {
+            include_globs.is_match(entry.path()) && !exclude_globs.is_match(entry.path())
+        })
+        .build()
+        .try_for_each(|entry| {
+            let entry = entry?;
+            let path = entry.path();
+            let stripped_path = path.strip_prefix(from)?;
+            let dest_path = to.join(stripped_path);
+
+            if path.is_dir() {
+                create_all(&dest_path, true).map_err(SourceError::FileSystemError)
             } else {
+                let file_options = fs_extra::file::CopyOptions {
+                    overwrite: options.overwrite,
+                    skip_exist: options.skip_exist,
+                    buffer_size: options.buffer_size,
+                };
+                fs_extra::file::copy(path, &dest_path, &file_options)
+                    .map_err(SourceError::FileSystemError)?;
+
                 tracing::debug!(
-                    "Successfully retried copying {} to {}",
-                    from.to_string_lossy(),
-                    to.to_string_lossy()
+                    "Copied {} to {}",
+                    path.to_string_lossy(),
+                    dest_path.to_string_lossy()
                 );
+                Ok(())
             }
+        })
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_copy_dir() {
+        let tmp_dir = tempfile::TempDir::new().unwrap();
+        let tmp_dir_path = tmp_dir.into_path();
+        let dir = tmp_dir_path.as_path().join("test_copy_dir");
+
+        fs_extra::dir::create_all(&dir, true).unwrap();
+        std::fs::write(dir.join("test.txt"), "test").unwrap();
+        std::fs::create_dir(dir.join("test_dir")).unwrap();
+        std::fs::write(dir.join("test_dir").join("test.md"), "test").unwrap();
+        std::fs::create_dir(dir.join("test_dir").join("test_dir2")).unwrap();
+
+        let dest_dir = tmp_dir_path.as_path().join("test_copy_dir_dest");
+        super::copy_dir(&dir, &dest_dir, &[], &[], false).unwrap();
+
+        for entry in walkdir::WalkDir::new(dest_dir) {
+            println!("{}", entry.unwrap().path().display());
+        }
+
+        let dest_dir_2 = tmp_dir_path.as_path().join("test_copy_dir_dest_2");
+        // ignore all txt files
+        super::copy_dir(&dir, &dest_dir_2, &["*.txt"], &[], false).unwrap();
+        println!("---------------------");
+        for entry in walkdir::WalkDir::new(dest_dir_2) {
+            println!("{}", entry.unwrap().path().display());
+        }
+
+        let dest_dir_2 = tmp_dir_path.as_path().join("test_copy_dir_dest_2");
+        // ignore all txt files
+        super::copy_dir(&dir, &dest_dir_2, &[], &["*.txt"], false).unwrap();
+        println!("---------------------");
+        for entry in walkdir::WalkDir::new(dest_dir_2) {
+            println!("{}", entry.unwrap().path().display());
         }
-        Err(e) => return Err(SourceError::FileSystemError(e)),
     }
-    Ok(())
 }