From aecb9f903a288ff762b5f28833997f766be36402 Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Thu, 9 Nov 2023 16:13:37 +0530 Subject: [PATCH 1/9] feat: git source with host git --- src/source/host_git_source.rs | 134 ++++++++++++++++++++++++++++++++++ src/source/mod.rs | 23 ++++-- 2 files changed, 149 insertions(+), 8 deletions(-) create mode 100644 src/source/host_git_source.rs diff --git a/src/source/host_git_source.rs b/src/source/host_git_source.rs new file mode 100644 index 000000000..7ea417469 --- /dev/null +++ b/src/source/host_git_source.rs @@ -0,0 +1,134 @@ +use std::{ + borrow::Cow, + path::{Path, PathBuf}, + process::Command, +}; + +use fs_extra::dir::remove; +use itertools::Itertools; + +use crate::recipe::parser::{GitSource, GitUrl}; + +use super::SourceError; + +// git clone file://C:/Users/user/../repo +type RepoPath<'a> = &'a Path; + +pub fn fetch_repo(repo_path: RepoPath<'a>, refspecs: &[String]) -> Result<(), SourceError> { + // might break on some platforms due to auth and ssh + // especially ssh with password + let refspecs_str = refspecs.into_iter().join(" "); + let output = Command::new("git") + .args(["fetch", "origin", refspecs_str.as_str()]) + .output() + .map_err(|err| SourceError::ValidationFailed)?; + + tracing::debug!("Repository fetched successfully!"); + Ok(()) +} + +pub fn git_src( + source: &GitSource, + cache_dir: &Path, + recipe_dir: &Path, +) -> Result { + // on windows there exist some path conversion issue, conda seems to have a solution for it, check + // it out + // figure out more: https://github.com/conda/conda-build/blob/c71c4abee1c85f5a36733c461f224941ab3ebbd1/conda_build/source.py#L38C1-L39C59 + // --- + // tool used: https://cygwin.com/cygwin-ug-net/cygpath.html + // to windows path: cygpath -w unix_path + // to unix path: cyppath -u win_path + // --- + // note: rust on windows handles some of these + + let filename = match &source.url() { + GitUrl::Url(url) => url.path_segments().unwrap().last().unwrap().to_string(), + GitUrl::Path(path) => recipe_dir + .join(path) + .canonicalize()? + .file_name() + .unwrap() + .to_string_lossy() + .to_string(), + }; + + let cache_name = PathBuf::from(filename); + let cache_path = cache_dir.join(cache_name); + + // Initialize or clone the repository depending on the source's git_url. + let fetched = match &source.url() { + GitUrl::Url(_) => { + // If the cache_path exists, initialize the repo and fetch the specified revision. + if cache_path.exists() { + let path = fetch_repo(&cache_path, &[source.rev().to_string()])?; + true + } else { + // TODO: Make configure the clone more so git_depth is also used. + if source.depth().is_some() { + tracing::warn!("No git depth implemented yet, will continue with full clone"); + } + + let out = Command::new("git") + .args(["clone", "--recursive", source.url().to_string().as_str()]) + .output() + .map_err(|_| SourceError::ValidationFailed)?; + if !out.status.success() { + return Err(SourceError::ValidationFailed); + } + let repo_path = String::from_utf8_lossy(&out.stdout); + + if source.rev() == "HEAD" { + // If the source is a path and the revision is HEAD, return the path to avoid git actions. + return Ok(PathBuf::from(&cache_path)); + } + true + } + } + GitUrl::Path(path) => { + if cache_path.exists() { + // Remove old cache so it can be overwritten. + if let Err(remove_error) = remove(&cache_path) { + tracing::error!("Failed to remove old cache directory: {}", remove_error); + return Err(SourceError::FileSystemError(remove_error)); + } + } + + let out = Command::new("git") + .args(["clone", "--recursive", format!("file://{}", path.display()).as_str(), cache_path.display().to_string().as_str()]) + .output() + .map_err(|_| SourceError::ValidationFailed)?; + if !out.status.success() { + return Err(SourceError::ValidationFailed); + } + let repo_path = String::from_utf8_lossy(&out.stdout); + + if source.rev() == "HEAD" { + // If the source is a path and the revision is HEAD, return the path to avoid git actions. + return Ok(PathBuf::from(&cache_path)); + } + true + } + }; + + if !fetched { + return Err(SourceError::GitError("Failed to fetch git repo".to_string())); + } + + // Resolve the reference and set the head to the specified revision. + // let ref_git = format!("refs/remotes/origin/{}", source.git_rev.to_string()); + // let reference = match repo.find_reference(&ref_git) { + let output = Command::new("git") + .args(["checkout", ref_git]) + .output() + .map_err(|_| SourceError::GitError("git checkout".to_string()))?; + + let output = Command::new("git") + .args(["reset", "--hard"]) + .output() + .map_err(|_| SourceError::GitError("git reset --hard".to_string()))?; + + tracing::info!("Checked out reference: '{}'", &source.rev()); + + Ok(cache_path) +} diff --git a/src/source/mod.rs b/src/source/mod.rs index 532c56114..7da7be926 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -7,8 +7,9 @@ use std::{ use crate::recipe::parser::Source; pub mod copy_dir; -#[cfg(feature = "git")] +// #[cfg(feature = "git")] pub mod git_source; +pub mod host_git_source; pub mod patch; pub mod url_source; @@ -38,6 +39,9 @@ pub enum SourceError { #[error("Failed to apply patch: {0}")] PatchFailed(String), + #[error("Failed to run git command: {0}")] + GitError(String), + #[cfg(feature = "git")] #[error("Failed to run git command: {0}")] GitError(#[from] git2::Error), @@ -61,14 +65,16 @@ pub async fn fetch_sources( for src in sources { match &src { - Source::Git(_src) => { - #[cfg(feature = "git")] + Source::Git(src) => { + // we don't seem to notify user if this is run unnecessarily + // #[cfg(feature = "git")] { - tracing::info!("Fetching source from GIT: {}", _src.url()); - let result = match git_source::git_src(_src, &cache_src, recipe_dir) { - Ok(path) => path, - Err(e) => return Err(e), - }; + tracing::info!("Fetching source from GIT: {}", src.url()); + let result = git_source::git_src(src, &cache_src, recipe_dir)?; + // let result = match git_source::git_src(src, &cache_src, recipe_dir) { + // Ok(path) => path, + // Err(e) => return Err(e), + // }; let dest_dir = if let Some(folder) = _src.folder() { work_dir.join(folder) } else { @@ -82,6 +88,7 @@ pub async fn fetch_sources( patch::apply_patches(_src.patches(), work_dir, recipe_dir)?; } } + tracing::info!("Fetching source from git repo: {}", src); } Source::Url(src) => { tracing::info!("Fetching source from URL: {}", src.url()); From 226f344a1cdb9dc22dfe6b2db9bf60a165c119ae Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Thu, 9 Nov 2023 17:47:35 +0530 Subject: [PATCH 2/9] fix: remove old_gitsource --- src/source/mod.rs | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/src/source/mod.rs b/src/source/mod.rs index 7da7be926..92e9c0256 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -8,7 +8,7 @@ use crate::recipe::parser::Source; pub mod copy_dir; // #[cfg(feature = "git")] -pub mod git_source; +// pub mod git_source; pub mod host_git_source; pub mod patch; pub mod url_source; @@ -67,28 +67,19 @@ pub async fn fetch_sources( match &src { Source::Git(src) => { // we don't seem to notify user if this is run unnecessarily - // #[cfg(feature = "git")] - { - tracing::info!("Fetching source from GIT: {}", src.url()); - let result = git_source::git_src(src, &cache_src, recipe_dir)?; - // let result = match git_source::git_src(src, &cache_src, recipe_dir) { - // Ok(path) => path, - // Err(e) => return Err(e), - // }; - let dest_dir = if let Some(folder) = _src.folder() { - work_dir.join(folder) - } else { - work_dir.to_path_buf() - }; - crate::source::copy_dir::CopyDir::new(&result, &dest_dir) - .use_gitignore(false) - .run()?; - - if !_src.patches().is_empty() { - patch::apply_patches(_src.patches(), work_dir, recipe_dir)?; - } + tracing::info!("Fetching source from git repo: {}", src.url()); + let result = host_git_source::git_src(src, &cache_src, recipe_dir)?; + let dest_dir = if let Some(folder) = src.folder() { + work_dir.join(folder) + } else { + work_dir.to_path_buf() + }; + crate::source::copy_dir::CopyDir::new(&result, &dest_dir) + .use_gitignore(false) + .run()?; + if !src.patches().is_empty() { + patch::apply_patches(src.patches(), work_dir, recipe_dir)?; } - tracing::info!("Fetching source from git repo: {}", src); } Source::Url(src) => { tracing::info!("Fetching source from URL: {}", src.url()); From cef2b6fd2143a067c1bfad0a94fc5dbb004bd191 Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Fri, 10 Nov 2023 10:59:43 +0530 Subject: [PATCH 3/9] add unit tests --- src/source/host_git_source.rs | 182 +++++++++++++++++++++++++++++----- 1 file changed, 158 insertions(+), 24 deletions(-) diff --git a/src/source/host_git_source.rs b/src/source/host_git_source.rs index 7ea417469..89e6732df 100644 --- a/src/source/host_git_source.rs +++ b/src/source/host_git_source.rs @@ -14,7 +14,7 @@ use super::SourceError; // git clone file://C:/Users/user/../repo type RepoPath<'a> = &'a Path; -pub fn fetch_repo(repo_path: RepoPath<'a>, refspecs: &[String]) -> Result<(), SourceError> { +pub fn fetch_repo<'a>(repo_path: RepoPath<'a>, refspecs: &[String]) -> Result<(), SourceError> { // might break on some platforms due to auth and ssh // especially ssh with password let refspecs_str = refspecs.into_iter().join(" "); @@ -42,6 +42,13 @@ pub fn git_src( // --- // note: rust on windows handles some of these + println!( + "git src:\n\tsource: {:?}\n\tcache_dir: {}\n\trecipe_dir: {}", + source, + cache_dir.display(), + recipe_dir.display() + ); + let filename = match &source.url() { GitUrl::Url(url) => url.path_segments().unwrap().last().unwrap().to_string(), GitUrl::Path(path) => recipe_dir @@ -57,12 +64,11 @@ pub fn git_src( let cache_path = cache_dir.join(cache_name); // Initialize or clone the repository depending on the source's git_url. - let fetched = match &source.url() { + match &source.url() { GitUrl::Url(_) => { // If the cache_path exists, initialize the repo and fetch the specified revision. if cache_path.exists() { - let path = fetch_repo(&cache_path, &[source.rev().to_string()])?; - true + fetch_repo(&cache_path, &[source.rev().to_string()])?; } else { // TODO: Make configure the clone more so git_depth is also used. if source.depth().is_some() { @@ -70,19 +76,23 @@ pub fn git_src( } let out = Command::new("git") - .args(["clone", "--recursive", source.url().to_string().as_str()]) + .args([ + "clone", + "--recursive", + source.url().to_string().as_str(), + cache_path.to_str().unwrap(), + ]) .output() - .map_err(|_| SourceError::ValidationFailed)?; + .unwrap(); if !out.status.success() { - return Err(SourceError::ValidationFailed); + return Err(SourceError::GitError( + "failed to execute command".to_string(), + )); } - let repo_path = String::from_utf8_lossy(&out.stdout); - if source.rev() == "HEAD" { // If the source is a path and the revision is HEAD, return the path to avoid git actions. return Ok(PathBuf::from(&cache_path)); } - true } } GitUrl::Path(path) => { @@ -94,41 +104,165 @@ pub fn git_src( } } - let out = Command::new("git") - .args(["clone", "--recursive", format!("file://{}", path.display()).as_str(), cache_path.display().to_string().as_str()]) - .output() - .map_err(|_| SourceError::ValidationFailed)?; - if !out.status.success() { - return Err(SourceError::ValidationFailed); - } - let repo_path = String::from_utf8_lossy(&out.stdout); + // TODO(swarnim): remove unwrap + let path = std::fs::canonicalize(path).unwrap(); + + let mut command = Command::new("git"); + let s = format!("{}/.git", path.display()); + command + .arg("clone") + .arg("--recursive") + .arg(format!("file://{}/.git", s).as_str()) + .arg(cache_path.as_os_str()); + let output = command.output().unwrap(); + // .map_err(|_| SourceError::ValidationFailed)?; + assert!( + output.status.success(), + "command: {:#?}\nstdout: {}\nstderr: {}", + command, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + // if !out.status.success() { + // return Err(SourceError::GitError( + // "failed to execute clone from file".to_string(), + // )); + // } if source.rev() == "HEAD" { // If the source is a path and the revision is HEAD, return the path to avoid git actions. return Ok(PathBuf::from(&cache_path)); } - true } - }; - - if !fetched { - return Err(SourceError::GitError("Failed to fetch git repo".to_string())); } // Resolve the reference and set the head to the specified revision. // let ref_git = format!("refs/remotes/origin/{}", source.git_rev.to_string()); // let reference = match repo.find_reference(&ref_git) { let output = Command::new("git") - .args(["checkout", ref_git]) + .current_dir(&cache_path) + .args(["rev-parse", source.rev()]) + .output() + .map_err(|_| SourceError::GitError("git rev-parse failed".to_string()))?; + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let ref_git = String::from_utf8(output.stdout) + .map_err(|_| SourceError::GitError("failed to parse git rev".to_string()))?; + println!("cache_path = {}", cache_path.display()); + let mut command = Command::new("git"); + command + .current_dir(&cache_path) + .arg("checkout") + .arg(ref_git.as_str().trim()); + let output = command .output() .map_err(|_| SourceError::GitError("git checkout".to_string()))?; + assert!( + output.status.success(), + "command: {:#?}\nstdout: {}\nstderr: {}", + command, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); let output = Command::new("git") + .current_dir(&cache_path) .args(["reset", "--hard"]) .output() .map_err(|_| SourceError::GitError("git reset --hard".to_string()))?; + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); tracing::info!("Checked out reference: '{}'", &source.rev()); Ok(cache_path) } + +#[cfg(test)] +mod tests { + use std::env; + + use crate::{ + recipe::parser::{GitSource, GitUrl}, + source::host_git_source::git_src, + }; + + #[test] + fn test_host_git_source() { + let cache_dir = std::env::temp_dir().join("rattler-build-test-git-source"); + let cases = vec![ + ( + GitSource::create( + GitUrl::Url( + "https://github.com/prefix-dev/rattler-build" + .parse() + .unwrap(), + ), + "v0.1.3".to_owned(), + None, + vec![], + None, + ), + "rattler-build", + ), + ( + GitSource::create( + GitUrl::Url( + "https://github.com/prefix-dev/rattler-build" + .parse() + .unwrap(), + ), + "v0.1.2".to_owned(), + None, + vec![], + None, + ), + "rattler-build", + ), + // ( + // GitSrc { + // git_rev: GitRev::from_str("main").unwrap(), + // git_depth: None, + // patches: None, + // git_url: GitUrl::Url( + // "https://github.com/prefix-dev/rattler-build" + // .parse() + // .unwrap(), + // ), + // folder: None, + // }, + // "rattler-build", + // ), + ( + GitSource::create( + GitUrl::Path("../rattler-build".parse().unwrap()), + "".to_owned(), + None, + vec![], + None, + ), + "rattler-build", + ), + ]; + for (source, repo_name) in cases { + let path = git_src( + &source, + cache_dir.as_ref(), + env::current_dir().unwrap().as_ref(), + ) + .unwrap(); + assert_eq!( + path.to_string_lossy(), + cache_dir.join(repo_name).to_string_lossy() + ); + } + } +} From 489c54823a02b61fc84d19a1ca232c7059c45058 Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Mon, 13 Nov 2023 09:05:11 +0530 Subject: [PATCH 4/9] fixes & code cleanup remove unwraps and debug stuff, improve error messages, ensure tests are working on mac os? --- Cargo.toml | 2 - src/source/host_git_source.rs | 128 ++++++++++++++++------------------ src/source/mod.rs | 10 +-- 3 files changed, 66 insertions(+), 74 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 449443a19..6cdaa2271 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ readme = "README.md" default = ['native-tls'] native-tls = ['reqwest/native-tls', 'rattler/native-tls'] rustls-tls = ['reqwest/rustls-tls', 'rattler/rustls-tls'] -git = ['dep:git2'] [dependencies] serde = { version = "1.0.190", features = ["derive"] } @@ -70,7 +69,6 @@ tempfile = "3.8.1" chrono = "0.4.31" sha1 = "0.10.6" spdx = "0.10.2" -git2 = { version = "0.18.1", features = ["vendored-openssl"], optional = true } fs_extra = "1.3.0" ignore = "0.4.20" globset = "0.4.13" diff --git a/src/source/host_git_source.rs b/src/source/host_git_source.rs index 89e6732df..8719ea38f 100644 --- a/src/source/host_git_source.rs +++ b/src/source/host_git_source.rs @@ -1,5 +1,4 @@ use std::{ - borrow::Cow, path::{Path, PathBuf}, process::Command, }; @@ -11,18 +10,21 @@ use crate::recipe::parser::{GitSource, GitUrl}; use super::SourceError; -// git clone file://C:/Users/user/../repo type RepoPath<'a> = &'a Path; pub fn fetch_repo<'a>(repo_path: RepoPath<'a>, refspecs: &[String]) -> Result<(), SourceError> { // might break on some platforms due to auth and ssh // especially ssh with password let refspecs_str = refspecs.into_iter().join(" "); + let cd = std::env::current_dir().ok(); + _ = std::env::set_current_dir(repo_path); let output = Command::new("git") .args(["fetch", "origin", refspecs_str.as_str()]) .output() - .map_err(|err| SourceError::ValidationFailed)?; - + .map_err(|_err| SourceError::ValidationFailed)?; + // TODO(swarnimarun): get rid of assert + assert!(output.status.success()); + _ = cd.map(|cd| std::env::set_current_dir(cd)); tracing::debug!("Repository fetched successfully!"); Ok(()) } @@ -42,7 +44,7 @@ pub fn git_src( // --- // note: rust on windows handles some of these - println!( + tracing::info!( "git src:\n\tsource: {:?}\n\tcache_dir: {}\n\trecipe_dir: {}", source, cache_dir.display(), @@ -55,7 +57,7 @@ pub fn git_src( .join(path) .canonicalize()? .file_name() - .unwrap() + .ok_or_else(|| SourceError::GitErrorStr("Failed to parse "))? .to_string_lossy() .to_string(), }; @@ -70,26 +72,23 @@ pub fn git_src( if cache_path.exists() { fetch_repo(&cache_path, &[source.rev().to_string()])?; } else { - // TODO: Make configure the clone more so git_depth is also used. - if source.depth().is_some() { - tracing::warn!("No git depth implemented yet, will continue with full clone"); + let mut command = Command::new("git"); + command.args([ + "clone", + "--recursive", + source.url().to_string().as_str(), + cache_path.to_str().unwrap(), + ]); + if let Some(depth) = source.depth() { + command.args(["--depth", depth.to_string().as_str()]); } - - let out = Command::new("git") - .args([ - "clone", - "--recursive", - source.url().to_string().as_str(), - cache_path.to_str().unwrap(), - ]) + let output = command .output() - .unwrap(); - if !out.status.success() { - return Err(SourceError::GitError( - "failed to execute command".to_string(), - )); + .map_err(|_e| SourceError::GitErrorStr("Failed to execute clone command"))?; + if !output.status.success() { + return Err(SourceError::GitErrorStr("Git clone failed for source")); } - if source.rev() == "HEAD" { + if source.rev() == "HEAD" || source.rev().trim().is_empty() { // If the source is a path and the revision is HEAD, return the path to avoid git actions. return Ok(PathBuf::from(&cache_path)); } @@ -103,33 +102,31 @@ pub fn git_src( return Err(SourceError::FileSystemError(remove_error)); } } - - // TODO(swarnim): remove unwrap - let path = std::fs::canonicalize(path).unwrap(); + let path = std::fs::canonicalize(path).map_err(|e| { + tracing::error!("Path not found on system: {}", e); + SourceError::GitError(format!("{}: Path not found on system", e.to_string())) + })?; let mut command = Command::new("git"); - let s = format!("{}/.git", path.display()); command .arg("clone") .arg("--recursive") - .arg(format!("file://{}/.git", s).as_str()) + .arg(format!("file://{}/.git", path.display()).as_str()) .arg(cache_path.as_os_str()); - let output = command.output().unwrap(); - // .map_err(|_| SourceError::ValidationFailed)?; - assert!( - output.status.success(), - "command: {:#?}\nstdout: {}\nstderr: {}", - command, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - // if !out.status.success() { - // return Err(SourceError::GitError( - // "failed to execute clone from file".to_string(), - // )); - // } + if let Some(depth) = source.depth() { + command.args(["--depth", depth.to_string().as_str()]); + } + let output = command + .output() + .map_err(|_| SourceError::ValidationFailed)?; + if !output.status.success() { + tracing::error!("Command failed: {:?}", command); + return Err(SourceError::GitErrorStr( + "failed to execute clone from file", + )); + } - if source.rev() == "HEAD" { + if source.rev() == "HEAD" || source.rev().trim().is_empty() { // If the source is a path and the revision is HEAD, return the path to avoid git actions. return Ok(PathBuf::from(&cache_path)); } @@ -143,43 +140,40 @@ pub fn git_src( .current_dir(&cache_path) .args(["rev-parse", source.rev()]) .output() - .map_err(|_| SourceError::GitError("git rev-parse failed".to_string()))?; - assert!( - output.status.success(), - "stdout: {}\nstderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); + .map_err(|_| SourceError::GitErrorStr("git rev-parse failed"))?; + if !output.status.success() { + tracing::error!("Command failed: \"git\" \"rev-parse\" \"{}\"", source.rev()); + return Err(SourceError::GitErrorStr("failed to get valid hash for rev")); + } let ref_git = String::from_utf8(output.stdout) - .map_err(|_| SourceError::GitError("failed to parse git rev".to_string()))?; - println!("cache_path = {}", cache_path.display()); + .map_err(|_| SourceError::GitErrorStr("failed to parse git rev as utf-8"))?; + tracing::info!("cache_path = {}", cache_path.display()); + let mut command = Command::new("git"); command .current_dir(&cache_path) .arg("checkout") .arg(ref_git.as_str().trim()); + let output = command .output() - .map_err(|_| SourceError::GitError("git checkout".to_string()))?; - assert!( - output.status.success(), - "command: {:#?}\nstdout: {}\nstderr: {}", - command, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); + .map_err(|_| SourceError::GitErrorStr("failed to execute git checkout"))?; + + if !output.status.success() { + tracing::error!("Command failed: {:?}", command); + return Err(SourceError::GitErrorStr("failed to checkout for ref")); + } let output = Command::new("git") .current_dir(&cache_path) .args(["reset", "--hard"]) .output() - .map_err(|_| SourceError::GitError("git reset --hard".to_string()))?; - assert!( - output.status.success(), - "stdout: {}\nstderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); + .map_err(|_| SourceError::GitErrorStr("failed to execute git reset"))?; + + if !output.status.success() { + tracing::error!("Command failed: \"git\" \"reset\" \"--hard\""); + return Err(SourceError::GitErrorStr("failed to git reset")); + } tracing::info!("Checked out reference: '{}'", &source.rev()); diff --git a/src/source/mod.rs b/src/source/mod.rs index 92e9c0256..af0e5a4d1 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -7,8 +7,6 @@ use std::{ use crate::recipe::parser::Source; pub mod copy_dir; -// #[cfg(feature = "git")] -// pub mod git_source; pub mod host_git_source; pub mod patch; pub mod url_source; @@ -42,6 +40,9 @@ pub enum SourceError { #[error("Failed to run git command: {0}")] GitError(String), + #[error("Failed to run git command: {0}")] + GitErrorStr(&'static str), + #[cfg(feature = "git")] #[error("Failed to run git command: {0}")] GitError(#[from] git2::Error), @@ -66,9 +67,8 @@ pub async fn fetch_sources( for src in sources { match &src { Source::Git(src) => { - // we don't seem to notify user if this is run unnecessarily - tracing::info!("Fetching source from git repo: {}", src.url()); - let result = host_git_source::git_src(src, &cache_src, recipe_dir)?; + tracing::info!("Fetching source from git repo: {}", src.url()); + let result = host_git_source::git_src(src, &cache_src, recipe_dir)?; let dest_dir = if let Some(folder) = src.folder() { work_dir.join(folder) } else { From 88614f436b19012eeb93f877a109cc0ecd89654e Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Mon, 13 Nov 2023 09:10:58 +0530 Subject: [PATCH 5/9] fix: fix clippy recs & update lock file --- Cargo.lock | 66 ----------------------------------- src/source/host_git_source.rs | 8 ++--- 2 files changed, 4 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0eb1f0dc..4ff47a0b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1144,21 +1144,6 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" -[[package]] -name = "git2" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" -dependencies = [ - "bitflags 2.4.1", - "libc", - "libgit2-sys", - "log", - "openssl-probe", - "openssl-sys", - "url", -] - [[package]] name = "glob" version = "0.3.1" @@ -1611,20 +1596,6 @@ version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" -[[package]] -name = "libgit2-sys" -version = "0.16.1+1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c" -dependencies = [ - "cc", - "libc", - "libssh2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", -] - [[package]] name = "libloading" version = "0.8.1" @@ -1641,32 +1612,6 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" -[[package]] -name = "libssh2-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "line-wrap" version = "0.1.1" @@ -2060,15 +2005,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-src" -version = "300.1.6+3.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.94" @@ -2077,7 +2013,6 @@ checksum = "2f55da20b29f956fb01f0add8683eb26ee13ebe3ebd935e49898717c6b4b2830" dependencies = [ "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] @@ -2432,7 +2367,6 @@ dependencies = [ "content_inspector", "fs_extra", "futures 0.3.29", - "git2", "globset", "goblin", "hex", diff --git a/src/source/host_git_source.rs b/src/source/host_git_source.rs index 8719ea38f..2e6ed7871 100644 --- a/src/source/host_git_source.rs +++ b/src/source/host_git_source.rs @@ -12,10 +12,10 @@ use super::SourceError; type RepoPath<'a> = &'a Path; -pub fn fetch_repo<'a>(repo_path: RepoPath<'a>, refspecs: &[String]) -> Result<(), SourceError> { +pub fn fetch_repo(repo_path: RepoPath, refspecs: &[String]) -> Result<(), SourceError> { // might break on some platforms due to auth and ssh // especially ssh with password - let refspecs_str = refspecs.into_iter().join(" "); + let refspecs_str = refspecs.iter().join(" "); let cd = std::env::current_dir().ok(); _ = std::env::set_current_dir(repo_path); let output = Command::new("git") @@ -24,7 +24,7 @@ pub fn fetch_repo<'a>(repo_path: RepoPath<'a>, refspecs: &[String]) -> Result<() .map_err(|_err| SourceError::ValidationFailed)?; // TODO(swarnimarun): get rid of assert assert!(output.status.success()); - _ = cd.map(|cd| std::env::set_current_dir(cd)); + _ = cd.map(std::env::set_current_dir); tracing::debug!("Repository fetched successfully!"); Ok(()) } @@ -104,7 +104,7 @@ pub fn git_src( } let path = std::fs::canonicalize(path).map_err(|e| { tracing::error!("Path not found on system: {}", e); - SourceError::GitError(format!("{}: Path not found on system", e.to_string())) + SourceError::GitError(format!("{}: Path not found on system", e)) })?; let mut command = Command::new("git"); From 930990b4fb19af4cabc310ec7ef3f8c674ca33d5 Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Mon, 13 Nov 2023 10:46:29 +0530 Subject: [PATCH 6/9] add git lfs support if installed --- src/source/host_git_source.rs | 65 +++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/src/source/host_git_source.rs b/src/source/host_git_source.rs index 2e6ed7871..3ec3a5a05 100644 --- a/src/source/host_git_source.rs +++ b/src/source/host_git_source.rs @@ -45,12 +45,18 @@ pub fn git_src( // note: rust on windows handles some of these tracing::info!( - "git src:\n\tsource: {:?}\n\tcache_dir: {}\n\trecipe_dir: {}", + "git source: ({:?}) cache_dir: ({}) recipe_dir: ({})", source, cache_dir.display(), recipe_dir.display() ); + // TODO: handle reporting for unavailability of git better, or perhaps pointing to git binary manually? + // currently a solution is to provide a `git` early in PATH with, + // ```bash + // export PATH="/path/to/git:$PATH" + // ``` + let filename = match &source.url() { GitUrl::Url(url) => url.path_segments().unwrap().last().unwrap().to_string(), GitUrl::Path(path) => recipe_dir @@ -175,11 +181,40 @@ pub fn git_src( return Err(SourceError::GitErrorStr("failed to git reset")); } + // TODO(swarnimarun): does this need to be handled! + let _lfs_success = git_lfs_pull()?; + tracing::info!("Checked out reference: '{}'", &source.rev()); Ok(cache_path) } +fn git_lfs_pull() -> Result { + // verify lfs install + let mut command = Command::new("git"); + command.args(["lfs", "install"]); + let output = command + .output() + .map_err(|_| SourceError::GitErrorStr("failed to execute command"))?; + if !output.status.success() { + tracing::error!("`git lfs install` failed!"); + return Ok(false); + } + + // git lfs pull + let mut command = Command::new("git"); + command.args(["lfs", "pull"]); + let output = command + .output() + .map_err(|_| SourceError::GitErrorStr("failed to execute command"))?; + if !output.status.success() { + tracing::error!("`git lfs pull` failed!"); + return Ok(false); + } + + Ok(true) +} + #[cfg(test)] mod tests { use std::env; @@ -193,6 +228,20 @@ mod tests { fn test_host_git_source() { let cache_dir = std::env::temp_dir().join("rattler-build-test-git-source"); let cases = vec![ + ( + GitSource::create( + GitUrl::Url( + "https://github.com/prefix-dev/rattler-build" + .parse() + .unwrap(), + ), + "main".to_owned(), + None, + vec![], + None, + ), + "rattler-build", + ), ( GitSource::create( GitUrl::Url( @@ -221,20 +270,6 @@ mod tests { ), "rattler-build", ), - // ( - // GitSrc { - // git_rev: GitRev::from_str("main").unwrap(), - // git_depth: None, - // patches: None, - // git_url: GitUrl::Url( - // "https://github.com/prefix-dev/rattler-build" - // .parse() - // .unwrap(), - // ), - // folder: None, - // }, - // "rattler-build", - // ), ( GitSource::create( GitUrl::Path("../rattler-build".parse().unwrap()), From abd2cadf665826a2cf317abdbf7574b41d1f6e8e Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Mon, 13 Nov 2023 15:26:40 +0530 Subject: [PATCH 7/9] fix: paths for windows --- Cargo.lock | 31 +++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/source/host_git_source.rs | 24 +++++++++++++++--------- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ff47a0b7..ef9aadfe1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,6 +809,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" + [[package]] name = "either" version = "1.9.0" @@ -2365,6 +2371,7 @@ dependencies = [ "comfy-table", "console", "content_inspector", + "dunce", "fs_extra", "futures 0.3.29", "globset", @@ -2407,6 +2414,7 @@ dependencies = [ "tracing-core", "tracing-indicatif", "tracing-subscriber", + "tracing-test", "url", "walkdir", ] @@ -3672,6 +3680,29 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tracing-test" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4" +dependencies = [ + "lazy_static", + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" +dependencies = [ + "lazy_static", + "quote", + "syn 1.0.109", +] + [[package]] name = "treediff" version = "4.0.2" diff --git a/Cargo.toml b/Cargo.toml index 31e81819b..2d6562207 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,10 +80,12 @@ globset = "0.4.13" tracing-indicatif = "0.3.5" clap-verbosity-flag = "2.1.0" tracing-core = "0.1.32" +dunce = "1.0.4" [dev-dependencies] insta = { version = "1.34.0", features = ["yaml"] } rstest = "0.18.2" +tracing-test = "0.2.4" [profile.dev.package."*"] opt-level = 3 diff --git a/src/source/host_git_source.rs b/src/source/host_git_source.rs index 3ec3a5a05..a4e6fdd41 100644 --- a/src/source/host_git_source.rs +++ b/src/source/host_git_source.rs @@ -23,7 +23,7 @@ pub fn fetch_repo(repo_path: RepoPath, refspecs: &[String]) -> Result<(), Source .output() .map_err(|_err| SourceError::ValidationFailed)?; // TODO(swarnimarun): get rid of assert - assert!(output.status.success()); + assert!(output.status.success(), "{:#?}", output); _ = cd.map(std::env::set_current_dir); tracing::debug!("Repository fetched successfully!"); Ok(()) @@ -58,12 +58,14 @@ pub fn git_src( // ``` let filename = match &source.url() { - GitUrl::Url(url) => url.path_segments().unwrap().last().unwrap().to_string(), + GitUrl::Url(url) => (|| Some(url.path_segments()?.last()?.to_string()))() + .ok_or_else(|| SourceError::GitErrorStr("failed to get filename from url"))?, GitUrl::Path(path) => recipe_dir .join(path) .canonicalize()? .file_name() - .ok_or_else(|| SourceError::GitErrorStr("Failed to parse "))? + // canonicalized paths shouldn't end with .. + .unwrap() .to_string_lossy() .to_string(), }; @@ -108,16 +110,18 @@ pub fn git_src( return Err(SourceError::FileSystemError(remove_error)); } } - let path = std::fs::canonicalize(path).map_err(|e| { + // git doesn't support UNC paths, hence we can't use std::fs::canonicalize + let path = dunce::canonicalize(path).map_err(|e| { tracing::error!("Path not found on system: {}", e); SourceError::GitError(format!("{}: Path not found on system", e)) })?; + let path = path.to_string_lossy(); let mut command = Command::new("git"); command .arg("clone") .arg("--recursive") - .arg(format!("file://{}/.git", path.display()).as_str()) + .arg(format!("file://{}/.git", path).as_str()) .arg(cache_path.as_os_str()); if let Some(depth) = source.depth() { command.args(["--depth", depth.to_string().as_str()]); @@ -217,16 +221,16 @@ fn git_lfs_pull() -> Result { #[cfg(test)] mod tests { - use std::env; - use crate::{ recipe::parser::{GitSource, GitUrl}, source::host_git_source::git_src, }; + #[tracing_test::traced_test] #[test] fn test_host_git_source() { - let cache_dir = std::env::temp_dir().join("rattler-build-test-git-source"); + let temp_dir = tempfile::tempdir().unwrap(); + let cache_dir = temp_dir.path().join("rattler-build-test-git-source"); let cases = vec![ ( GitSource::create( @@ -285,7 +289,9 @@ mod tests { let path = git_src( &source, cache_dir.as_ref(), - env::current_dir().unwrap().as_ref(), + // TODO: this test assumes current dir is the root folder of the project which may + // not be necessary for local runs. + std::env::current_dir().unwrap().as_ref(), ) .unwrap(); assert_eq!( From c6900c3844395a7d1e63d8eeda55e99c42584a4c Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Tue, 14 Nov 2023 13:15:47 +0530 Subject: [PATCH 8/9] fix: handle lfs better --- src/source/host_git_source.rs | 41 +++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/source/host_git_source.rs b/src/source/host_git_source.rs index a4e6fdd41..8360f035d 100644 --- a/src/source/host_git_source.rs +++ b/src/source/host_git_source.rs @@ -34,16 +34,6 @@ pub fn git_src( cache_dir: &Path, recipe_dir: &Path, ) -> Result { - // on windows there exist some path conversion issue, conda seems to have a solution for it, check - // it out - // figure out more: https://github.com/conda/conda-build/blob/c71c4abee1c85f5a36733c461f224941ab3ebbd1/conda_build/source.py#L38C1-L39C59 - // --- - // tool used: https://cygwin.com/cygwin-ug-net/cygpath.html - // to windows path: cygpath -w unix_path - // to unix path: cyppath -u win_path - // --- - // note: rust on windows handles some of these - tracing::info!( "git source: ({:?}) cache_dir: ({}) recipe_dir: ({})", source, @@ -185,14 +175,41 @@ pub fn git_src( return Err(SourceError::GitErrorStr("failed to git reset")); } - // TODO(swarnimarun): does this need to be handled! - let _lfs_success = git_lfs_pull()?; + if git_lfs_required(&cache_path) { + if !git_lfs_pull()? { + // failed to do lfs pull, likely lfs not installed + // TODO: should we consider erroring out? + // return Err(SourceError::GitErrorStr( + // "failed to perform lfs pull, possibly git-lfs not installed", + // )); + } + } tracing::info!("Checked out reference: '{}'", &source.rev()); Ok(cache_path) } +// TODO: we can use parallelization and work splitting for much faster search +// not sure if it's required though +fn git_lfs_required(repo_path: RepoPath) -> bool { + // scan `**/.gitattributes` + walkdir::WalkDir::new(repo_path) + .follow_links(false) + .into_iter() + .filter_entry(|d| { + // ignore .git folder (or folders in case of submodules) + (d.file_type().is_dir() && !d.file_name().to_string_lossy().contains(".git")) + || d.file_name() + .to_string_lossy() + .starts_with(".gitattributes") + }) + .filter_map(|d| d.ok()) + .filter(|d| d.file_type().is_file()) + .filter_map(|d| std::fs::read_to_string(d.path()).ok()) + .any(|s| s.lines().any(|l| l.contains("lfs"))) +} + fn git_lfs_pull() -> Result { // verify lfs install let mut command = Command::new("git"); From e5df397f3fe7fa0f9cd9c0c8eaec768c58614ab4 Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Tue, 14 Nov 2023 14:12:12 +0530 Subject: [PATCH 9/9] add e2e test --- rust-tests/src/lib.rs | 14 ++ src/source/git_source.rs | 220 ------------------------- src/source/host_git_source.rs | 14 +- test-data/recipes/llamacpp/recipe.yaml | 28 ++++ 4 files changed, 48 insertions(+), 228 deletions(-) delete mode 100644 src/source/git_source.rs create mode 100644 test-data/recipes/llamacpp/recipe.yaml diff --git a/rust-tests/src/lib.rs b/rust-tests/src/lib.rs index c1a2a48e9..2deed3870 100644 --- a/rust-tests/src/lib.rs +++ b/rust-tests/src/lib.rs @@ -326,4 +326,18 @@ mod tests { assert_eq!(std::fs::read_to_string(installer).unwrap().trim(), "conda"); check_info(pkg, recipes().join("toml/expected")); } + + #[test] + fn test_git_source() { + let tmp = tmp(); + let rattler_build = + rattler().build::<_, _, &str>(recipes().join("llamacpp"), tmp.as_dir(), None); + assert!(rattler_build.is_ok()); + let pkg = get_extracted_package(tmp.as_dir(), "llama.cpp"); + // this is to ensure that the clone happens correctly + let license = pkg.join("info/licenses/LICENSE"); + assert!(license.exists()); + let src = std::fs::read_to_string(license).unwrap(); + assert!(src.contains(" Georgi ")); + } } diff --git a/src/source/git_source.rs b/src/source/git_source.rs deleted file mode 100644 index 598a582ee..000000000 --- a/src/source/git_source.rs +++ /dev/null @@ -1,220 +0,0 @@ -use std::ops::Deref; -use std::path::{Path, PathBuf}; - -// use crate::metadata::GitUrl; -use git2::{Cred, FetchOptions, ObjectType, RemoteCallbacks, Repository, ResetType}; - -use crate::recipe::parser::{GitSource, GitUrl}; - -// use super::super::metadata::GitSrc; -use super::SourceError; - -use fs_extra::dir::remove; - -/// Fetch the repo and its specific refspecs. -fn fetch_repo(repo: &Repository, refspecs: &[String]) -> Result<(), git2::Error> { - let mut remote = repo.find_remote("origin")?; - - let mut callbacks = RemoteCallbacks::new(); - callbacks.credentials(|_url, username_from_url, _allowed_types| { - Cred::ssh_key_from_agent(username_from_url.unwrap_or("git")) - }); - - let mut fetch_options = FetchOptions::new(); - fetch_options.remote_callbacks(callbacks); - - remote.fetch(refspecs, Some(&mut fetch_options), None)?; - - tracing::debug!("Repository fetched successfully!"); - - Ok(()) -} - -/// Find or create the cache for a git repository. -/// -/// This function has multiple paths depending on the source of the Git repository: -/// 1. The source is a git URL: -/// a. If the cache for the package exists, fetch and checkout the specified revision. -/// b. If there is no cache, perform a recursive clone. -/// 2. The source is a local path: -/// a. If the specified revision is HEAD, do a local clone and return the cache path, because no action on the repo is needed. -/// b. If any other revision is specified, clone the repo to the cache path and perform a checkout of the specified revision. -/// -/// # Arguments -/// - source: The GitSrc struct containing information about the source git repository. -/// - cache_dir: The base cache directory where the repository will be stored. -/// -/// # Returns -/// - A Result containing the PathBuf to the cache, or a SourceError if an error occurs during the process. -pub(crate) fn git_src<'a>( - source: &'a GitSource, - cache_dir: &'a Path, - recipe_dir: &'a Path, -) -> Result { - // Create cache path based on given cache dir and name of the source package. - let filename = match &source.url() { - GitUrl::Url(url) => url.path_segments().unwrap().last().unwrap().to_string(), - GitUrl::Path(path) => recipe_dir - .join(path) - .canonicalize()? - .file_name() - .unwrap() - .to_string_lossy() - .to_string(), - }; - let cache_name = PathBuf::from(filename); - let cache_path = cache_dir.join(cache_name); - - // Initialize or clone the repository depending on the source's git_url. - let repo = match &source.url() { - GitUrl::Url(_) => { - // If the cache_path exists, initialize the repo and fetch the specified revision. - if cache_path.exists() { - let repo = Repository::init(&cache_path).unwrap(); - fetch_repo(&repo, &[source.rev().to_string()])?; - repo - } else { - // TODO: Make configure the clone more so git_depth is also used. - if source.depth().is_some() { - tracing::warn!("No git depth implemented yet, will continue with full clone"); - } - - // Clone the repository recursively to include all submodules. - match Repository::clone_recurse(&source.url().to_string(), &cache_path) { - Ok(repo) => repo, - Err(e) => return Err(SourceError::GitError(e)), - } - } - } - GitUrl::Path(path) => { - if cache_path.exists() { - // Remove old cache so it can be overwritten. - if let Err(remove_error) = remove(&cache_path) { - tracing::error!("Failed to remove old cache directory: {}", remove_error); - return Err(SourceError::FileSystemError(remove_error)); - } - } - - let repo = Repository::clone_recurse( - recipe_dir - .join(path) - .canonicalize()? - .to_string_lossy() - .deref(), - &cache_path, - )?; - - if source.rev() == "HEAD" { - // If the source is a path and the revision is HEAD, return the path to avoid git actions. - return Ok(PathBuf::from(&cache_path)); - } - repo - } - }; - - // Resolve the reference and set the head to the specified revision. - // let ref_git = format!("refs/remotes/origin/{}", source.git_rev.to_string()); - // let reference = match repo.find_reference(&ref_git) { - let reference = match repo.resolve_reference_from_short_name(source.rev()) { - Ok(reference) => reference, - Err(_) => { - match repo.resolve_reference_from_short_name(&format!("origin/{}", source.rev())) { - Ok(reference) => reference, - Err(e) => { - return Err(SourceError::GitError(e)); - } - } - } - }; - let object = reference.peel(ObjectType::Commit).unwrap(); - repo.set_head(reference.name().unwrap())?; - repo.reset(&object, ResetType::Hard, None)?; - tracing::info!("Checked out reference: '{}'", &source.rev()); - - // TODO: Implement support for pulling Git LFS files, as git2 does not support it. - Ok(cache_path) -} - -#[cfg(test)] -mod tests { - use std::env; - - use git2::Repository; - - use crate::{ - recipe::parser::{GitSource, GitUrl}, - source::git_source::git_src, - }; - - #[test] - fn test_git_source() { - let cache_dir = "/tmp/rattler-build-test-git-source"; - let cases = vec![ - ( - GitSource::create( - GitUrl::Url( - "https://github.com/prefix-dev/rattler-build" - .parse() - .unwrap(), - ), - "v0.1.3".to_owned(), - None, - vec![], - None, - ), - "rattler-build", - ), - ( - GitSource::create( - GitUrl::Url( - "https://github.com/prefix-dev/rattler-build" - .parse() - .unwrap(), - ), - "v0.1.2".to_owned(), - None, - vec![], - None, - ), - "rattler-build", - ), - // ( - // GitSrc { - // git_rev: GitRev::from_str("main").unwrap(), - // git_depth: None, - // patches: None, - // git_url: GitUrl::Url( - // "https://github.com/prefix-dev/rattler-build" - // .parse() - // .unwrap(), - // ), - // folder: None, - // }, - // "rattler-build", - // ), - ( - GitSource::create( - GitUrl::Path("../rattler-build".parse().unwrap()), - "".to_owned(), - None, - vec![], - None, - ), - "rattler-build", - ), - ]; - for (source, repo_name) in cases { - let path = git_src( - &source, - cache_dir.as_ref(), - env::current_dir().unwrap().as_ref(), - ) - .unwrap(); - Repository::init(&path).expect("Could not create repo with the path speciefied."); - assert_eq!( - path.to_string_lossy(), - (cache_dir.to_owned() + "/" + repo_name) - ); - } - } -} diff --git a/src/source/host_git_source.rs b/src/source/host_git_source.rs index 8360f035d..a14316f7b 100644 --- a/src/source/host_git_source.rs +++ b/src/source/host_git_source.rs @@ -175,14 +175,12 @@ pub fn git_src( return Err(SourceError::GitErrorStr("failed to git reset")); } - if git_lfs_required(&cache_path) { - if !git_lfs_pull()? { - // failed to do lfs pull, likely lfs not installed - // TODO: should we consider erroring out? - // return Err(SourceError::GitErrorStr( - // "failed to perform lfs pull, possibly git-lfs not installed", - // )); - } + if git_lfs_required(&cache_path) && !git_lfs_pull()? { + // failed to do lfs pull, likely lfs not installed + // TODO: should we consider erroring out? + // return Err(SourceError::GitErrorStr( + // "failed to perform lfs pull, possibly git-lfs not installed", + // )); } tracing::info!("Checked out reference: '{}'", &source.rev()); diff --git a/test-data/recipes/llamacpp/recipe.yaml b/test-data/recipes/llamacpp/recipe.yaml new file mode 100644 index 000000000..db1d4f1e9 --- /dev/null +++ b/test-data/recipes/llamacpp/recipe.yaml @@ -0,0 +1,28 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json + +context: + name: llama.cpp + version: b1513 + +package: + name: "${{ name|lower }}" + version: "${{ version }}" + +source: + git_url: https://github.com/ggerganov/llama.cpp.git + git_rev: "${{ version }}" + +build: + number: 0 + script: cmake -B build + +requirements: + build: + - ${{ compiler('c') }} + - cmake + +about: + homepage: https://github.com/ggerganov/llama.cpp + license: MIT + license_file: LICENSE + summary: Port of Facebook's LLaMA model in C/C++