diff --git a/Cargo.lock b/Cargo.lock index d69c1ac8f87..cf16cb72289 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1633,6 +1633,7 @@ dependencies = [ "git-tempfile", "git-testtools", "git-validate", + "git-worktree", "memmap2", "nom", "serde", @@ -1898,7 +1899,6 @@ dependencies = [ "futures-lite", "git-features", "git-repository", - "git-transport", "gitoxide-core", "owo-colors", "prodash", diff --git a/Cargo.toml b/Cargo.toml index 579702a4261..e19b7e8c678 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ gitoxide-core-tools = ["gitoxide-core/organize", "gitoxide-core/estimate-hours"] ## Use blocking client networking. gitoxide-core-blocking-client = ["gitoxide-core/blocking-client"] ## Support synchronous 'http' and 'https' transports (e.g. for clone, fetch and push) at the expense of compile times and binary size. -http-client-curl = ["git-transport-for-configuration-only/http-client-curl"] +http-client-curl = ["git-repository/blocking-http-transport"] ## Use async client networking. gitoxide-core-async-client = ["gitoxide-core/async-client", "futures-lite"] @@ -86,8 +86,6 @@ gitoxide-core = { version = "^0.20.0", path = "gitoxide-core" } git-features = { version = "^0.23.1", path = "git-features" } git-repository = { version = "^0.27.0", path = "git-repository", default-features = false } -git-transport-for-configuration-only = { package = "git-transport", optional = true, version = "^0.21.2", path = "git-transport" } - clap = { version = "3.2.5", features = ["derive", "cargo"] } prodash = { version = "21", optional = true, default-features = false } atty = { version = "0.2.14", optional = true, default-features = false } diff --git a/STABILITY.md b/STABILITY.md index 439a6a052d7..9a1b1bd0dde 100644 --- a/STABILITY.md +++ b/STABILITY.md @@ -136,6 +136,7 @@ How do we avoid staying in the initial development phase (IDP) forever? There is a couple of questions to ask and answer positively: +- _Is everything in it's tracking issue named "`` towards 1.0" resolved?_ - _Does the crate fulfill its intended purpose well enough?_ - _Do the dependent workspace crates fulfill their intended purposes well enough?_ - _Do they hide types and functionality of lower-tier workspace crates and external IDP crates?_ diff --git a/git-config-value/src/types.rs b/git-config-value/src/types.rs index e5a47fe5074..239679c703d 100644 --- a/git-config-value/src/types.rs +++ b/git-config-value/src/types.rs @@ -25,7 +25,7 @@ pub struct Color { /// suffix after fetching the value. [`integer::Suffix`] provides /// [`bitwise_offset()`][integer::Suffix::bitwise_offset] to help with the /// math, or [to_decimal()][Integer::to_decimal()] for obtaining a usable value in one step. -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] +#[derive(Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] pub struct Integer { /// The value, without any suffix modification pub value: i64, @@ -34,7 +34,7 @@ pub struct Integer { } /// Any value that can be interpreted as a boolean. -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] +#[derive(Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)] #[allow(missing_docs)] pub struct Boolean(pub bool); diff --git a/git-glob/tests/matching/mod.rs b/git-glob/tests/matching/mod.rs deleted file mode 100644 index 8b137891791..00000000000 --- a/git-glob/tests/matching/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/git-protocol/Cargo.toml b/git-protocol/Cargo.toml index 3a1e7898f30..3d629cb3df6 100644 --- a/git-protocol/Cargo.toml +++ b/git-protocol/Cargo.toml @@ -40,7 +40,7 @@ required-features = ["async-client"] [dependencies] git-features = { version = "^0.23.1", path = "../git-features", features = ["progress"] } -git-transport = { version = "^0.21.1", path = "../git-transport" } +git-transport = { version = "^0.21.2", path = "../git-transport" } git-hash = { version = "^0.9.11", path = "../git-hash" } git-credentials = { version = "^0.6.1", path = "../git-credentials" } diff --git a/git-protocol/src/fetch/command.rs b/git-protocol/src/command/mod.rs similarity index 94% rename from git-protocol/src/fetch/command.rs rename to git-protocol/src/command/mod.rs index 0a82d6ec2f3..bfe4657891e 100644 --- a/git-protocol/src/fetch/command.rs +++ b/git-protocol/src/command/mod.rs @@ -1,14 +1,9 @@ -/// The kind of command to invoke on the server side. -#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] -pub enum Command { - /// List references. - LsRefs, - /// Fetch a pack. - Fetch, -} +//! V2 command abstraction to validate invocations and arguments, like a database of what we know about them. +use super::Command; +use std::borrow::Cow; /// A key value pair of values known at compile time. -pub type Feature = (&'static str, Option<&'static str>); +pub type Feature = (&'static str, Option>); impl Command { /// Produce the name of the command as known by the server side. @@ -25,7 +20,7 @@ mod with_io { use bstr::{BString, ByteSlice}; use git_transport::client::Capabilities; - use crate::fetch::{agent, command::Feature, Command}; + use crate::{command::Feature, Command}; impl Command { /// Only V2 @@ -134,14 +129,13 @@ mod with_io { feature => server_capabilities.contains(feature), }) .map(|s| (s, None)) - .chain(Some(agent())) .collect() } git_transport::Protocol::V2 => { let supported_features: Vec<_> = server_capabilities .iter() .find_map(|c| { - if c.name() == Command::Fetch.as_str().as_bytes().as_bstr() { + if c.name() == Command::Fetch.as_str() { c.values().map(|v| v.map(|f| f.to_owned()).collect()) } else { None @@ -153,11 +147,10 @@ mod with_io { .copied() .filter(|feature| supported_features.iter().any(|supported| supported == feature)) .map(|s| (s, None)) - .chain(Some(agent())) .collect() } }, - Command::LsRefs => vec![agent()], + Command::LsRefs => vec![], } } /// Panics if the given arguments and features don't match what's statically known. It's considered a bug in the delegate. @@ -212,3 +205,6 @@ mod with_io { } } } + +#[cfg(test)] +mod tests; diff --git a/git-protocol/src/fetch/tests/command.rs b/git-protocol/src/command/tests.rs similarity index 89% rename from git-protocol/src/fetch/tests/command.rs rename to git-protocol/src/command/tests.rs index 10b87547b0e..c7ced620031 100644 --- a/git-protocol/src/fetch/tests/command.rs +++ b/git-protocol/src/command/tests.rs @@ -8,11 +8,8 @@ mod v1 { const GITHUB_CAPABILITIES: &str = "multi_ack thin-pack side-band ofs-delta shallow deepen-since deepen-not deepen-relative no-progress include-tag allow-tip-sha1-in-want allow-reachable-sha1-in-want no-done symref=HEAD:refs/heads/main filter agent=git/github-gdf51a71f0236"; mod fetch { mod default_features { - use crate::fetch::{ - self, - tests::command::v1::{capabilities, GITHUB_CAPABILITIES}, - Command, - }; + use crate::command::tests::v1::{capabilities, GITHUB_CAPABILITIES}; + use crate::Command; #[test] fn it_chooses_the_best_multi_ack_and_sideband() { @@ -21,7 +18,7 @@ mod v1 { git_transport::Protocol::V1, &capabilities("multi_ack side-band side-band-64k multi_ack_detailed") ), - &[("side-band-64k", None), ("multi_ack_detailed", None), fetch::agent()] + &[("side-band-64k", None), ("multi_ack_detailed", None),] ); } @@ -42,7 +39,6 @@ mod v1 { ("allow-reachable-sha1-in-want", None), ("no-done", None), ("filter", None), - fetch::agent() ], "we don't enforce include-tag or no-progress" ); @@ -61,7 +57,8 @@ mod v2 { mod fetch { mod default_features { - use crate::fetch::{self, tests::command::v2::capabilities, Command}; + use crate::command::tests::v2::capabilities; + use crate::Command; #[test] fn all_features() { @@ -73,7 +70,6 @@ mod v2 { ["shallow", "filter", "ref-in-want", "sideband-all", "packfile-uris"] .iter() .map(|s| (*s, None)) - .chain(Some(fetch::agent())) .collect::>() ) } @@ -82,7 +78,8 @@ mod v2 { mod initial_arguments { use bstr::ByteSlice; - use crate::fetch::{tests::command::v2::capabilities, Command}; + use crate::command::tests::v2::capabilities; + use crate::Command; #[test] fn for_all_features() { @@ -102,7 +99,8 @@ mod v2 { mod ls_refs { mod default_features { - use crate::fetch::{self, tests::command::v2::capabilities, Command}; + use crate::command::tests::v2::capabilities; + use crate::Command; #[test] fn default_as_there_are_no_features() { @@ -111,7 +109,7 @@ mod v2 { git_transport::Protocol::V2, &capabilities("something-else", "does not matter as there are none") ), - &[fetch::agent()] + &[] ); } } @@ -119,7 +117,8 @@ mod v2 { mod validate { use bstr::ByteSlice; - use crate::fetch::{tests::command::v2::capabilities, Command}; + use crate::command::tests::v2::capabilities; + use crate::Command; #[test] fn ref_prefixes_can_always_be_used() { diff --git a/git-protocol/src/fetch/arguments/async_io.rs b/git-protocol/src/fetch/arguments/async_io.rs index 2510e2a343d..222fc45fab0 100644 --- a/git-protocol/src/fetch/arguments/async_io.rs +++ b/git-protocol/src/fetch/arguments/async_io.rs @@ -1,7 +1,8 @@ use futures_lite::io::AsyncWriteExt; use git_transport::{client, client::TransportV2Ext}; -use crate::fetch::{Arguments, Command}; +use crate::fetch::Arguments; +use crate::Command; impl Arguments { /// Send fetch arguments to the server, and indicate this is the end of negotiations only if `add_done_argument` is present. diff --git a/git-protocol/src/fetch/arguments/blocking_io.rs b/git-protocol/src/fetch/arguments/blocking_io.rs index 98ac93f75b2..159f66c7694 100644 --- a/git-protocol/src/fetch/arguments/blocking_io.rs +++ b/git-protocol/src/fetch/arguments/blocking_io.rs @@ -2,7 +2,8 @@ use std::io::Write; use git_transport::{client, client::TransportV2Ext}; -use crate::fetch::{Arguments, Command}; +use crate::fetch::Arguments; +use crate::Command; impl Arguments { /// Send fetch arguments to the server, and indicate this is the end of negotiations only if `add_done_argument` is present. @@ -45,7 +46,9 @@ impl Arguments { } transport.invoke( Command::Fetch.as_str(), - self.features.iter().filter(|(_, v)| v.is_some()).cloned(), + self.features + .iter() + .filter_map(|(k, v)| v.as_ref().map(|v| (*k, Some(v.as_ref())))), Some(std::mem::replace(&mut self.args, retained_state).into_iter()), ) } diff --git a/git-protocol/src/fetch/arguments/mod.rs b/git-protocol/src/fetch/arguments/mod.rs index 64c3ae72389..1fd30336044 100644 --- a/git-protocol/src/fetch/arguments/mod.rs +++ b/git-protocol/src/fetch/arguments/mod.rs @@ -7,7 +7,7 @@ use bstr::{BStr, BString, ByteSlice, ByteVec}; pub struct Arguments { /// The active features/capabilities of the fetch invocation #[cfg(any(feature = "async-client", feature = "blocking-client"))] - features: Vec, + features: Vec, args: Vec, haves: Vec, @@ -136,8 +136,8 @@ impl Arguments { /// Create a new instance to help setting up arguments to send to the server as part of a `fetch` operation /// for which `features` are the available and configured features to use. #[cfg(any(feature = "async-client", feature = "blocking-client"))] - pub fn new(version: git_transport::Protocol, features: Vec) -> Self { - use crate::fetch::Command; + pub fn new(version: git_transport::Protocol, features: Vec) -> Self { + use crate::Command; let has = |name: &str| features.iter().any(|f| f.0 == name); let filter = has("filter"); let shallow = has("shallow"); diff --git a/git-protocol/src/fetch/delegate.rs b/git-protocol/src/fetch/delegate.rs index 1d116beb1d3..54f91f37184 100644 --- a/git-protocol/src/fetch/delegate.rs +++ b/git-protocol/src/fetch/delegate.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::{ io, ops::{Deref, DerefMut}, @@ -6,7 +7,8 @@ use std::{ use bstr::BString; use git_transport::client::Capabilities; -use crate::fetch::{Arguments, Ref, Response}; +use crate::fetch::{Arguments, Response}; +use crate::handshake::Ref; /// Defines what to do next after certain [`Delegate`] operations. #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] @@ -17,18 +19,6 @@ pub enum Action { Cancel, } -/// What to do after [`DelegateBlocking::prepare_ls_refs`]. -#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] -pub enum LsRefsAction { - /// Continue by sending a 'ls-refs' command. - Continue, - /// Skip 'ls-refs' entirely. - /// - /// This is valid if the 'ref-in-want' capability is taken advantage of. The delegate must then send 'want-ref's in - /// [`DelegateBlocking::negotiate`]. - Skip, -} - /// The non-IO protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation, sparing /// the IO parts. /// Async implementations must treat it as blocking and unblock it by evaluating it elsewhere. @@ -48,16 +38,16 @@ pub trait DelegateBlocking { /// Note that some arguments are preset based on typical use, and `features` are preset to maximize options. /// The `server` capabilities can be used to see which additional capabilities the server supports as per the handshake which happened prior. /// - /// If the delegate returns [`LsRefsAction::Skip`], no 'ls-refs` command is sent to the server. + /// If the delegate returns [`ls_refs::Action::Skip`], no 'ls-refs` command is sent to the server. /// /// Note that this is called only if we are using protocol version 2. fn prepare_ls_refs( &mut self, _server: &Capabilities, _arguments: &mut Vec, - _features: &mut Vec<(&str, Option<&str>)>, - ) -> std::io::Result { - Ok(LsRefsAction::Continue) + _features: &mut Vec<(&str, Option>)>, + ) -> std::io::Result { + Ok(ls_refs::Action::Continue) } /// Called before invoking the 'fetch' interaction with `features` pre-filled for typical use @@ -75,7 +65,7 @@ pub trait DelegateBlocking { &mut self, _version: git_transport::Protocol, _server: &Capabilities, - _features: &mut Vec<(&str, Option<&str>)>, + _features: &mut Vec<(&str, Option>)>, _refs: &[Ref], ) -> std::io::Result { Ok(Action::Continue) @@ -125,8 +115,8 @@ impl DelegateBlocking for Box { &mut self, _server: &Capabilities, _arguments: &mut Vec, - _features: &mut Vec<(&str, Option<&str>)>, - ) -> io::Result { + _features: &mut Vec<(&str, Option>)>, + ) -> io::Result { self.deref_mut().prepare_ls_refs(_server, _arguments, _features) } @@ -134,7 +124,7 @@ impl DelegateBlocking for Box { &mut self, _version: git_transport::Protocol, _server: &Capabilities, - _features: &mut Vec<(&str, Option<&str>)>, + _features: &mut Vec<(&str, Option>)>, _refs: &[Ref], ) -> io::Result { self.deref_mut().prepare_fetch(_version, _server, _features, _refs) @@ -159,8 +149,8 @@ impl DelegateBlocking for &mut T { &mut self, _server: &Capabilities, _arguments: &mut Vec, - _features: &mut Vec<(&str, Option<&str>)>, - ) -> io::Result { + _features: &mut Vec<(&str, Option>)>, + ) -> io::Result { self.deref_mut().prepare_ls_refs(_server, _arguments, _features) } @@ -168,7 +158,7 @@ impl DelegateBlocking for &mut T { &mut self, _version: git_transport::Protocol, _server: &Capabilities, - _features: &mut Vec<(&str, Option<&str>)>, + _features: &mut Vec<(&str, Option>)>, _refs: &[Ref], ) -> io::Result { self.deref_mut().prepare_fetch(_version, _server, _features, _refs) @@ -193,7 +183,8 @@ mod blocking_io { use git_features::progress::Progress; - use crate::fetch::{DelegateBlocking, Ref, Response}; + use crate::fetch::{DelegateBlocking, Response}; + use crate::handshake::Ref; /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation. /// @@ -253,7 +244,8 @@ mod async_io { use futures_io::AsyncBufRead; use git_features::progress::Progress; - use crate::fetch::{DelegateBlocking, Ref, Response}; + use crate::fetch::{DelegateBlocking, Response}; + use crate::handshake::Ref; /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation. /// @@ -309,5 +301,6 @@ mod async_io { } } } +use crate::ls_refs; #[cfg(feature = "async-client")] pub use async_io::Delegate; diff --git a/git-protocol/src/fetch/error.rs b/git-protocol/src/fetch/error.rs index 00db99c7d26..349f21dc7e8 100644 --- a/git-protocol/src/fetch/error.rs +++ b/git-protocol/src/fetch/error.rs @@ -2,7 +2,8 @@ use std::io; use git_transport::client; -use crate::fetch::{handshake, refs, response}; +use crate::fetch::response; +use crate::{handshake, ls_refs}; /// The error used in [`fetch()`][crate::fetch()]. #[derive(Debug, thiserror::Error)] @@ -15,7 +16,7 @@ pub enum Error { #[error(transparent)] Transport(#[from] client::Error), #[error(transparent)] - Refs(#[from] refs::Error), + LsRefs(#[from] ls_refs::Error), #[error(transparent)] Response(#[from] response::Error), } diff --git a/git-protocol/src/fetch/handshake.rs b/git-protocol/src/fetch/handshake.rs index 9b06a3c81c5..9e4a2f64f5f 100644 --- a/git-protocol/src/fetch/handshake.rs +++ b/git-protocol/src/fetch/handshake.rs @@ -1,141 +1,25 @@ -use git_transport::client::Capabilities; - -use crate::fetch::Ref; - -/// The result of the [`handshake()`][super::handshake()] function. -#[derive(Default, Debug, Clone)] -#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] -pub struct Outcome { - /// The protocol version the server responded with. It might have downgraded the desired version. - pub server_protocol_version: git_transport::Protocol, - /// The references reported as part of the Protocol::V1 handshake, or `None` otherwise as V2 requires a separate request. - pub refs: Option>, - /// The server capabilities. - pub capabilities: Capabilities, -} - -mod error { - use git_transport::client; - - use crate::{credentials, fetch::refs}; - - /// The error returned by [`handshake()`][crate::fetch::handshake()]. - #[derive(Debug, thiserror::Error)] - #[allow(missing_docs)] - pub enum Error { - #[error(transparent)] - Credentials(#[from] credentials::protocol::Error), - #[error(transparent)] - Transport(#[from] client::Error), - #[error("The transport didn't accept the advertised server version {actual_version:?} and closed the connection client side")] - TransportProtocolPolicyViolation { actual_version: git_transport::Protocol }, - #[error(transparent)] - ParseRefs(#[from] refs::parse::Error), - } -} -pub use error::Error; - -pub(crate) mod function { - use git_features::{progress, progress::Progress}; - use git_transport::{client, client::SetServiceResponse, Service}; - use maybe_async::maybe_async; - - use super::{Error, Outcome}; - use crate::{credentials, fetch::refs}; - - /// Perform a handshake with the server on the other side of `transport`, with `authenticate` being used if authentication - /// turns out to be required. `extra_parameters` are the parameters `(name, optional value)` to add to the handshake, - /// each time it is performed in case authentication is required. - /// `progress` is used to inform about what's currently happening. - #[allow(clippy::result_large_err)] - #[maybe_async] - pub async fn handshake( - mut transport: T, - mut authenticate: AuthFn, - extra_parameters: Vec<(String, Option)>, - progress: &mut impl Progress, - ) -> Result - where - AuthFn: FnMut(credentials::helper::Action) -> credentials::protocol::Result, - T: client::Transport, - { - let (server_protocol_version, refs, capabilities) = { - progress.init(None, progress::steps()); - progress.set_name("handshake"); - progress.step(); - - let extra_parameters: Vec<_> = extra_parameters - .iter() - .map(|(k, v)| (k.as_str(), v.as_ref().map(|s| s.as_str()))) - .collect(); - let supported_versions: Vec<_> = transport.supported_protocol_versions().into(); - - let result = transport.handshake(Service::UploadPack, &extra_parameters).await; - let SetServiceResponse { - actual_protocol, - capabilities, - refs, - } = match result { - Ok(v) => Ok(v), - Err(client::Error::Io { ref err }) if err.kind() == std::io::ErrorKind::PermissionDenied => { - drop(result); // needed to workaround this: https://github.com/rust-lang/rust/issues/76149 - let url = transport.to_url(); - progress.set_name("authentication"); - let credentials::protocol::Outcome { identity, next } = - authenticate(credentials::helper::Action::get_for_url(url))? - .expect("FILL provides an identity or errors"); - transport.set_identity(identity)?; - progress.step(); - progress.set_name("handshake (authenticated)"); - match transport.handshake(Service::UploadPack, &extra_parameters).await { - Ok(v) => { - authenticate(next.store())?; - Ok(v) - } - // Still no permission? Reject the credentials. - Err(client::Error::Io { err }) if err.kind() == std::io::ErrorKind::PermissionDenied => { - authenticate(next.erase())?; - Err(client::Error::Io { err }) - } - // Otherwise, do nothing, as we don't know if it actually got to try the credentials. - // If they were previously stored, they remain. In the worst case, the user has to enter them again - // next time they try. - Err(err) => Err(err), - } - } - Err(err) => Err(err), - }?; - - if !supported_versions.is_empty() && !supported_versions.contains(&actual_protocol) { - return Err(Error::TransportProtocolPolicyViolation { - actual_version: actual_protocol, - }); - } - - let parsed_refs = match refs { - Some(mut refs) => { - assert_eq!( - actual_protocol, - git_transport::Protocol::V1, - "Only V1 auto-responds with refs" - ); - Some( - refs::from_v1_refs_received_as_part_of_handshake_and_capabilities( - &mut refs, - capabilities.iter(), - ) - .await?, - ) - } - None => None, - }; - (actual_protocol, parsed_refs, capabilities) - }; // this scope is needed, see https://github.com/rust-lang/rust/issues/76149 - - Ok(Outcome { - server_protocol_version, - refs, - capabilities, - }) - } +use crate::credentials; +use git_features::progress::Progress; +use git_transport::{client, Service}; + +use crate::handshake::{Error, Outcome}; +use maybe_async::maybe_async; + +/// Perform a handshake with the server on the other side of `transport`, with `authenticate` being used if authentication +/// turns out to be required. `extra_parameters` are the parameters `(name, optional value)` to add to the handshake, +/// each time it is performed in case authentication is required. +/// `progress` is used to inform about what's currently happening. +#[allow(clippy::result_large_err)] +#[maybe_async] +pub async fn upload_pack( + transport: T, + authenticate: AuthFn, + extra_parameters: Vec<(String, Option)>, + progress: &mut impl Progress, +) -> Result +where + AuthFn: FnMut(credentials::helper::Action) -> credentials::protocol::Result, + T: client::Transport, +{ + crate::handshake(transport, Service::UploadPack, authenticate, extra_parameters, progress).await } diff --git a/git-protocol/src/fetch/mod.rs b/git-protocol/src/fetch/mod.rs index 080e05cb390..0828ea733a2 100644 --- a/git-protocol/src/fetch/mod.rs +++ b/git-protocol/src/fetch/mod.rs @@ -1,51 +1,20 @@ mod arguments; pub use arguments::Arguments; -/// -pub mod command; -pub use command::Command; - -/// Returns the name of the agent as key-value pair, commonly used in HTTP headers. -pub fn agent() -> (&'static str, Option<&'static str>) { - ("agent", Some(concat!("git/oxide-", env!("CARGO_PKG_VERSION")))) -} - /// pub mod delegate; #[cfg(any(feature = "async-client", feature = "blocking-client"))] pub use delegate::Delegate; -pub use delegate::{Action, DelegateBlocking, LsRefsAction}; +pub use delegate::{Action, DelegateBlocking}; mod error; pub use error::Error; /// -pub mod refs; -pub use refs::{function::refs, Ref}; -/// pub mod response; pub use response::Response; -/// -pub mod handshake; -pub use handshake::function::handshake; - -/// Send a message to indicate the remote side that there is nothing more to expect from us, indicating a graceful shutdown. -#[maybe_async::maybe_async] -pub async fn indicate_end_of_interaction( - mut transport: impl git_transport::client::Transport, -) -> Result<(), git_transport::client::Error> { - // An empty request marks the (early) end of the interaction. Only relevant in stateful transports though. - if transport.connection_persists_across_multiple_requests() { - transport - .request( - git_transport::client::WriteMode::Binary, - git_transport::client::MessageKind::Flush, - )? - .into_read() - .await?; - } - Ok(()) -} +mod handshake; +pub use handshake::upload_pack as handshake; #[cfg(test)] mod tests; diff --git a/git-protocol/src/fetch/refs/function.rs b/git-protocol/src/fetch/refs/function.rs deleted file mode 100644 index 83899ad153a..00000000000 --- a/git-protocol/src/fetch/refs/function.rs +++ /dev/null @@ -1,68 +0,0 @@ -use bstr::BString; -use git_features::progress::Progress; -use git_transport::{ - client::{Capabilities, Transport, TransportV2Ext}, - Protocol, -}; -use maybe_async::maybe_async; - -use super::Error; -use crate::fetch::{indicate_end_of_interaction, refs::from_v2_refs, Command, LsRefsAction, Ref}; - -/// Invoke an ls-refs command on `transport` (assuming `protocol_version` 2 or panic), which requires a prior handshake that yielded -/// server `capabilities`. `prepare_ls_refs(capabilities, arguments, features)` can be used to alter the _ls-refs_. `progress` is used to provide feedback. -#[maybe_async] -pub async fn refs( - mut transport: impl Transport, - protocol_version: Protocol, - capabilities: &Capabilities, - prepare_ls_refs: impl FnOnce( - &Capabilities, - &mut Vec, - &mut Vec<(&str, Option<&str>)>, - ) -> std::io::Result, - progress: &mut impl Progress, -) -> Result, Error> { - assert_eq!( - protocol_version, - Protocol::V2, - "Only V2 needs a separate request to get specific refs" - ); - - let ls_refs = Command::LsRefs; - let mut ls_features = ls_refs.default_features(protocol_version, capabilities); - let mut ls_args = ls_refs.initial_arguments(&ls_features); - if capabilities - .capability("ls-refs") - .and_then(|cap| cap.supports("unborn")) - .unwrap_or_default() - { - ls_args.push("unborn".into()); - } - let refs = match prepare_ls_refs(capabilities, &mut ls_args, &mut ls_features) { - Ok(LsRefsAction::Skip) => Vec::new(), - Ok(LsRefsAction::Continue) => { - ls_refs.validate_argument_prefixes_or_panic(protocol_version, capabilities, &ls_args, &ls_features); - - progress.step(); - progress.set_name("list refs"); - let mut remote_refs = transport - .invoke( - ls_refs.as_str(), - ls_features.into_iter(), - if ls_args.is_empty() { - None - } else { - Some(ls_args.into_iter()) - }, - ) - .await?; - from_v2_refs(&mut remote_refs).await? - } - Err(err) => { - indicate_end_of_interaction(transport).await?; - return Err(err.into()); - } - }; - Ok(refs) -} diff --git a/git-protocol/src/fetch/response/mod.rs b/git-protocol/src/fetch/response/mod.rs index 0b7011ab112..4534a741815 100644 --- a/git-protocol/src/fetch/response/mod.rs +++ b/git-protocol/src/fetch/response/mod.rs @@ -1,7 +1,7 @@ use bstr::BString; use git_transport::{client, Protocol}; -use crate::fetch::command::Feature; +use crate::command::Feature; /// The error returned in the [response module][crate::fetch::response]. #[derive(Debug, thiserror::Error)] diff --git a/git-protocol/src/fetch/tests/arguments.rs b/git-protocol/src/fetch/tests/arguments.rs index bc81f0d6f4b..60c56a1344b 100644 --- a/git-protocol/src/fetch/tests/arguments.rs +++ b/git-protocol/src/fetch/tests/arguments.rs @@ -18,11 +18,13 @@ struct Transport { #[cfg(feature = "blocking-client")] mod impls { + use bstr::BStr; use git_transport::{ client, client::{Error, MessageKind, RequestWriter, SetServiceResponse, WriteMode}, Protocol, Service, }; + use std::borrow::Cow; use crate::fetch::tests::arguments::Transport; @@ -35,7 +37,7 @@ mod impls { self.inner.request(write_mode, on_into_read) } - fn to_url(&self) -> String { + fn to_url(&self) -> Cow<'_, BStr> { self.inner.to_url() } @@ -69,11 +71,13 @@ mod impls { #[cfg(feature = "async-client")] mod impls { use async_trait::async_trait; + use bstr::BStr; use git_transport::{ client, client::{Error, MessageKind, RequestWriter, SetServiceResponse, WriteMode}, Protocol, Service, }; + use std::borrow::Cow; use crate::fetch::tests::arguments::Transport; impl client::TransportWithoutIO for Transport { @@ -85,7 +89,7 @@ mod impls { self.inner.request(write_mode, on_into_read) } - fn to_url(&self) -> String { + fn to_url(&self) -> Cow<'_, BStr> { self.inner.to_url() } diff --git a/git-protocol/src/fetch/tests/mod.rs b/git-protocol/src/fetch/tests/mod.rs index 2f9f3023669..465ac0dc320 100644 --- a/git-protocol/src/fetch/tests/mod.rs +++ b/git-protocol/src/fetch/tests/mod.rs @@ -1,5 +1,4 @@ #[cfg(any(feature = "async-client", feature = "blocking-client"))] mod arguments; -mod command; #[cfg(any(feature = "blocking-client", feature = "async-client"))] mod refs; diff --git a/git-protocol/src/fetch/tests/refs.rs b/git-protocol/src/fetch/tests/refs.rs index 4f4259df55b..35424af270e 100644 --- a/git-protocol/src/fetch/tests/refs.rs +++ b/git-protocol/src/fetch/tests/refs.rs @@ -1,7 +1,7 @@ use git_testtools::hex_to_id as oid; use git_transport::{client, client::Capabilities}; -use crate::fetch::{refs, refs::shared::InternalRef, Ref}; +use crate::handshake::{refs, refs::shared::InternalRef, Ref}; #[maybe_async::test(feature = "blocking-client", async(feature = "async-client", async_std::test))] async fn extract_references_from_v2_refs() { diff --git a/git-protocol/src/fetch_fn.rs b/git-protocol/src/fetch_fn.rs index e5787199f8b..0976566afae 100644 --- a/git-protocol/src/fetch_fn.rs +++ b/git-protocol/src/fetch_fn.rs @@ -1,10 +1,12 @@ use git_features::progress::Progress; use git_transport::client; use maybe_async::maybe_async; +use std::borrow::Cow; use crate::{ credentials, - fetch::{handshake, indicate_end_of_interaction, Action, Arguments, Command, Delegate, Error, Response}, + fetch::{Action, Arguments, Delegate, Error, Response}, + indicate_end_of_interaction, Command, }; /// A way to indicate how to treat the connection underlying the transport, potentially allowing to reuse it. @@ -41,6 +43,7 @@ impl Default for FetchConnection { /// * `authenticate(operation_to_perform)` is used to receive credentials for the connection and potentially store it /// if the server indicates 'permission denied'. Note that not all transport support authentication or authorization. /// * `progress` is used to emit progress messages. +/// * `name` is the name of the git client to present as `agent`, like `"my-app (v2.0)"`". /// /// _Note_ that depending on the `delegate`, the actual action performed can be `ls-refs`, `clone` or `fetch`. #[allow(clippy::result_large_err)] @@ -51,6 +54,7 @@ pub async fn fetch( authenticate: F, mut progress: P, fetch_mode: FetchConnection, + agent: impl Into, ) -> Result<(), Error> where F: FnMut(credentials::helper::Action) -> credentials::protocol::Result, @@ -59,7 +63,7 @@ where P: Progress, P::SubProgress: 'static, { - let handshake::Outcome { + let crate::handshake::Outcome { server_protocol_version: protocol_version, refs, capabilities, @@ -71,14 +75,18 @@ where ) .await?; + let agent = crate::agent(agent); let refs = match refs { Some(refs) => refs, None => { - crate::fetch::refs( + crate::ls_refs( &mut transport, - protocol_version, &capabilities, - |a, b, c| delegate.prepare_ls_refs(a, b, c), + |a, b, c| { + let res = delegate.prepare_ls_refs(a, b, c); + c.push(("agent", Some(Cow::Owned(agent.clone())))); + res + }, &mut progress, ) .await? @@ -108,6 +116,7 @@ where Response::check_required_features(protocol_version, &fetch_features)?; let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); + fetch_features.push(("agent", Some(Cow::Owned(agent)))); let mut arguments = Arguments::new(protocol_version, fetch_features); let mut previous_response = None::; let mut round = 1; diff --git a/git-protocol/src/handshake/function.rs b/git-protocol/src/handshake/function.rs new file mode 100644 index 00000000000..54548446371 --- /dev/null +++ b/git-protocol/src/handshake/function.rs @@ -0,0 +1,100 @@ +use git_features::{progress, progress::Progress}; +use git_transport::{client, client::SetServiceResponse, Service}; +use maybe_async::maybe_async; + +use super::{Error, Outcome}; +use crate::{credentials, handshake::refs}; + +/// Perform a handshake with the server on the other side of `transport`, with `authenticate` being used if authentication +/// turns out to be required. `extra_parameters` are the parameters `(name, optional value)` to add to the handshake, +/// each time it is performed in case authentication is required. +/// `progress` is used to inform about what's currently happening. +#[allow(clippy::result_large_err)] +#[maybe_async] +pub async fn handshake( + mut transport: T, + service: Service, + mut authenticate: AuthFn, + extra_parameters: Vec<(String, Option)>, + progress: &mut impl Progress, +) -> Result +where + AuthFn: FnMut(credentials::helper::Action) -> credentials::protocol::Result, + T: client::Transport, +{ + let (server_protocol_version, refs, capabilities) = { + progress.init(None, progress::steps()); + progress.set_name("handshake"); + progress.step(); + + let extra_parameters: Vec<_> = extra_parameters + .iter() + .map(|(k, v)| (k.as_str(), v.as_ref().map(|s| s.as_str()))) + .collect(); + let supported_versions: Vec<_> = transport.supported_protocol_versions().into(); + + let result = transport.handshake(service, &extra_parameters).await; + let SetServiceResponse { + actual_protocol, + capabilities, + refs, + } = match result { + Ok(v) => Ok(v), + Err(client::Error::Io { ref err }) if err.kind() == std::io::ErrorKind::PermissionDenied => { + drop(result); // needed to workaround this: https://github.com/rust-lang/rust/issues/76149 + let url = transport.to_url(); + progress.set_name("authentication"); + let credentials::protocol::Outcome { identity, next } = + authenticate(credentials::helper::Action::get_for_url(url.into_owned()))? + .expect("FILL provides an identity or errors"); + transport.set_identity(identity)?; + progress.step(); + progress.set_name("handshake (authenticated)"); + match transport.handshake(service, &extra_parameters).await { + Ok(v) => { + authenticate(next.store())?; + Ok(v) + } + // Still no permission? Reject the credentials. + Err(client::Error::Io { err }) if err.kind() == std::io::ErrorKind::PermissionDenied => { + authenticate(next.erase())?; + Err(client::Error::Io { err }) + } + // Otherwise, do nothing, as we don't know if it actually got to try the credentials. + // If they were previously stored, they remain. In the worst case, the user has to enter them again + // next time they try. + Err(err) => Err(err), + } + } + Err(err) => Err(err), + }?; + + if !supported_versions.is_empty() && !supported_versions.contains(&actual_protocol) { + return Err(Error::TransportProtocolPolicyViolation { + actual_version: actual_protocol, + }); + } + + let parsed_refs = match refs { + Some(mut refs) => { + assert_eq!( + actual_protocol, + git_transport::Protocol::V1, + "Only V1 auto-responds with refs" + ); + Some( + refs::from_v1_refs_received_as_part_of_handshake_and_capabilities(&mut refs, capabilities.iter()) + .await?, + ) + } + None => None, + }; + (actual_protocol, parsed_refs, capabilities) + }; // this scope is needed, see https://github.com/rust-lang/rust/issues/76149 + + Ok(Outcome { + server_protocol_version, + refs, + capabilities, + }) +} diff --git a/git-protocol/src/handshake/mod.rs b/git-protocol/src/handshake/mod.rs new file mode 100644 index 00000000000..e7fbde94490 --- /dev/null +++ b/git-protocol/src/handshake/mod.rs @@ -0,0 +1,83 @@ +use bstr::BString; +use git_transport::client::Capabilities; + +/// A git reference, commonly referred to as 'ref', as returned by a git server before sending a pack. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] +pub enum Ref { + /// A ref pointing to a `tag` object, which in turns points to an `object`, usually a commit + Peeled { + /// The name at which the ref is located, like `refs/tags/1.0`. + full_ref_name: BString, + /// The hash of the tag the ref points to. + tag: git_hash::ObjectId, + /// The hash of the object the `tag` points to. + object: git_hash::ObjectId, + }, + /// A ref pointing to a commit object + Direct { + /// The name at which the ref is located, like `refs/heads/main`. + full_ref_name: BString, + /// The hash of the object the ref points to. + object: git_hash::ObjectId, + }, + /// A symbolic ref pointing to `target` ref, which in turn points to an `object` + Symbolic { + /// The name at which the symbolic ref is located, like `HEAD`. + full_ref_name: BString, + /// The path of the ref the symbolic ref points to, like `refs/heads/main`. + /// + /// See issue [#205] for details + /// + /// [#205]: https://github.com/Byron/gitoxide/issues/205 + target: BString, + /// The hash of the object the `target` ref points to. + object: git_hash::ObjectId, + }, + /// A ref is unborn on the remote and just points to the initial, unborn branch, as is the case in a newly initialized repository + /// or dangling symbolic refs. + Unborn { + /// The name at which the ref is located, typically `HEAD`. + full_ref_name: BString, + /// The path of the ref the symbolic ref points to, like `refs/heads/main`, even though the `target` does not yet exist. + target: BString, + }, +} + +/// The result of the [`handshake()`][super::handshake()] function. +#[derive(Default, Debug, Clone)] +#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The protocol version the server responded with. It might have downgraded the desired version. + pub server_protocol_version: git_transport::Protocol, + /// The references reported as part of the Protocol::V1 handshake, or `None` otherwise as V2 requires a separate request. + pub refs: Option>, + /// The server capabilities. + pub capabilities: Capabilities, +} + +mod error { + use git_transport::client; + + use crate::{credentials, handshake::refs}; + + /// The error returned by [`handshake()`][crate::fetch::handshake()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + Credentials(#[from] credentials::protocol::Error), + #[error(transparent)] + Transport(#[from] client::Error), + #[error("The transport didn't accept the advertised server version {actual_version:?} and closed the connection client side")] + TransportProtocolPolicyViolation { actual_version: git_transport::Protocol }, + #[error(transparent)] + ParseRefs(#[from] refs::parse::Error), + } +} +pub use error::Error; + +pub(crate) mod function; + +/// +pub mod refs; diff --git a/git-protocol/src/fetch/refs/async_io.rs b/git-protocol/src/handshake/refs/async_io.rs similarity index 96% rename from git-protocol/src/fetch/refs/async_io.rs rename to git-protocol/src/handshake/refs/async_io.rs index 3fa1a99ce1b..474b893ef49 100644 --- a/git-protocol/src/fetch/refs/async_io.rs +++ b/git-protocol/src/handshake/refs/async_io.rs @@ -1,7 +1,7 @@ use futures_io::AsyncBufRead; use futures_lite::AsyncBufReadExt; -use crate::fetch::{refs, refs::parse::Error, Ref}; +use crate::handshake::{refs, refs::parse::Error, Ref}; /// Parse refs from the given input line by line. Protocol V2 is required for this to succeed. pub async fn from_v2_refs(in_refs: &mut (dyn AsyncBufRead + Unpin)) -> Result, Error> { diff --git a/git-protocol/src/fetch/refs/blocking_io.rs b/git-protocol/src/handshake/refs/blocking_io.rs similarity index 96% rename from git-protocol/src/fetch/refs/blocking_io.rs rename to git-protocol/src/handshake/refs/blocking_io.rs index bc1e3250087..5d55663bf85 100644 --- a/git-protocol/src/fetch/refs/blocking_io.rs +++ b/git-protocol/src/handshake/refs/blocking_io.rs @@ -1,6 +1,6 @@ use std::io; -use crate::fetch::{refs, refs::parse::Error, Ref}; +use crate::handshake::{refs, refs::parse::Error, Ref}; /// Parse refs from the given input line by line. Protocol V2 is required for this to succeed. pub fn from_v2_refs(in_refs: &mut dyn io::BufRead) -> Result, Error> { diff --git a/git-protocol/src/fetch/refs/mod.rs b/git-protocol/src/handshake/refs/mod.rs similarity index 50% rename from git-protocol/src/fetch/refs/mod.rs rename to git-protocol/src/handshake/refs/mod.rs index 7c8f4c4c9be..2ed210ab5aa 100644 --- a/git-protocol/src/fetch/refs/mod.rs +++ b/git-protocol/src/handshake/refs/mod.rs @@ -1,21 +1,5 @@ -use bstr::{BStr, BString}; - -mod error { - use crate::fetch::refs::parse; - - /// The error returned by [refs()][crate::fetch::refs()]. - #[derive(Debug, thiserror::Error)] - #[allow(missing_docs)] - pub enum Error { - #[error(transparent)] - Io(#[from] std::io::Error), - #[error(transparent)] - Transport(#[from] git_transport::client::Error), - #[error(transparent)] - Parse(#[from] parse::Error), - } -} -pub use error::Error; +use super::Ref; +use bstr::BStr; /// pub mod parse { @@ -44,49 +28,6 @@ pub mod parse { } } -/// A git reference, commonly referred to as 'ref', as returned by a git server before sending a pack. -#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] -#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] -pub enum Ref { - /// A ref pointing to a `tag` object, which in turns points to an `object`, usually a commit - Peeled { - /// The name at which the ref is located, like `refs/tags/1.0`. - full_ref_name: BString, - /// The hash of the tag the ref points to. - tag: git_hash::ObjectId, - /// The hash of the object the `tag` points to. - object: git_hash::ObjectId, - }, - /// A ref pointing to a commit object - Direct { - /// The name at which the ref is located, like `refs/heads/main`. - full_ref_name: BString, - /// The hash of the object the ref points to. - object: git_hash::ObjectId, - }, - /// A symbolic ref pointing to `target` ref, which in turn points to an `object` - Symbolic { - /// The name at which the symbolic ref is located, like `HEAD`. - full_ref_name: BString, - /// The path of the ref the symbolic ref points to, like `refs/heads/main`. - /// - /// See issue [#205] for details - /// - /// [#205]: https://github.com/Byron/gitoxide/issues/205 - target: BString, - /// The hash of the object the `target` ref points to. - object: git_hash::ObjectId, - }, - /// A ref is unborn on the remote and just points to the initial, unborn branch, as is the case in a newly initialized repository - /// or dangling symbolic refs. - Unborn { - /// The name at which the ref is located, typically `HEAD`. - full_ref_name: BString, - /// The path of the ref the symbolic ref points to, like `refs/heads/main`, even though the `target` does not yet exist. - target: BString, - }, -} - impl Ref { /// Provide shared fields referring to the ref itself, namely `(name, target, [peeled])`. /// In case of peeled refs, the tag object itself is returned as it is what the ref directly refers to, and target of the tag is returned @@ -111,8 +52,6 @@ impl Ref { } } -pub(crate) mod function; - #[cfg(any(feature = "blocking-client", feature = "async-client"))] pub(crate) mod shared; diff --git a/git-protocol/src/fetch/refs/shared.rs b/git-protocol/src/handshake/refs/shared.rs similarity index 98% rename from git-protocol/src/fetch/refs/shared.rs rename to git-protocol/src/handshake/refs/shared.rs index 38ae4995000..4ba356465b4 100644 --- a/git-protocol/src/fetch/refs/shared.rs +++ b/git-protocol/src/handshake/refs/shared.rs @@ -1,6 +1,6 @@ use bstr::{BString, ByteSlice}; -use crate::fetch::{refs::parse::Error, Ref}; +use crate::handshake::{refs::parse::Error, Ref}; impl From for Ref { fn from(v: InternalRef) -> Self { @@ -106,7 +106,7 @@ pub(crate) fn from_capabilities<'a>( Ok(out_refs) } -pub(in crate::fetch::refs) fn parse_v1( +pub(in crate::handshake::refs) fn parse_v1( num_initial_out_refs: usize, out_refs: &mut Vec, line: &str, @@ -166,7 +166,7 @@ pub(in crate::fetch::refs) fn parse_v1( Ok(()) } -pub(in crate::fetch::refs) fn parse_v2(line: &str) -> Result { +pub(in crate::handshake::refs) fn parse_v2(line: &str) -> Result { let trimmed = line.trim_end(); let mut tokens = trimmed.splitn(3, ' '); match (tokens.next(), tokens.next()) { diff --git a/git-protocol/src/lib.rs b/git-protocol/src/lib.rs index f857b27e428..43c6260a495 100644 --- a/git-protocol/src/lib.rs +++ b/git-protocol/src/lib.rs @@ -10,6 +10,16 @@ #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![deny(missing_docs, rust_2018_idioms, unsafe_code)] +/// A selector for V2 commands to invoke on the server for purpose of pre-invocation validation. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +pub enum Command { + /// List references. + LsRefs, + /// Fetch a pack. + Fetch, +} +pub mod command; + #[cfg(feature = "async-trait")] pub use async_trait; #[cfg(feature = "futures-io")] @@ -35,3 +45,20 @@ pub use remote_progress::RemoteProgress; #[cfg(all(feature = "blocking-client", feature = "async-client"))] compile_error!("Cannot set both 'blocking-client' and 'async-client' features as they are mutually exclusive"); + +/// +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +pub mod handshake; +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +pub use handshake::function::handshake; + +/// +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +pub mod ls_refs; +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +pub use ls_refs::function::ls_refs; + +mod util; +pub use util::agent; +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +pub use util::indicate_end_of_interaction; diff --git a/git-protocol/src/ls_refs.rs b/git-protocol/src/ls_refs.rs new file mode 100644 index 00000000000..22c5ed1e304 --- /dev/null +++ b/git-protocol/src/ls_refs.rs @@ -0,0 +1,98 @@ +mod error { + use crate::handshake::refs::parse; + + /// The error returned by [ls_refs()][crate::ls_refs()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Transport(#[from] git_transport::client::Error), + #[error(transparent)] + Parse(#[from] parse::Error), + } +} +pub use error::Error; + +/// What to do after preparing ls-refs in [ls_refs()][crate::ls_refs()]. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +pub enum Action { + /// Continue by sending a 'ls-refs' command. + Continue, + /// Skip 'ls-refs' entirely. + /// + /// This is useful if the `ref-in-want` capability is taken advantage of. When fetching, one must must then send + /// `want-ref`s during the negotiation phase. + Skip, +} + +pub(crate) mod function { + use bstr::BString; + use git_features::progress::Progress; + use git_transport::client::{Capabilities, Transport, TransportV2Ext}; + use maybe_async::maybe_async; + use std::borrow::Cow; + + use super::{Action, Error}; + use crate::handshake::{refs::from_v2_refs, Ref}; + use crate::indicate_end_of_interaction; + use crate::Command; + + /// Invoke an ls-refs V2 command on `transport`, which requires a prior handshake that yielded + /// server `capabilities`. `prepare_ls_refs(capabilities, arguments, features)` can be used to alter the _ls-refs_. `progress` is used to provide feedback. + /// Note that `prepare_ls_refs()` is expected to add the `(agent, Some(name))` to the list of `features`. + #[maybe_async] + pub async fn ls_refs( + mut transport: impl Transport, + capabilities: &Capabilities, + prepare_ls_refs: impl FnOnce( + &Capabilities, + &mut Vec, + &mut Vec<(&str, Option>)>, + ) -> std::io::Result, + progress: &mut impl Progress, + ) -> Result, Error> { + let ls_refs = Command::LsRefs; + let mut ls_features = ls_refs.default_features(git_transport::Protocol::V2, capabilities); + let mut ls_args = ls_refs.initial_arguments(&ls_features); + if capabilities + .capability("ls-refs") + .and_then(|cap| cap.supports("unborn")) + .unwrap_or_default() + { + ls_args.push("unborn".into()); + } + let refs = match prepare_ls_refs(capabilities, &mut ls_args, &mut ls_features) { + Ok(Action::Skip) => Vec::new(), + Ok(Action::Continue) => { + ls_refs.validate_argument_prefixes_or_panic( + git_transport::Protocol::V2, + capabilities, + &ls_args, + &ls_features, + ); + + progress.step(); + progress.set_name("list refs"); + let mut remote_refs = transport + .invoke( + ls_refs.as_str(), + ls_features.into_iter(), + if ls_args.is_empty() { + None + } else { + Some(ls_args.into_iter()) + }, + ) + .await?; + from_v2_refs(&mut remote_refs).await? + } + Err(err) => { + indicate_end_of_interaction(transport).await?; + return Err(err.into()); + } + }; + Ok(refs) + } +} diff --git a/git-protocol/src/util.rs b/git-protocol/src/util.rs new file mode 100644 index 00000000000..6a15a4e2f6f --- /dev/null +++ b/git-protocol/src/util.rs @@ -0,0 +1,27 @@ +/// The name of the `git` client in a format suitable for presentation to a `git` server, using `name` as user-defined portion of the value. +pub fn agent(name: impl Into) -> String { + let mut name = name.into(); + if !name.starts_with("git/") { + name.insert_str(0, "git/"); + } + name +} + +/// Send a message to indicate the remote side that there is nothing more to expect from us, indicating a graceful shutdown. +#[cfg(any(feature = "blocking-client", feature = "async-client"))] +#[maybe_async::maybe_async] +pub async fn indicate_end_of_interaction( + mut transport: impl git_transport::client::Transport, +) -> Result<(), git_transport::client::Error> { + // An empty request marks the (early) end of the interaction. Only relevant in stateful transports though. + if transport.connection_persists_across_multiple_requests() { + transport + .request( + git_transport::client::WriteMode::Binary, + git_transport::client::MessageKind::Flush, + )? + .into_read() + .await?; + } + Ok(()) +} diff --git a/git-protocol/tests/fetch/mod.rs b/git-protocol/tests/fetch/mod.rs index 29e5c4ca4cb..f84355dc3f7 100644 --- a/git-protocol/tests/fetch/mod.rs +++ b/git-protocol/tests/fetch/mod.rs @@ -1,7 +1,9 @@ +use std::borrow::Cow; use std::io; use bstr::{BString, ByteSlice}; -use git_protocol::fetch::{self, Action, Arguments, LsRefsAction, Ref, Response}; +use git_protocol::fetch::{self, Action, Arguments, Response}; +use git_protocol::{handshake, ls_refs}; use git_transport::client::Capabilities; use crate::fixture_bytes; @@ -27,8 +29,8 @@ impl fetch::DelegateBlocking for CloneDelegate { &mut self, _version: git_transport::Protocol, _server: &Capabilities, - _features: &mut Vec<(&str, Option<&str>)>, - _refs: &[fetch::Ref], + _features: &mut Vec<(&str, Option>)>, + _refs: &[handshake::Ref], ) -> io::Result { match self.abort_with.take() { Some(err) => Err(err), @@ -37,7 +39,7 @@ impl fetch::DelegateBlocking for CloneDelegate { } fn negotiate( &mut self, - refs: &[Ref], + refs: &[handshake::Ref], arguments: &mut Arguments, _previous_response: Option<&Response>, ) -> io::Result { @@ -60,10 +62,10 @@ pub struct CloneRefInWantDelegate { pack_bytes: usize, /// Refs advertised by `ls-refs` -- should always be empty, as we skip `ls-refs`. - refs: Vec, + refs: Vec, /// Refs advertised as `wanted-ref` -- should always match `want_refs` - wanted_refs: Vec, + wanted_refs: Vec, } impl fetch::DelegateBlocking for CloneRefInWantDelegate { @@ -71,23 +73,28 @@ impl fetch::DelegateBlocking for CloneRefInWantDelegate { &mut self, _server: &Capabilities, _arguments: &mut Vec, - _features: &mut Vec<(&str, Option<&str>)>, - ) -> io::Result { - Ok(LsRefsAction::Skip) + _features: &mut Vec<(&str, Option>)>, + ) -> io::Result { + Ok(ls_refs::Action::Skip) } fn prepare_fetch( &mut self, _version: git_transport::Protocol, _server: &Capabilities, - _features: &mut Vec<(&str, Option<&str>)>, - refs: &[fetch::Ref], + _features: &mut Vec<(&str, Option>)>, + refs: &[handshake::Ref], ) -> io::Result { self.refs = refs.to_owned(); Ok(Action::Continue) } - fn negotiate(&mut self, _refs: &[Ref], arguments: &mut Arguments, _prev: Option<&Response>) -> io::Result { + fn negotiate( + &mut self, + _refs: &[handshake::Ref], + arguments: &mut Arguments, + _prev: Option<&Response>, + ) -> io::Result { for wanted_ref in &self.want_refs { arguments.want_ref(wanted_ref.as_ref()) } @@ -98,7 +105,7 @@ impl fetch::DelegateBlocking for CloneRefInWantDelegate { #[derive(Default)] pub struct LsRemoteDelegate { - refs: Vec, + refs: Vec, abort_with: Option, } @@ -110,26 +117,26 @@ impl fetch::DelegateBlocking for LsRemoteDelegate { &mut self, _server: &Capabilities, _arguments: &mut Vec, - _features: &mut Vec<(&str, Option<&str>)>, - ) -> std::io::Result { + _features: &mut Vec<(&str, Option>)>, + ) -> std::io::Result { match self.abort_with.take() { Some(err) => Err(err), - None => Ok(LsRefsAction::Continue), + None => Ok(ls_refs::Action::Continue), } } fn prepare_fetch( &mut self, _version: git_transport::Protocol, _server: &Capabilities, - _features: &mut Vec<(&str, Option<&str>)>, - refs: &[fetch::Ref], + _features: &mut Vec<(&str, Option>)>, + refs: &[handshake::Ref], ) -> io::Result { self.refs = refs.to_owned(); Ok(fetch::Action::Cancel) } fn negotiate( &mut self, - _refs: &[Ref], + _refs: &[handshake::Ref], _arguments: &mut Arguments, _previous_response: Option<&Response>, ) -> io::Result { @@ -142,10 +149,8 @@ mod blocking_io { use std::io; use git_features::progress::Progress; - use git_protocol::{ - fetch, - fetch::{Ref, Response}, - }; + use git_protocol::handshake::Ref; + use git_protocol::{fetch, fetch::Response, handshake}; use crate::fetch::{CloneDelegate, CloneRefInWantDelegate, LsRemoteDelegate}; @@ -171,7 +176,7 @@ mod blocking_io { response: &Response, ) -> io::Result<()> { for wanted in response.wanted_refs() { - self.wanted_refs.push(fetch::Ref::Direct { + self.wanted_refs.push(handshake::Ref::Direct { full_ref_name: wanted.path.clone(), object: wanted.id, }); @@ -201,10 +206,8 @@ mod async_io { use async_trait::async_trait; use futures_io::AsyncBufRead; use git_features::progress::Progress; - use git_protocol::{ - fetch, - fetch::{Ref, Response}, - }; + use git_protocol::handshake::Ref; + use git_protocol::{fetch, fetch::Response, handshake}; use crate::fetch::{CloneDelegate, CloneRefInWantDelegate, LsRemoteDelegate}; @@ -232,7 +235,7 @@ mod async_io { response: &Response, ) -> io::Result<()> { for wanted in response.wanted_refs() { - self.wanted_refs.push(fetch::Ref::Direct { + self.wanted_refs.push(handshake::Ref::Direct { full_ref_name: wanted.path.clone(), object: wanted.id, }); diff --git a/git-protocol/tests/fetch/v1.rs b/git-protocol/tests/fetch/v1.rs index d689b094dc3..0744e4a5735 100644 --- a/git-protocol/tests/fetch/v1.rs +++ b/git-protocol/tests/fetch/v1.rs @@ -1,6 +1,6 @@ use bstr::ByteSlice; use git_features::progress; -use git_protocol::{fetch, FetchConnection}; +use git_protocol::{handshake, FetchConnection}; use git_transport::Protocol; use crate::fetch::{helper_unused, oid, transport, CloneDelegate, LsRemoteDelegate}; @@ -25,6 +25,7 @@ async fn clone() -> crate::Result { helper_unused, progress::Discard, FetchConnection::TerminateOnSuccessfulCompletion, + "agent", ) .await?; assert_eq!(dlg.pack_bytes, 876, "{}: It be able to read pack bytes", fixture); @@ -48,18 +49,19 @@ async fn ls_remote() -> crate::Result { helper_unused, progress::Discard, FetchConnection::AllowReuse, + "agent", ) .await?; assert_eq!( delegate.refs, vec![ - fetch::Ref::Symbolic { + handshake::Ref::Symbolic { full_ref_name: "HEAD".into(), object: oid("808e50d724f604f69ab93c6da2919c014667bedb"), target: "refs/heads/master".into() }, - fetch::Ref::Direct { + handshake::Ref::Direct { full_ref_name: "refs/heads/master".into(), object: oid("808e50d724f604f69ab93c6da2919c014667bedb") } @@ -89,6 +91,7 @@ async fn ls_remote_handshake_failure_due_to_downgrade() -> crate::Result { helper_unused, progress::Discard, FetchConnection::AllowReuse, + "agent", ) .await { diff --git a/git-protocol/tests/fetch/v2.rs b/git-protocol/tests/fetch/v2.rs index ba3bb1ca820..90a751bf338 100644 --- a/git-protocol/tests/fetch/v2.rs +++ b/git-protocol/tests/fetch/v2.rs @@ -1,6 +1,6 @@ use bstr::ByteSlice; use git_features::progress; -use git_protocol::{fetch, FetchConnection}; +use git_protocol::{fetch, handshake, ls_refs, FetchConnection}; use git_transport::Protocol; use crate::fetch::{helper_unused, oid, transport, CloneDelegate, CloneRefInWantDelegate, LsRemoteDelegate}; @@ -18,12 +18,14 @@ async fn clone_abort_prep() -> crate::Result { Protocol::V2, git_transport::client::git::ConnectMode::Daemon, ); + let agent = "agent"; let err = git_protocol::fetch( &mut transport, &mut dlg, helper_unused, progress::Discard, FetchConnection::TerminateOnSuccessfulCompletion, + "agent", ) .await .expect_err("fetch aborted"); @@ -33,11 +35,11 @@ async fn clone_abort_prep() -> crate::Result { transport.into_inner().1.as_bstr(), format!( "002fgit-upload-pack does/not/matter\0\0version=2\00014command=ls-refs -001bagent={} +0014agent={} 0001000csymrefs 0009peel 00000000", - fetch::agent().1.expect("value set") + git_protocol::agent(agent) ) .as_bytes() .as_bstr() @@ -62,24 +64,26 @@ async fn ls_remote() -> crate::Result { Protocol::V2, git_transport::client::git::ConnectMode::Daemon, ); + let agent = "agent"; git_protocol::fetch( &mut transport, &mut delegate, helper_unused, progress::Discard, FetchConnection::AllowReuse, + "agent", ) .await?; assert_eq!( delegate.refs, vec![ - fetch::Ref::Symbolic { + handshake::Ref::Symbolic { full_ref_name: "HEAD".into(), object: oid("808e50d724f604f69ab93c6da2919c014667bedb"), target: "refs/heads/master".into() }, - fetch::Ref::Direct { + handshake::Ref::Direct { full_ref_name: "refs/heads/master".into(), object: oid("808e50d724f604f69ab93c6da2919c014667bedb") } @@ -89,11 +93,11 @@ async fn ls_remote() -> crate::Result { transport.into_inner().1.as_bstr(), format!( "0044git-upload-pack does/not/matter\0\0version=2\0value-only\0key=value\00014command=ls-refs -001bagent={} +0014agent={} 0001000csymrefs 0009peel 0000", - fetch::agent().1.expect("value set") + git_protocol::agent(agent) ) .as_bytes() .as_bstr(), @@ -121,6 +125,7 @@ async fn ls_remote_abort_in_prep_ls_refs() -> crate::Result { helper_unused, progress::Discard, FetchConnection::AllowReuse, + "agent", ) .await .expect_err("ls-refs preparation is aborted"); @@ -131,7 +136,7 @@ async fn ls_remote_abort_in_prep_ls_refs() -> crate::Result { b"0044git-upload-pack does/not/matter\x00\x00version=2\x00value-only\x00key=value\x000000".as_bstr() ); match err { - fetch::Error::Refs(fetch::refs::Error::Io(err)) => { + fetch::Error::LsRefs(ls_refs::Error::Io(err)) => { assert_eq!(err.kind(), std::io::ErrorKind::Other); assert_eq!(err.get_ref().expect("other error").to_string(), "hello world"); } @@ -154,19 +159,21 @@ async fn ref_in_want() -> crate::Result { git_transport::client::git::ConnectMode::Daemon, ); + let agent = "agent"; git_protocol::fetch( &mut transport, &mut delegate, helper_unused, progress::Discard, FetchConnection::TerminateOnSuccessfulCompletion, + "agent", ) .await?; assert!(delegate.refs.is_empty(), "Should not receive any ref advertisement"); assert_eq!( delegate.wanted_refs, - vec![fetch::Ref::Direct { + vec![handshake::Ref::Direct { full_ref_name: "refs/heads/main".into(), object: oid("9e320b9180e0b5580af68fa3255b7f3d9ecd5af0"), }] @@ -176,14 +183,14 @@ async fn ref_in_want() -> crate::Result { transport.into_inner().1.as_bstr(), format!( "002fgit-upload-pack does/not/matter\0\0version=2\00012command=fetch -001bagent={} +0014agent={} 0001000ethin-pack 0010include-tag 000eofs-delta 001dwant-ref refs/heads/main 0009done 00000000", - fetch::agent().1.expect("value set") + git_protocol::agent(agent) ) .as_bytes() .as_bstr() diff --git a/git-ref/Cargo.toml b/git-ref/Cargo.toml index 6dc93e3d168..8c7ef3e1a5e 100644 --- a/git-ref/Cargo.toml +++ b/git-ref/Cargo.toml @@ -44,6 +44,7 @@ document-features = { version = "0.2.1", optional = true } [dev-dependencies] git-testtools = { path = "../tests/tools" } git-discover = { path = "../git-discover" } +git-worktree = { path = "../git-worktree" } git-odb = { path = "../git-odb" } tempfile = "3.2.0" diff --git a/git-ref/src/store/file/loose/iter.rs b/git-ref/src/store/file/loose/iter.rs index 4c74b637657..33a9b9804d2 100644 --- a/git-ref/src/store/file/loose/iter.rs +++ b/git-ref/src/store/file/loose/iter.rs @@ -9,16 +9,18 @@ use crate::{file::iter::LooseThenPacked, store_impl::file, BString, FullName}; pub(in crate::store_impl::file) struct SortedLoosePaths { pub(crate) base: PathBuf, filename_prefix: Option, - file_walk: DirEntryIter, + file_walk: Option, } impl SortedLoosePaths { pub fn at(path: impl AsRef, base: impl Into, filename_prefix: Option) -> Self { - let file_walk = git_features::fs::walkdir_sorted_new(path).into_iter(); + let path = path.as_ref(); SortedLoosePaths { base: base.into(), filename_prefix, - file_walk, + file_walk: path + .is_dir() + .then(|| git_features::fs::walkdir_sorted_new(path).into_iter()), } } } @@ -27,7 +29,7 @@ impl Iterator for SortedLoosePaths { type Item = std::io::Result<(PathBuf, FullName)>; fn next(&mut self) -> Option { - for entry in self.file_walk.by_ref() { + for entry in self.file_walk.as_mut()?.by_ref() { match entry { Ok(entry) => { if !entry.file_type().is_file() { diff --git a/git-ref/src/store/file/loose/reflog.rs b/git-ref/src/store/file/loose/reflog.rs index 4ca90c7dcbe..23037b3d94c 100644 --- a/git-ref/src/store/file/loose/reflog.rs +++ b/git-ref/src/store/file/loose/reflog.rs @@ -101,7 +101,6 @@ pub mod create_or_update { pub(crate) fn reflog_create_or_append( &self, name: &FullNameRef, - _lock: &git_lock::Marker, previous_oid: Option, new: &oid, committer: git_actor::SignatureRef<'_>, diff --git a/git-ref/src/store/file/loose/reflog/create_or_update/tests.rs b/git-ref/src/store/file/loose/reflog/create_or_update/tests.rs index 410523db504..baff1d98d61 100644 --- a/git-ref/src/store/file/loose/reflog/create_or_update/tests.rs +++ b/git-ref/src/store/file/loose/reflog/create_or_update/tests.rs @@ -1,7 +1,6 @@ use std::{convert::TryInto, path::Path}; use git_actor::{Sign, Signature, Time}; -use git_lock::acquire::Fail; use git_object::bstr::ByteSlice; use git_testtools::hex_to_id; use tempfile::TempDir; @@ -17,16 +16,6 @@ fn empty_store(writemode: WriteReflog) -> Result<(TempDir, file::Store)> { Ok((dir, store)) } -fn reflock(store: &file::Store, full_name: &str) -> Result { - let full_name: &FullNameRef = full_name.try_into()?; - git_lock::Marker::acquire_to_hold_resource( - store.reference_path(full_name), - Fail::Immediately, - Some(store.git_dir.clone()), - ) - .map_err(Into::into) -} - fn reflog_lines(store: &file::Store, name: &str, buf: &mut Vec) -> Result> { store .reflog_iter(name, buf)? @@ -56,7 +45,6 @@ fn missing_reflog_creates_it_even_if_similarly_named_empty_dir_exists_and_append let (_keep, store) = empty_store(*mode)?; let full_name_str = "refs/heads/main"; let full_name: &FullNameRef = full_name_str.try_into()?; - let lock = reflock(&store, full_name_str)?; let new = hex_to_id("28ce6a8b26aa170e1de65536fe8abe1832bd3242"); let committer = Signature { name: "committer".into(), @@ -69,7 +57,6 @@ fn missing_reflog_creates_it_even_if_similarly_named_empty_dir_exists_and_append }; store.reflog_create_or_append( full_name, - &lock, None, &new, committer.to_ref(), @@ -92,7 +79,6 @@ fn missing_reflog_creates_it_even_if_similarly_named_empty_dir_exists_and_append let previous = hex_to_id("0000000000000000000000111111111111111111"); store.reflog_create_or_append( full_name, - &lock, Some(previous), &new, committer.to_ref(), @@ -123,14 +109,12 @@ fn missing_reflog_creates_it_even_if_similarly_named_empty_dir_exists_and_append // create onto existing directory let full_name_str = "refs/heads/other"; let full_name: &FullNameRef = full_name_str.try_into()?; - let lock = reflock(&store, full_name_str)?; let reflog_path = store.reflog_path(full_name_str.try_into().expect("valid")); let directory_in_place_of_reflog = reflog_path.join("empty-a").join("empty-b"); std::fs::create_dir_all(&directory_in_place_of_reflog)?; store.reflog_create_or_append( full_name, - &lock, None, &new, committer.to_ref(), diff --git a/git-ref/src/store/file/packed.rs b/git-ref/src/store/file/packed.rs index 51bb87cbe9b..70dc00d9d55 100644 --- a/git-ref/src/store/file/packed.rs +++ b/git-ref/src/store/file/packed.rs @@ -44,6 +44,12 @@ impl file::Store { pub fn packed_refs_path(&self) -> PathBuf { self.common_dir_resolved().join("packed-refs") } + + pub(crate) fn packed_refs_lock_path(&self) -> PathBuf { + let mut p = self.packed_refs_path(); + p.set_extension("lock"); + p + } } /// diff --git a/git-ref/src/store/file/transaction/commit.rs b/git-ref/src/store/file/transaction/commit.rs index 2db1f5e07d3..914c4fe40b4 100644 --- a/git-ref/src/store/file/transaction/commit.rs +++ b/git-ref/src/store/file/transaction/commit.rs @@ -36,7 +36,7 @@ impl<'s, 'p> Transaction<'s, 'p> { match &change.update.change { // reflog first, then reference Change::Update { log, new, expected } => { - let lock = change.lock.take().expect("each ref is locked"); + let lock = change.lock.take(); let (update_ref, update_reflog) = match log.mode { RefLog::Only => (false, true), RefLog::AndReference => (true, true), @@ -67,7 +67,6 @@ impl<'s, 'p> Transaction<'s, 'p> { if do_update { self.store.reflog_create_or_append( change.update.name.as_ref(), - &lock, previous, new_oid, committer, @@ -80,12 +79,12 @@ impl<'s, 'p> Transaction<'s, 'p> { // Don't do anything else while keeping the lock after potentially updating the reflog. // We delay deletion of the reference and dropping the lock to after the packed-refs were // safely written. - if delete_loose_refs { - change.lock = Some(lock); + if delete_loose_refs && matches!(new, Target::Peeled(_)) { + change.lock = lock; continue; } if update_ref { - if let Err(err) = lock.commit() { + if let Some(Err(err)) = lock.map(|l| l.commit()) { // TODO: when Kind::IsADirectory becomes stable, use that. let err = if err.instance.resource_path().is_dir() { git_tempfile::remove_dir::empty_depth_first(err.instance.resource_path()) @@ -146,12 +145,13 @@ impl<'s, 'p> Transaction<'s, 'p> { let take_lock_and_delete = match &change.update.change { Change::Update { log: LogChange { mode, .. }, + new, .. - } => delete_loose_refs && *mode == RefLog::AndReference, + } => delete_loose_refs && *mode == RefLog::AndReference && matches!(new, Target::Peeled(_)), Change::Delete { log: mode, .. } => *mode == RefLog::AndReference, }; if take_lock_and_delete { - let lock = change.lock.take().expect("lock must still be present in delete mode"); + let lock = change.lock.take(); let reference_path = self.store.reference_path(change.update.name.as_ref()); if let Err(err) = std::fs::remove_file(reference_path) { if err.kind() != std::io::ErrorKind::NotFound { diff --git a/git-ref/src/store/file/transaction/mod.rs b/git-ref/src/store/file/transaction/mod.rs index 5a1c7267bcb..9eefb78700f 100644 --- a/git-ref/src/store/file/transaction/mod.rs +++ b/git-ref/src/store/file/transaction/mod.rs @@ -1,5 +1,6 @@ use git_hash::ObjectId; use git_object::bstr::BString; +use std::fmt::Formatter; use crate::{ store_impl::{file, file::Transaction}, @@ -24,6 +25,7 @@ pub enum PackedRefs<'a> { DeletionsAndNonSymbolicUpdates(Box>), /// Propagate deletions as well as updates to references which are peeled, that is contain an object id. Furthermore delete the /// reference which is originally updated if it exists. If it doesn't, the new value will be written into the packed ref right away. + /// Note that this doesn't affect symbolic references at all, which can't be placed into packed refs. DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(Box>), } @@ -89,6 +91,15 @@ impl<'s, 'p> Transaction<'s, 'p> { } } +impl std::fmt::Debug for Transaction<'_, '_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Transaction") + .field("store", self.store) + .field("edits", &self.updates.as_ref().map(|u| u.len())) + .finish_non_exhaustive() + } +} + /// pub mod prepare; diff --git a/git-ref/src/store/file/transaction/prepare.rs b/git-ref/src/store/file/transaction/prepare.rs index c74c694a3f9..938f6521471 100644 --- a/git-ref/src/store/file/transaction/prepare.rs +++ b/git-ref/src/store/file/transaction/prepare.rs @@ -12,12 +12,16 @@ use crate::{ FullName, FullNameRef, Reference, Target, }; +use crate::{packed::transaction::buffer_into_transaction, transaction::PreviousValue}; + impl<'s, 'p> Transaction<'s, 'p> { fn lock_ref_and_apply_change( store: &file::Store, lock_fail_mode: git_lock::acquire::Fail, packed: Option<&packed::Buffer>, change: &mut Edit, + has_global_lock: bool, + direct_to_packed_refs: bool, ) -> Result<(), Error> { use std::io::Write; assert!( @@ -52,15 +56,21 @@ impl<'s, 'p> Transaction<'s, 'p> { let lock = match &mut change.update.change { Change::Delete { expected, .. } => { let (base, relative_path) = store.reference_path_with_base(change.update.name.as_ref()); - let lock = git_lock::Marker::acquire_to_hold_resource( - base.join(relative_path), - lock_fail_mode, - Some(base.into_owned()), - ) - .map_err(|err| Error::LockAcquire { - source: err, - full_name: "borrowchk won't allow change.name()".into(), - })?; + let lock = if has_global_lock { + None + } else { + git_lock::Marker::acquire_to_hold_resource( + base.join(relative_path.as_ref()), + lock_fail_mode, + Some(base.clone().into_owned()), + ) + .map_err(|err| Error::LockAcquire { + source: err, + full_name: "borrowchk won't allow change.name()".into(), + })? + .into() + }; + let existing_ref = existing_ref?; match (&expected, &existing_ref) { (PreviousValue::MustNotExist, _) => { @@ -98,15 +108,18 @@ impl<'s, 'p> Transaction<'s, 'p> { } Change::Update { expected, new, .. } => { let (base, relative_path) = store.reference_path_with_base(change.update.name.as_ref()); - let mut lock = git_lock::File::acquire_to_update_resource( - base.join(relative_path), - lock_fail_mode, - Some(base.into_owned()), - ) - .map_err(|err| Error::LockAcquire { - source: err, - full_name: "borrowchk won't allow change.name() and this will be corrected by caller".into(), - })?; + let obtain_lock = || { + git_lock::File::acquire_to_update_resource( + base.join(relative_path.as_ref()), + lock_fail_mode, + Some(base.clone().into_owned()), + ) + .map_err(|err| Error::LockAcquire { + source: err, + full_name: "borrowchk won't allow change.name() and this will be corrected by caller".into(), + }) + }; + let mut lock = (!has_global_lock).then(obtain_lock).transpose()?; let existing_ref = existing_ref?; match (&expected, &existing_ref) { @@ -150,19 +163,37 @@ impl<'s, 'p> Transaction<'s, 'p> { } }; - if let Some(existing) = existing_ref { + fn new_would_change_existing(new: &Target, existing: &Target) -> (bool, bool) { + match (new, existing) { + (Target::Peeled(new), Target::Peeled(old)) => (old != new, false), + (Target::Symbolic(new), Target::Symbolic(old)) => (old != new, true), + (Target::Peeled(_), _) => (true, false), + (Target::Symbolic(_), _) => (true, true), + } + } + + let (is_effective, is_symbolic) = if let Some(existing) = existing_ref { + let (effective, is_symbolic) = new_would_change_existing(new, &existing.target); *expected = PreviousValue::MustExistAndMatch(existing.target); + (effective, is_symbolic) + } else { + (true, matches!(new, Target::Symbolic(_))) }; - lock.with_mut(|file| match new { - Target::Peeled(oid) => write!(file, "{}", oid), - Target::Symbolic(name) => write!(file, "ref: {}", name.0), - })?; + if (is_effective && !direct_to_packed_refs) || is_symbolic { + let mut lock = lock.take().map(Ok).unwrap_or_else(obtain_lock)?; - lock.close()? + lock.with_mut(|file| match new { + Target::Peeled(oid) => write!(file, "{}", oid), + Target::Symbolic(name) => write!(file, "ref: {}", name.0), + })?; + Some(lock.close()?) + } else { + None + } } }; - change.lock = Some(lock); + change.lock = lock; Ok(()) } } @@ -213,7 +244,10 @@ impl<'s, 'p> Transaction<'s, 'p> { | PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(_) => Some(0_usize), PackedRefs::DeletionsOnly => None, }; - if maybe_updates_for_packed_refs.is_some() || self.store.packed_refs_path().is_file() { + if maybe_updates_for_packed_refs.is_some() + || self.store.packed_refs_path().is_file() + || self.store.packed_refs_lock_path().is_file() + { let mut edits_for_packed_transaction = Vec::::new(); let mut needs_packed_refs_lookups = false; for edit in updates.iter() { @@ -265,28 +299,29 @@ impl<'s, 'p> Transaction<'s, 'p> { // What follows means that we will only create a transaction if we have to access packed refs for looking // up current ref values, or that we definitely have a transaction if we need to make updates. Otherwise // we may have no transaction at all which isn't required if we had none and would only try making deletions. - let packed_transaction: Option<_> = if maybe_updates_for_packed_refs.unwrap_or(0) > 0 { - // We have to create a packed-ref even if it doesn't exist - self.store - .packed_transaction(packed_refs_lock_fail_mode) - .map_err(|err| match err { - file::packed::transaction::Error::BufferOpen(err) => Error::from(err), - file::packed::transaction::Error::TransactionLock(err) => { - Error::PackedTransactionAcquire(err) - } - })? - .into() - } else { - // A packed transaction is optional - we only have deletions that can't be made if - // no packed-ref file exists anyway - self.store - .assure_packed_refs_uptodate()? - .map(|p| { - buffer_into_transaction(p, packed_refs_lock_fail_mode) - .map_err(Error::PackedTransactionAcquire) - }) - .transpose()? - }; + let packed_transaction: Option<_> = + if maybe_updates_for_packed_refs.unwrap_or(0) > 0 || self.store.packed_refs_lock_path().is_file() { + // We have to create a packed-ref even if it doesn't exist + self.store + .packed_transaction(packed_refs_lock_fail_mode) + .map_err(|err| match err { + file::packed::transaction::Error::BufferOpen(err) => Error::from(err), + file::packed::transaction::Error::TransactionLock(err) => { + Error::PackedTransactionAcquire(err) + } + })? + .into() + } else { + // A packed transaction is optional - we only have deletions that can't be made if + // no packed-ref file exists anyway + self.store + .assure_packed_refs_uptodate()? + .map(|p| { + buffer_into_transaction(p, packed_refs_lock_fail_mode) + .map_err(Error::PackedTransactionAcquire) + }) + .transpose()? + }; if let Some(transaction) = packed_transaction { self.packed_transaction = Some(match &mut self.packed_refs { PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(f) @@ -309,6 +344,11 @@ impl<'s, 'p> Transaction<'s, 'p> { ref_files_lock_fail_mode, self.packed_transaction.as_ref().and_then(|t| t.buffer()), change, + self.packed_transaction.is_some(), + matches!( + self.packed_refs, + PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(_) + ), ) { let err = match err { Error::LockAcquire { @@ -351,6 +391,20 @@ impl<'s, 'p> Transaction<'s, 'p> { self.updates = Some(updates); Ok(self) } + + /// Rollback all intermediate state and return the `RefEdits` as we know them thus far. + /// + /// Note that they have been altered compared to what was initially provided as they have + /// been split and know about their current state on disk. + /// + /// # Note + /// + /// A rollback happens automatically as this instance is dropped as well. + pub fn rollback(self) -> Vec { + self.updates + .map(|updates| updates.into_iter().map(|u| u.update).collect()) + .unwrap_or_default() + } } fn possibly_adjust_name_for_prefixes(name: &FullNameRef) -> Option { @@ -423,5 +477,3 @@ mod error { } pub use error::Error; - -use crate::{packed::transaction::buffer_into_transaction, transaction::PreviousValue}; diff --git a/git-ref/src/store/packed/transaction.rs b/git-ref/src/store/packed/transaction.rs index 91cb8eb78d3..1ec11381f42 100644 --- a/git-ref/src/store/packed/transaction.rs +++ b/git-ref/src/store/packed/transaction.rs @@ -1,3 +1,4 @@ +use std::fmt::Formatter; use std::io::Write; use crate::{ @@ -24,6 +25,15 @@ impl packed::Transaction { } } +impl std::fmt::Debug for packed::Transaction { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("packed::Transaction") + .field("edits", &self.edits.as_ref().map(|e| e.len())) + .field("lock", &self.lock) + .finish_non_exhaustive() + } +} + /// Access impl packed::Transaction { /// Returns our packed buffer diff --git a/git-ref/tests/file/store/iter.rs b/git-ref/tests/file/store/iter.rs index 682cb477716..5223c90b5bd 100644 --- a/git-ref/tests/file/store/iter.rs +++ b/git-ref/tests/file/store/iter.rs @@ -9,6 +9,15 @@ mod with_namespace { use git_object::bstr::{BString, ByteSlice}; use crate::file::store_at; + use crate::file::transaction::prepare_and_commit::empty_store; + + #[test] + fn missing_refs_dir_yields_empty_iteration() -> crate::Result { + let (_dir, store) = empty_store()?; + assert_eq!(store.iter()?.all()?.count(), 0); + assert_eq!(store.loose_iter()?.count(), 0); + Ok(()) + } #[test] fn iteration_can_trivially_use_namespaces_as_prefixes() -> crate::Result { diff --git a/git-ref/tests/file/transaction/mod.rs b/git-ref/tests/file/transaction/mod.rs index d9a2e6b7628..6c93b8af489 100644 --- a/git-ref/tests/file/transaction/mod.rs +++ b/git-ref/tests/file/transaction/mod.rs @@ -2,7 +2,10 @@ pub(crate) mod prepare_and_commit { use git_actor::{Sign, Time}; use git_hash::ObjectId; use git_object::bstr::BString; - use git_ref::file; + use git_ref::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; + use git_ref::{file, Target}; + use git_testtools::hex_to_id; + use std::convert::TryInto; fn reflog_lines(store: &file::Store, name: &str) -> crate::Result> { let mut buf = Vec::new(); @@ -14,7 +17,7 @@ pub(crate) mod prepare_and_commit { Ok(res) } - fn empty_store() -> crate::Result<(tempfile::TempDir, file::Store)> { + pub(crate) fn empty_store() -> crate::Result<(tempfile::TempDir, file::Store)> { let dir = tempfile::TempDir::new().unwrap(); let store = file::Store::at(dir.path(), git_ref::store::WriteReflog::Normal, git_hash::Kind::Sha1); Ok((dir, store)) @@ -41,6 +44,45 @@ pub(crate) mod prepare_and_commit { } } + fn create_at(name: &str) -> RefEdit { + RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: true, + message: "log peeled".into(), + }, + expected: PreviousValue::MustNotExist, + new: Target::Peeled(hex_to_id("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391")), + }, + name: name.try_into().expect("valid"), + deref: false, + } + } + + fn create_symbolic_at(name: &str, symbolic_target: &str) -> RefEdit { + RefEdit { + change: Change::Update { + log: LogChange::default(), + expected: PreviousValue::MustNotExist, + new: Target::Symbolic(symbolic_target.try_into().expect("valid target name")), + }, + name: name.try_into().expect("valid"), + deref: false, + } + } + + fn delete_at(name: &str) -> RefEdit { + RefEdit { + change: Change::Delete { + expected: PreviousValue::Any, + log: RefLog::AndReference, + }, + name: name.try_into().expect("valid name"), + deref: false, + } + } + mod create_or_update; mod delete; diff --git a/git-ref/tests/file/transaction/prepare_and_commit/create_or_update/collisions.rs b/git-ref/tests/file/transaction/prepare_and_commit/create_or_update/collisions.rs new file mode 100644 index 00000000000..6cf94b17884 --- /dev/null +++ b/git-ref/tests/file/transaction/prepare_and_commit/create_or_update/collisions.rs @@ -0,0 +1,224 @@ +use crate::file::transaction::prepare_and_commit::{committer, create_at, create_symbolic_at, delete_at, empty_store}; +use git_lock::acquire::Fail; +use git_ref::file::transaction::PackedRefs; +use git_ref::transaction::{Change, LogChange, PreviousValue, RefEdit}; +use git_ref::Target; +use git_testtools::hex_to_id; +use std::convert::TryInto; + +fn case_sensitive(tmp_dir: &std::path::Path) -> bool { + std::fs::write(tmp_dir.join("config"), "").expect("can create file once"); + !git_worktree::fs::Capabilities::probe(tmp_dir).ignore_case +} + +#[test] +fn conflicting_creation_without_packed_refs() -> crate::Result { + let (dir, store) = empty_store()?; + let res = store.transaction().prepare( + [create_at("refs/a"), create_at("refs/A")], + Fail::Immediately, + Fail::Immediately, + ); + + let case_sensitive = case_sensitive(dir.path()); + match res { + Ok(_) if case_sensitive => {} + Ok(_) if !case_sensitive => panic!("should fail as 'a' and 'A' clash"), + Err(err) if case_sensitive => panic!( + "should work as case sensitivity allows 'a' and 'A' to coexist: {:?}", + err + ), + Err(err) if !case_sensitive => { + assert_eq!(err.to_string(), "A lock could not be obtained for reference \"refs/A\"") + } + _ => unreachable!("actually everything is covered"), + } + Ok(()) +} + +#[test] +fn non_conflicting_creation_without_packed_refs_work() -> crate::Result { + let (_dir, store) = empty_store()?; + let ongoing = store + .transaction() + .prepare([create_at("refs/new")], Fail::Immediately, Fail::Immediately) + .unwrap(); + + let t2 = store.transaction().prepare( + [create_at("refs/non-conflicting")], + Fail::Immediately, + Fail::Immediately, + )?; + + t2.commit(committer().to_ref())?; + ongoing.commit(committer().to_ref())?; + + assert!(store.reflog_exists("refs/new")?); + assert!(store.reflog_exists("refs/non-conflicting")?); + + Ok(()) +} + +#[test] +fn packed_refs_lock_is_mandatory_for_multiple_ongoing_transactions_even_if_one_does_not_need_it() -> crate::Result { + let (_dir, store) = empty_store()?; + let ref_name = "refs/a"; + let _t1 = store + .transaction() + .packed_refs(PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference( + Box::new(|_, _| Ok(Some(git_object::Kind::Commit))), + )) + .prepare([create_at(ref_name)], Fail::Immediately, Fail::Immediately)?; + + let t2res = store + .transaction() + .prepare([delete_at(ref_name)], Fail::Immediately, Fail::Immediately); + assert_eq!(&t2res.unwrap_err().to_string()[..51], "The lock for the packed-ref file could not be obtai", "if packed-refs are about to be created, other transactions always acquire a packed-refs lock as to not miss anything"); + Ok(()) +} + +#[test] +fn conflicting_creation_into_packed_refs() -> crate::Result { + let (_dir, store) = empty_store()?; + store + .transaction() + .packed_refs(PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference( + Box::new(|_, _| Ok(Some(git_object::Kind::Commit))), + )) + .prepare( + [ + create_at("refs/a"), + create_at("refs/A"), + create_symbolic_at("refs/symbolic", "refs/heads/target"), + ], + Fail::Immediately, + Fail::Immediately, + )? + .commit(committer().to_ref())?; + + assert_eq!( + store.cached_packed_buffer()?.expect("created").iter()?.count(), + 2, + "packed-refs can store everything in case-insensitive manner" + ); + assert_eq!( + store.loose_iter()?.count(), + 1, + "symbolic refs can't be packed and stay loose" + ); + assert!(store.reflog_exists("refs/a")?); + assert!(store.reflog_exists("refs/A")?); + assert!(!store.reflog_exists("refs/symbolic")?, "and they can't have reflogs"); + + // The following works because locks aren't actually obtained if there would be no change. + // Otherwise there would be a conflict on case-insensitive filesystems + store + .transaction() + .packed_refs(PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference( + Box::new(|_, _| Ok(Some(git_object::Kind::Commit))), + )) + .prepare( + [ + RefEdit { + change: Change::Update { + log: LogChange::default(), + expected: PreviousValue::Any, + new: Target::Peeled(git_hash::Kind::Sha1.null()), + }, + name: "refs/a".try_into().expect("valid"), + deref: false, + }, + RefEdit { + change: Change::Update { + log: LogChange::default(), + expected: PreviousValue::MustExistAndMatch(Target::Peeled(hex_to_id( + "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + ))), + new: Target::Peeled(git_hash::Kind::Sha1.null()), + }, + name: "refs/A".try_into().expect("valid"), + deref: false, + }, + ], + Fail::Immediately, + Fail::Immediately, + )? + .commit(committer().to_ref())?; + assert_eq!(store.iter()?.all()?.count(), 3); + + { + let _ongoing = store + .transaction() + .prepare([create_at("refs/new")], Fail::Immediately, Fail::Immediately)?; + + let t2res = store.transaction().prepare( + [create_at("refs/non-conflicting")], + Fail::Immediately, + Fail::Immediately, + ); + + assert_eq!( + &t2res.unwrap_err().to_string()[..40], + "The lock for the packed-ref file could n", + "packed-refs files will always be locked if they are present as we have to look up their content" + ); + } + + { + let _ongoing = store + .transaction() + .prepare([delete_at("refs/a")], Fail::Immediately, Fail::Immediately)?; + + let t2res = store + .transaction() + .prepare([delete_at("refs/A")], Fail::Immediately, Fail::Immediately); + + assert_eq!( + &t2res.unwrap_err().to_string()[..40], + "The lock for the packed-ref file could n", + "once again, packed-refs save the day" + ); + } + + // Create a loose ref at a path + assert_eq!(store.loose_iter()?.count(), 1, "a symref"); + store + .transaction() + .prepare( + [RefEdit { + change: Change::Update { + log: LogChange::default(), + expected: PreviousValue::Any, + new: Target::Symbolic("refs/heads/does-not-matter".try_into().expect("valid")), + }, + name: "refs/a".try_into().expect("valid"), + deref: false, + }], + Fail::Immediately, + Fail::Immediately, + )? + .commit(committer().to_ref())?; + assert_eq!( + store.loose_iter()?.count(), + 2, + "we created a loose ref, overlaying the packed one, and have a symbolic one" + ); + + store + .transaction() + .prepare( + [delete_at("refs/a"), delete_at("refs/A"), delete_at("refs/symbolic")], + Fail::Immediately, + Fail::Immediately, + )? + .commit(committer().to_ref())?; + + assert_eq!( + store.iter()?.all()?.count(), + 0, + "we deleted our only two packed refs and one loose ref with the same name" + ); + assert!(!store.reflog_exists("refs/a")?); + assert!(!store.reflog_exists("refs/A")?); + Ok(()) +} diff --git a/git-ref/tests/file/transaction/prepare_and_commit/create_or_update.rs b/git-ref/tests/file/transaction/prepare_and_commit/create_or_update/mod.rs similarity index 92% rename from git-ref/tests/file/transaction/prepare_and_commit/create_or_update.rs rename to git-ref/tests/file/transaction/prepare_and_commit/create_or_update/mod.rs index e6a3ef1d527..c14abf4ac6b 100644 --- a/git-ref/tests/file/transaction/prepare_and_commit/create_or_update.rs +++ b/git-ref/tests/file/transaction/prepare_and_commit/create_or_update/mod.rs @@ -15,11 +15,43 @@ use git_ref::{ }; use git_testtools::hex_to_id; +use crate::file::transaction::prepare_and_commit::{create_at, create_symbolic_at, delete_at}; use crate::file::{ store_with_packed_refs, store_writable, transaction::prepare_and_commit::{committer, empty_store, log_line, reflog_lines}, }; +mod collisions; + +#[test] +fn intermediate_directories_are_removed_on_rollback() -> crate::Result { + for explicit_rollback in [false, true] { + let (dir, store) = empty_store()?; + + let transaction = store.transaction().prepare( + [create_at("refs/heads/a/b/ref"), create_at("refs/heads/a/c/ref")], + Fail::Immediately, + Fail::Immediately, + )?; + + assert!( + dir.path().join("refs/heads/a/b").exists(), + "lock files have been created in their place to avoid concurrent modification" + ); + assert!(dir.path().join("refs/heads/a/c").exists()); + + if explicit_rollback { + transaction.rollback(); + } else { + drop(transaction); + } + + assert!(!dir.path().join("refs/heads").exists()); + assert!(!dir.path().join("refs").exists(), "we go all in right now and also remove the refs directory. 'git' might not do that, but it's not a problem either"); + } + Ok(()) +} + #[test] fn reference_with_equally_named_empty_or_non_empty_directory_already_in_place_can_potentially_recover() -> crate::Result { @@ -202,19 +234,9 @@ fn reference_with_must_not_exist_constraint_cannot_be_created_if_it_exists_alrea let head = store.try_find_loose("HEAD")?.expect("head exists already"); let target = head.target; - let res = store.transaction().prepare( - Some(RefEdit { - change: Change::Update { - log: LogChange::default(), - new: Target::Peeled(git_hash::Kind::Sha1.null()), - expected: PreviousValue::MustNotExist, - }, - name: "HEAD".try_into()?, - deref: false, - }), - Fail::Immediately, - Fail::Immediately, - ); + let res = store + .transaction() + .prepare(Some(create_at("HEAD")), Fail::Immediately, Fail::Immediately); match res { Err(transaction::prepare::Error::MustNotExist { full_name, actual, .. }) => { assert_eq!(full_name, "HEAD"); @@ -229,55 +251,16 @@ fn reference_with_must_not_exist_constraint_cannot_be_created_if_it_exists_alrea fn namespaced_updates_or_deletions_are_transparent_and_not_observable() -> crate::Result { let (_keep, mut store) = empty_store()?; store.namespace = git_ref::namespace::expand("foo")?.into(); + let actual = vec![ + delete_at("refs/for/deletion"), + create_symbolic_at("HEAD", "refs/heads/hello"), + ]; let edits = store .transaction() - .prepare( - vec![ - RefEdit { - change: Change::Delete { - expected: PreviousValue::Any, - log: RefLog::AndReference, - }, - name: "refs/for/deletion".try_into()?, - deref: false, - }, - RefEdit { - change: Change::Update { - log: LogChange::default(), - new: Target::Symbolic("refs/heads/hello".try_into()?), - expected: PreviousValue::MustNotExist, - }, - name: "HEAD".try_into()?, - deref: false, - }, - ], - Fail::Immediately, - Fail::Immediately, - )? + .prepare(actual.clone(), Fail::Immediately, Fail::Immediately)? .commit(committer().to_ref())?; - assert_eq!( - edits, - vec![ - RefEdit { - change: Change::Delete { - expected: PreviousValue::Any, - log: RefLog::AndReference, - }, - name: "refs/for/deletion".try_into()?, - deref: false, - }, - RefEdit { - change: Change::Update { - log: LogChange::default(), - new: Target::Symbolic("refs/heads/hello".try_into()?), - expected: PreviousValue::MustNotExist, - }, - name: "HEAD".try_into()?, - deref: false, - } - ] - ); + assert_eq!(edits, actual); Ok(()) } @@ -386,15 +369,7 @@ fn cancellation_after_preparation_leaves_no_change() -> crate::Result { ); let tx = tx.prepare( - Some(RefEdit { - change: Change::Update { - log: LogChange::default(), - new: Target::Symbolic("refs/heads/main".try_into().unwrap()), - expected: PreviousValue::MustNotExist, - }, - name: "HEAD".try_into()?, - deref: false, - }), + Some(create_symbolic_at("HEAD", "refs/heads/main")), Fail::Immediately, Fail::Immediately, )?; diff --git a/git-repository/src/clone/fetch/mod.rs b/git-repository/src/clone/fetch/mod.rs index 4bad614696d..f5ac611572a 100644 --- a/git-repository/src/clone/fetch/mod.rs +++ b/git-repository/src/clone/fetch/mod.rs @@ -103,6 +103,7 @@ impl PrepareFetch { b }; let outcome = pending_pack + .with_write_packed_refs_only(true) .with_reflog_message(RefLogMessage::Override { message: reflog_message.clone(), }) diff --git a/git-repository/src/clone/fetch/util.rs b/git-repository/src/clone/fetch/util.rs index ae7562c033f..9685c277cf8 100644 --- a/git-repository/src/clone/fetch/util.rs +++ b/git-repository/src/clone/fetch/util.rs @@ -48,7 +48,7 @@ pub fn replace_changed_local_config_file(repo: &mut Repository, mut config: git_ /// if we have to, as it might not have been naturally included in the ref-specs. pub fn update_head( repo: &mut Repository, - remote_refs: &[git_protocol::fetch::Ref], + remote_refs: &[git_protocol::handshake::Ref], reflog_message: &BStr, remote_name: &str, ) -> Result<(), Error> { @@ -58,15 +58,15 @@ pub fn update_head( }; let (head_peeled_id, head_ref) = match remote_refs.iter().find_map(|r| { Some(match r { - git_protocol::fetch::Ref::Symbolic { + git_protocol::handshake::Ref::Symbolic { full_ref_name, target, object, } if full_ref_name == "HEAD" => (Some(object.as_ref()), Some(target)), - git_protocol::fetch::Ref::Direct { full_ref_name, object } if full_ref_name == "HEAD" => { + git_protocol::handshake::Ref::Direct { full_ref_name, object } if full_ref_name == "HEAD" => { (Some(object.as_ref()), None) } - git_protocol::fetch::Ref::Unborn { full_ref_name, target } if full_ref_name == "HEAD" => { + git_protocol::handshake::Ref::Unborn { full_ref_name, target } if full_ref_name == "HEAD" => { (None, Some(target)) } _ => return None, diff --git a/git-repository/src/config/cache/access.rs b/git-repository/src/config/cache/access.rs index db1ddc155a4..b44650742cf 100644 --- a/git-repository/src/config/cache/access.rs +++ b/git-repository/src/config/cache/access.rs @@ -2,8 +2,9 @@ use std::{borrow::Cow, convert::TryInto, path::PathBuf, time::Duration}; use git_lock::acquire::Fail; +use crate::config::cache::util::ApplyLeniencyDefault; use crate::{ - config::{cache::util::check_lenient_default, checkout_options, Cache}, + config::{checkout_options, Cache}, remote, repository::identity, }; @@ -14,36 +15,49 @@ impl Cache { use crate::config::diff::algorithm::Error; self.diff_algorithm .get_or_try_init(|| { - let res = { - let name = self - .resolved - .string("diff", None, "algorithm") - .unwrap_or_else(|| Cow::Borrowed("myers".into())); - if name.eq_ignore_ascii_case(b"myers") || name.eq_ignore_ascii_case(b"default") { - Ok(git_diff::blob::Algorithm::Myers) - } else if name.eq_ignore_ascii_case(b"minimal") { - Ok(git_diff::blob::Algorithm::MyersMinimal) - } else if name.eq_ignore_ascii_case(b"histogram") { + let name = self + .resolved + .string("diff", None, "algorithm") + .unwrap_or_else(|| Cow::Borrowed("myers".into())); + if name.eq_ignore_ascii_case(b"myers") || name.eq_ignore_ascii_case(b"default") { + Ok(git_diff::blob::Algorithm::Myers) + } else if name.eq_ignore_ascii_case(b"minimal") { + Ok(git_diff::blob::Algorithm::MyersMinimal) + } else if name.eq_ignore_ascii_case(b"histogram") { + Ok(git_diff::blob::Algorithm::Histogram) + } else if name.eq_ignore_ascii_case(b"patience") { + if self.lenient_config { Ok(git_diff::blob::Algorithm::Histogram) - } else if name.eq_ignore_ascii_case(b"patience") { - if self.lenient_config { - Ok(git_diff::blob::Algorithm::Histogram) - } else { - Err(Error::Unimplemented { - name: name.into_owned(), - }) - } } else { - Err(Error::Unknown { + Err(Error::Unimplemented { name: name.into_owned(), }) } - }; - check_lenient_default(res, self.lenient_config, || git_diff::blob::Algorithm::Myers) + } else { + Err(Error::Unknown { + name: name.into_owned(), + }) + } + .with_lenient_default(self.lenient_config) }) .copied() } + /// Returns a user agent for use with servers. + #[cfg(any(feature = "async-network-client", feature = "blocking-network-client"))] + pub(crate) fn user_agent_tuple(&self) -> (&'static str, Option>) { + let agent = self + .user_agent + .get_or_init(|| { + self.resolved + .string("gitoxide", None, "userAgent") + .map(|s| s.to_string()) + .unwrap_or_else(|| crate::env::agent().into()) + }) + .to_owned(); + ("agent", Some(git_protocol::agent(agent).into())) + } + pub(crate) fn personas(&self) -> &identity::Personas { self.personas .get_or_init(|| identity::Personas::from_config_and_env(&self.resolved, self.git_prefix)) diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index d1e6899a7e3..41b846db347 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -1,4 +1,5 @@ use super::{interpolate_context, util, Error, StageOne}; +use crate::config::cache::util::ApplyLeniency; use crate::{bstr::BString, config::Cache, repository}; /// Initialization @@ -115,7 +116,7 @@ impl Cache { globals }; - let hex_len = util::check_lenient(util::parse_core_abbrev(&config, object_hash), lenient_config)?; + let hex_len = util::parse_core_abbrev(&config, object_hash).with_leniency(lenient_config)?; use util::config_bool; let reflog = util::query_refupdates(&config, lenient_config)?; @@ -136,6 +137,7 @@ impl Cache { xdg_config_home_env, home_env, lenient_config, + user_agent: Default::default(), personas: Default::default(), url_rewrite: Default::default(), #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] @@ -165,7 +167,7 @@ impl Cache { /// in one that it them makes the default. pub fn reread_values_and_clear_caches(&mut self) -> Result<(), Error> { let config = &self.resolved; - let hex_len = util::check_lenient(util::parse_core_abbrev(config, self.object_hash), self.lenient_config)?; + let hex_len = util::parse_core_abbrev(config, self.object_hash).with_leniency(self.lenient_config)?; use util::config_bool; let ignore_case = config_bool(config, "core.ignoreCase", false, self.lenient_config)?; @@ -177,6 +179,7 @@ impl Cache { self.object_kind_hint = object_kind_hint; self.reflog = reflog; + self.user_agent = Default::default(); self.personas = Default::default(); self.url_rewrite = Default::default(); self.diff_algorithm = Default::default(); diff --git a/git-repository/src/config/cache/mod.rs b/git-repository/src/config/cache/mod.rs index ae42c218836..1904c5ea91e 100644 --- a/git-repository/src/config/cache/mod.rs +++ b/git-repository/src/config/cache/mod.rs @@ -13,5 +13,6 @@ impl std::fmt::Debug for Cache { mod access; -mod util; +pub(crate) mod util; + pub(crate) use util::interpolate_context; diff --git a/git-repository/src/config/cache/util.rs b/git-repository/src/config/cache/util.rs index 9f618601a34..88bcff2f276 100644 --- a/git-repository/src/config/cache/util.rs +++ b/git-repository/src/config/cache/util.rs @@ -28,17 +28,14 @@ pub(crate) fn config_bool( lenient: bool, ) -> Result { let (section, key) = key.split_once('.').expect("valid section.key format"); - match config + config .boolean(section, None, key) .unwrap_or(Ok(default)) .map_err(|err| Error::DecodeBoolean { value: err.input, key: key.into(), - }) { - Ok(v) => Ok(v), - Err(_err) if lenient => Ok(default), - Err(err) => Err(err), - } + }) + .with_lenient_default(lenient) } pub(crate) fn query_refupdates( @@ -64,19 +61,35 @@ pub(crate) fn query_refupdates( } } -pub(crate) fn check_lenient(v: Result, E>, lenient: bool) -> Result, E> { - match v { - Ok(v) => Ok(v), - Err(_) if lenient => Ok(None), - Err(err) => Err(err), +// TODO: Use a specialization here once trait specialization is stabilized. Would be perfect here for `T: Default`. +pub trait ApplyLeniency { + fn with_leniency(self, is_lenient: bool) -> Self; +} + +pub trait ApplyLeniencyDefault { + fn with_lenient_default(self, is_lenient: bool) -> Self; +} + +impl ApplyLeniency for Result, E> { + fn with_leniency(self, is_lenient: bool) -> Self { + match self { + Ok(v) => Ok(v), + Err(_) if is_lenient => Ok(None), + Err(err) => Err(err), + } } } -pub(crate) fn check_lenient_default(v: Result, lenient: bool, default: impl FnOnce() -> T) -> Result { - match v { - Ok(v) => Ok(v), - Err(_) if lenient => Ok(default()), - Err(err) => Err(err), +impl ApplyLeniencyDefault for Result +where + T: Default, +{ + fn with_lenient_default(self, is_lenient: bool) -> Self { + match self { + Ok(v) => Ok(v), + Err(_) if is_lenient => Ok(T::default()), + Err(err) => Err(err), + } } } diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index 84e375c2bff..61164b2fb38 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -107,6 +107,50 @@ pub mod checkout_options { } } +/// +pub mod transport { + use crate::bstr; + + /// The error produced when configuring a transport for a particular protocol. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error( + "Could not interpret configuration key {key:?} as {kind} integer of desired range with value: {actual}" + )] + InvalidInteger { + key: &'static str, + kind: &'static str, + actual: i64, + }, + #[error("Could not interpret configuration key {key:?}")] + ConfigValue { + source: git_config::value::Error, + key: &'static str, + }, + #[error("Could not decode value at key {key:?} as UTF-8 string")] + IllformedUtf8 { + key: &'static str, + source: bstr::FromUtf8Error, + }, + #[error("Invalid URL passed for configuration")] + ParseUrl(#[from] git_url::parse::Error), + #[error("Could obtain configuration for an HTTP url")] + Http(#[from] http::Error), + } + + /// + pub mod http { + /// The error produced when configuring a HTTP transport. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error("TBD")] + TBD, + } + } +} + /// Utility type to keep pre-obtained configuration values, only for those required during initial setup /// and other basic operations that are common enough to warrant a permanent cache. /// @@ -124,6 +168,8 @@ pub(crate) struct Cache { pub use_multi_pack_index: bool, /// The representation of `core.logallrefupdates`, or `None` if the variable wasn't set. pub reflog: Option, + /// The configured user agent for presentation to servers. + pub(crate) user_agent: OnceCell, /// identities for later use, lazy initialization. pub(crate) personas: OnceCell, /// A lazily loaded rewrite list for remote urls diff --git a/git-repository/src/env.rs b/git-repository/src/env.rs index 148e197ef34..08306d79ba2 100644 --- a/git-repository/src/env.rs +++ b/git-repository/src/env.rs @@ -1,7 +1,17 @@ +//! Utilities to handle program arguments and other values of interest. use std::ffi::{OsStr, OsString}; use crate::bstr::{BString, ByteVec}; +/// Returns the name of the agent for identification towards a remote server as statically known when compiling the crate. +/// Suitable for both `git` servers and HTTP servers, and used unless configured otherwise. +/// +/// Note that it's meant to be used in conjunction with [`protocol::agent()`][crate::protocol::agent()] which +/// prepends `git/`. +pub fn agent() -> &'static str { + concat!("oxide-", env!("CARGO_PKG_VERSION")) +} + /// Equivalent to `std::env::args_os()`, but with precomposed unicode on MacOS and other apple platforms. #[cfg(not(target_vendor = "apple"))] pub fn args_os() -> impl Iterator { diff --git a/git-repository/src/id.rs b/git-repository/src/id.rs index d04171078cb..4dec925d9d1 100644 --- a/git-repository/src/id.rs +++ b/git-repository/src/id.rs @@ -158,6 +158,12 @@ mod impls { } } + impl<'repo> std::fmt::Display for Id<'repo> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.inner.fmt(f) + } + } + impl<'repo> AsRef for Id<'repo> { fn as_ref(&self) -> &oid { &self.inner diff --git a/git-repository/src/lib.rs b/git-repository/src/lib.rs index 3a77ddf985e..38eea783eca 100644 --- a/git-repository/src/lib.rs +++ b/git-repository/src/lib.rs @@ -338,7 +338,6 @@ pub mod state { /// pub mod discover; -/// pub mod env; mod kind; diff --git a/git-repository/src/open.rs b/git-repository/src/open.rs index 1661cfd6977..baf4ef3abc1 100644 --- a/git-repository/src/open.rs +++ b/git-repository/src/open.rs @@ -309,6 +309,11 @@ impl ThreadSafeRepository { /// Note that this will read various `GIT_*` environment variables to check for overrides, and is probably most useful when implementing /// custom hooks. // TODO: tests, with hooks, GIT_QUARANTINE for ref-log and transaction control (needs git-sec support to remove write access in git-ref) + // TODO: The following vars should end up as overrides of the respective configuration values (see git-config). + // GIT_HTTP_PROXY_AUTHMETHOD, GIT_PROXY_SSL_CERT, GIT_PROXY_SSL_KEY, GIT_PROXY_SSL_CERT_PASSWORD_PROTECTED. + // GIT_PROXY_SSL_CAINFO, GIT_SSL_VERSION, GIT_SSL_CIPHER_LIST, GIT_HTTP_MAX_REQUESTS, GIT_CURL_FTP_NO_EPSV, + // GIT_HTTP_LOW_SPEED_LIMIT, GIT_HTTP_LOW_SPEED_TIME, GIT_HTTP_USER_AGENT, + // no_proxy, NO_PROXY, http_proxy, HTTPS_PROXY, https_proxy, ALL_PROXY, all_proxy pub fn open_with_environment_overrides( fallback_directory: impl Into, trust_map: git_sec::trust::Mapping, diff --git a/git-repository/src/remote/access.rs b/git-repository/src/remote/access.rs index 7caa1987a1e..534cd505a30 100644 --- a/git-repository/src/remote/access.rs +++ b/git-repository/src/remote/access.rs @@ -5,6 +5,10 @@ use crate::{bstr::BStr, remote, Remote}; /// Access impl<'repo> Remote<'repo> { /// Return the name of this remote or `None` if it wasn't persisted to disk yet. + // TODO: name can also be a URL but we don't see it like this. There is a problem with accessing such names + // too as they would require a BStr, but valid subsection names are strings, so some degeneration must happen + // for access at least. Argh. Probably use `reference::remote::Name` and turn it into `remote::Name` as it's + // actually correct. pub fn name(&self) -> Option<&str> { self.name.as_deref() } diff --git a/git-repository/src/remote/connect.rs b/git-repository/src/remote/connect.rs index 110457d2d35..4333471fcf9 100644 --- a/git-repository/src/remote/connect.rs +++ b/git-repository/src/remote/connect.rs @@ -39,6 +39,7 @@ impl<'repo> Remote<'repo> { Connection { remote: self, authenticate: None, + transport_options: None, transport, progress, } diff --git a/git-repository/src/remote/connection/access.rs b/git-repository/src/remote/connection/access.rs index dbdbc9204ff..a8ec076b3be 100644 --- a/git-repository/src/remote/connection/access.rs +++ b/git-repository/src/remote/connection/access.rs @@ -19,6 +19,18 @@ impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { self.authenticate = Some(Box::new(helper)); self } + + /// Provide configuration to be used before the first handshake is conducted. + /// It's typically created by initializing it with [`Repository::transport_options()`][crate::Repository::transport_options()], which + /// is also the default if this isn't set explicitly. Note that all of the default configuration is created from `git` + /// configuration, which can also be manipulated through overrides to affect the default configuration. + /// + /// Use this method to provide transport configuration with custom backend configuration that is not configurable by other means and + /// custom to the application at hand. + pub fn with_transport_options(mut self, config: Box) -> Self { + self.transport_options = Some(config); + self + } } /// Access @@ -41,7 +53,10 @@ impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { self.remote } - /// Provide a mutable transport to allow configuring it with [`configure()`][git_protocol::transport::client::TransportWithoutIO::configure()] + /// Provide a mutable transport to allow interacting with it according to its actual type. + /// Note that the caller _should not_ call [`configure()`][git_protocol::transport::client::TransportWithoutIO::configure()] + /// as we will call it automatically before performing the handshake. Instead, to bring in custom configuration, + /// call [`with_transport_options()`][Connection::with_transport_options()]. pub fn transport_mut(&mut self) -> &mut T { &mut self.transport } diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 0f78ee87a17..89644ea1c2a 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -11,6 +11,7 @@ use crate::{ }; mod error; +use crate::remote::fetch::WritePackedRefs; pub use error::Error; /// The way reflog messages should be composed whenever a ref is written with recent objects from a remote. @@ -112,6 +113,7 @@ where ref_map, dry_run: DryRun::No, reflog_message: None, + write_packed_refs: WritePackedRefs::Never, }) } } @@ -141,6 +143,7 @@ where ref_map: RefMap, dry_run: DryRun, reflog_message: Option, + write_packed_refs: WritePackedRefs, } /// Builder @@ -156,6 +159,15 @@ where self } + /// If enabled, don't write ref updates to loose refs, but put them exclusively to packed-refs. + /// + /// This improves performances and allows case-sensitive filesystems to deal with ref names that would otherwise + /// collide. + pub fn with_write_packed_refs_only(mut self, enabled: bool) -> Self { + self.write_packed_refs = enabled.then(|| WritePackedRefs::Only).unwrap_or(WritePackedRefs::Never); + self + } + /// Set the reflog message to use when updating refs after fetching a pack. pub fn with_reflog_message(mut self, reflog_message: RefLogMessage) -> Self { self.reflog_message = reflog_message.into(); @@ -175,14 +187,14 @@ where // Right now we block the executor by forcing this communication, but that only // happens if the user didn't actually try to receive a pack, which consumes the // connection in an async context. - git_protocol::futures_lite::future::block_on(git_protocol::fetch::indicate_end_of_interaction( + git_protocol::futures_lite::future::block_on(git_protocol::indicate_end_of_interaction( &mut con.transport, )) .ok(); } #[cfg(not(feature = "async-network-client"))] { - git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); + git_protocol::indicate_end_of_interaction(&mut con.transport).ok(); } } } diff --git a/git-repository/src/remote/connection/fetch/receive_pack.rs b/git-repository/src/remote/connection/fetch/receive_pack.rs index a4d23eb6b80..6c16b1040d9 100644 --- a/git-repository/src/remote/connection/fetch/receive_pack.rs +++ b/git-repository/src/remote/connection/fetch/receive_pack.rs @@ -56,6 +56,11 @@ where /// /// Currently the entire process of resolving a pack is blocking the executor. This can be fixed using the `blocking` crate, but it /// didn't seem worth the tradeoff of having more complex code. + /// + /// ### Configuration + /// + /// - `gitoxide.userAgent` is read to obtain the application user agent for git servers and for HTTP servers as well. + /// #[git_protocol::maybe_async::maybe_async] pub async fn receive(mut self, should_interrupt: &AtomicBool) -> Result { let mut con = self.con.take().expect("receive() can only be called once"); @@ -63,16 +68,20 @@ where let handshake = &self.ref_map.handshake; let protocol_version = handshake.server_protocol_version; - let fetch = git_protocol::fetch::Command::Fetch; - let fetch_features = fetch.default_features(protocol_version, &handshake.capabilities); + let fetch = git_protocol::Command::Fetch; + let progress = &mut con.progress; + let repo = con.remote.repo; + let fetch_features = { + let mut f = fetch.default_features(protocol_version, &handshake.capabilities); + f.push(repo.config.user_agent_tuple()); + f + }; git_protocol::fetch::Response::check_required_features(protocol_version, &fetch_features)?; let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); let mut arguments = git_protocol::fetch::Arguments::new(protocol_version, fetch_features); let mut previous_response = None::; let mut round = 1; - let progress = &mut con.progress; - let repo = con.remote.repo; if self.ref_map.object_hash != repo.object_hash() { return Err(Error::IncompatibleObjectHash { @@ -94,9 +103,7 @@ where previous_response.as_ref(), ) { Ok(_) if arguments.is_empty() => { - git_protocol::fetch::indicate_end_of_interaction(&mut con.transport) - .await - .ok(); + git_protocol::indicate_end_of_interaction(&mut con.transport).await.ok(); return Ok(Outcome { ref_map: std::mem::take(&mut self.ref_map), status: Status::NoChange, @@ -104,9 +111,7 @@ where } Ok(is_done) => is_done, Err(err) => { - git_protocol::fetch::indicate_end_of_interaction(&mut con.transport) - .await - .ok(); + git_protocol::indicate_end_of_interaction(&mut con.transport).await.ok(); return Err(err.into()); } }; @@ -160,9 +165,7 @@ where }; if matches!(protocol_version, git_protocol::transport::Protocol::V2) { - git_protocol::fetch::indicate_end_of_interaction(&mut con.transport) - .await - .ok(); + git_protocol::indicate_end_of_interaction(&mut con.transport).await.ok(); } let update_refs = refs::update( @@ -173,6 +176,7 @@ where &self.ref_map.mappings, con.remote.refspecs(remote::Direction::Fetch), self.dry_run, + self.write_packed_refs, )?; if let Some(bundle) = write_pack_bundle.as_mut() { diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index f132d71a9ff..8e8eeb5d2ce 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -47,6 +47,7 @@ pub(crate) fn update( mappings: &[fetch::Mapping], refspecs: &[git_refspec::RefSpec], dry_run: fetch::DryRun, + write_packed_refs: fetch::WritePackedRefs, ) -> Result { let mut edits = Vec::new(); let mut updates = Vec::new(); @@ -169,7 +170,7 @@ pub(crate) fn update( message: message.compose(reflog_message), }, expected: previous_value, - new: if let Source::Ref(git_protocol::fetch::Ref::Symbolic { target, .. }) = &remote { + new: if let Source::Ref(git_protocol::handshake::Ref::Symbolic { target, .. }) = &remote { match mappings.iter().find_map(|m| { m.remote.as_name().and_then(|name| { (name == target) @@ -211,14 +212,18 @@ pub(crate) fn update( .map_err(crate::reference::edit::Error::from)?; repo.refs .transaction() - .packed_refs(git_ref::file::transaction::PackedRefs::DeletionsAndNonSymbolicUpdates( - Box::new(|oid, buf| { - repo.objects - .try_find(oid, buf) - .map(|obj| obj.map(|obj| obj.kind)) - .map_err(|err| Box::new(err) as Box) - }), - )) + .packed_refs( + match write_packed_refs { + fetch::WritePackedRefs::Only => { + git_ref::file::transaction::PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(Box::new(|oid, buf| { + repo.objects + .try_find(oid, buf) + .map(|obj| obj.map(|obj| obj.kind)) + .map_err(|err| Box::new(err) as Box) + }))}, + fetch::WritePackedRefs::Never => git_ref::file::transaction::PackedRefs::DeletionsOnly + } + ) .prepare(edits, file_lock_fail, packed_refs_lock_fail) .map_err(crate::reference::edit::Error::from)? .commit(repo.committer_or_default()) diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index 9fca4ad619d..3d9e05ef59d 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -129,6 +129,7 @@ mod update { &mapping, &specs, reflog_message.map(|_| fetch::DryRun::Yes).unwrap_or(fetch::DryRun::No), + fetch::WritePackedRefs::Never, ) .unwrap(); @@ -167,7 +168,7 @@ mod update { } #[test] - fn checked_out_branches_in_worktrees_are_rejected_with_additional_infromation() -> Result { + fn checked_out_branches_in_worktrees_are_rejected_with_additional_information() -> Result { let root = git_path::realpath(git_testtools::scripted_fixture_repo_read_only_with_args( "make_fetch_repos.sh", [base_repo_path()], @@ -184,7 +185,14 @@ mod update { ] { let spec = format!("refs/heads/main:refs/heads/{}", branch); let (mappings, specs) = mapping_from_spec(&spec, &repo); - let out = fetch::refs::update(&repo, prefixed("action"), &mappings, &specs, fetch::DryRun::Yes)?; + let out = fetch::refs::update( + &repo, + prefixed("action"), + &mappings, + &specs, + fetch::DryRun::Yes, + fetch::WritePackedRefs::Never, + )?; assert_eq!( out.updates, @@ -206,7 +214,15 @@ mod update { let repo = repo("two-origins"); for source in ["refs/heads/main", "refs/heads/symbolic", "HEAD"] { let (mappings, specs) = mapping_from_spec(&format!("{source}:refs/heads/symbolic"), &repo); - let out = fetch::refs::update(&repo, prefixed("action"), &mappings, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update( + &repo, + prefixed("action"), + &mappings, + &specs, + fetch::DryRun::Yes, + fetch::WritePackedRefs::Never, + ) + .unwrap(); assert_eq!(out.edits.len(), 0); assert_eq!( @@ -225,14 +241,22 @@ mod update { let repo = repo("two-origins"); let (mut mappings, specs) = mapping_from_spec("refs/heads/symbolic:refs/remotes/origin/new", &repo); mappings.push(Mapping { - remote: Source::Ref(git_protocol::fetch::Ref::Direct { + remote: Source::Ref(git_protocol::handshake::Ref::Direct { full_ref_name: "refs/heads/main".try_into().unwrap(), object: hex_to_id("f99771fe6a1b535783af3163eba95a927aae21d5"), }), local: Some("refs/heads/symbolic".into()), spec_index: 0, }); - let out = fetch::refs::update(&repo, prefixed("action"), &mappings, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update( + &repo, + prefixed("action"), + &mappings, + &specs, + fetch::DryRun::Yes, + fetch::WritePackedRefs::Never, + ) + .unwrap(); assert_eq!(out.edits.len(), 1); assert_eq!( @@ -265,7 +289,15 @@ mod update { fn local_direct_refs_are_never_written_with_symbolic_ones_but_see_only_the_destination() { let repo = repo("two-origins"); let (mappings, specs) = mapping_from_spec("refs/heads/symbolic:refs/heads/not-currently-checked-out", &repo); - let out = fetch::refs::update(&repo, prefixed("action"), &mappings, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update( + &repo, + prefixed("action"), + &mappings, + &specs, + fetch::DryRun::Yes, + fetch::WritePackedRefs::Never, + ) + .unwrap(); assert_eq!(out.edits.len(), 1); assert_eq!( @@ -281,7 +313,15 @@ mod update { fn remote_refs_cannot_map_to_local_head() { let repo = repo("two-origins"); let (mappings, specs) = mapping_from_spec("refs/heads/main:HEAD", &repo); - let out = fetch::refs::update(&repo, prefixed("action"), &mappings, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update( + &repo, + prefixed("action"), + &mappings, + &specs, + fetch::DryRun::Yes, + fetch::WritePackedRefs::Never, + ) + .unwrap(); assert_eq!(out.edits.len(), 1); assert_eq!( @@ -314,14 +354,22 @@ mod update { let repo = repo("two-origins"); let (mut mappings, specs) = mapping_from_spec("HEAD:refs/remotes/origin/new-HEAD", &repo); mappings.push(Mapping { - remote: Source::Ref(git_protocol::fetch::Ref::Direct { + remote: Source::Ref(git_protocol::handshake::Ref::Direct { full_ref_name: "refs/heads/main".try_into().unwrap(), object: hex_to_id("f99771fe6a1b535783af3163eba95a927aae21d5"), }), local: Some("refs/remotes/origin/main".into()), spec_index: 0, }); - let out = fetch::refs::update(&repo, prefixed("action"), &mappings, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update( + &repo, + prefixed("action"), + &mappings, + &specs, + fetch::DryRun::Yes, + fetch::WritePackedRefs::Never, + ) + .unwrap(); assert_eq!( out.updates, @@ -365,6 +413,7 @@ mod update { &mappings, &specs, fetch::DryRun::Yes, + fetch::WritePackedRefs::Never, ) .unwrap(); @@ -390,7 +439,15 @@ mod update { fn non_fast_forward_is_rejected_if_dry_run_is_disabled() { let (repo, _tmp) = repo_rw("two-origins"); let (mappings, specs) = mapping_from_spec("refs/remotes/origin/g:refs/heads/not-currently-checked-out", &repo); - let out = fetch::refs::update(&repo, prefixed("action"), &mappings, &specs, fetch::DryRun::No).unwrap(); + let out = fetch::refs::update( + &repo, + prefixed("action"), + &mappings, + &specs, + fetch::DryRun::No, + fetch::WritePackedRefs::Never, + ) + .unwrap(); assert_eq!( out.updates, @@ -402,7 +459,15 @@ mod update { assert_eq!(out.edits.len(), 0); let (mappings, specs) = mapping_from_spec("refs/heads/main:refs/remotes/origin/g", &repo); - let out = fetch::refs::update(&repo, prefixed("prefix"), &mappings, &specs, fetch::DryRun::No).unwrap(); + let out = fetch::refs::update( + &repo, + prefixed("prefix"), + &mappings, + &specs, + fetch::DryRun::No, + fetch::WritePackedRefs::Never, + ) + .unwrap(); assert_eq!( out.updates, @@ -425,7 +490,15 @@ mod update { fn fast_forwards_are_called_out_even_if_force_is_given() { let (repo, _tmp) = repo_rw("two-origins"); let (mappings, specs) = mapping_from_spec("+refs/heads/main:refs/remotes/origin/g", &repo); - let out = fetch::refs::update(&repo, prefixed("prefix"), &mappings, &specs, fetch::DryRun::No).unwrap(); + let out = fetch::refs::update( + &repo, + prefixed("prefix"), + &mappings, + &specs, + fetch::DryRun::No, + fetch::WritePackedRefs::Never, + ) + .unwrap(); assert_eq!( out.updates, @@ -469,17 +542,17 @@ mod update { (mappings, vec![spec.to_owned()]) } - fn into_remote_ref(mut r: git::Reference<'_>) -> git_protocol::fetch::Ref { + fn into_remote_ref(mut r: git::Reference<'_>) -> git_protocol::handshake::Ref { let full_ref_name = r.name().as_bstr().into(); match r.target() { - TargetRef::Peeled(id) => git_protocol::fetch::Ref::Direct { + TargetRef::Peeled(id) => git_protocol::handshake::Ref::Direct { full_ref_name, object: id.into(), }, TargetRef::Symbolic(name) => { let target = name.as_bstr().into(); let id = r.peel_to_id_in_place().unwrap(); - git_protocol::fetch::Ref::Symbolic { + git_protocol::handshake::Ref::Symbolic { full_ref_name, target, object: id.detach(), @@ -488,7 +561,7 @@ mod update { } } - fn remote_ref_to_item(r: &git_protocol::fetch::Ref) -> git_refspec::match_group::Item<'_> { + fn remote_ref_to_item(r: &git_protocol::handshake::Ref) -> git_refspec::match_group::Item<'_> { let (full_ref_name, target, object) = r.unpack(); git_refspec::match_group::Item { full_ref_name, diff --git a/git-repository/src/remote/connection/mod.rs b/git-repository/src/remote/connection/mod.rs index 22ed4d82f66..79058938935 100644 --- a/git-repository/src/remote/connection/mod.rs +++ b/git-repository/src/remote/connection/mod.rs @@ -1,8 +1,8 @@ use crate::Remote; pub(crate) struct HandshakeWithRefs { - outcome: git_protocol::fetch::handshake::Outcome, - refs: Vec, + outcome: git_protocol::handshake::Outcome, + refs: Vec, } /// A function that performs a given credential action, trying to obtain credentials for an operation that needs it. @@ -15,6 +15,7 @@ pub type AuthenticateFn<'a> = Box pub struct Connection<'a, 'repo, T, P> { pub(crate) remote: &'a Remote<'repo>, pub(crate) authenticate: Option>, + pub(crate) transport_options: Option>, pub(crate) transport: T, pub(crate) progress: P, } diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 6a95d449b12..727376b8f21 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -13,12 +13,19 @@ use crate::{ #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { + #[error("Failed to configure the transport before connecting to {url:?}")] + GatherTransportConfig { + url: BString, + source: crate::config::transport::Error, + }, + #[error("Failed to configure the transport layer")] + ConfigureTransport(#[from] Box), #[error(transparent)] - Handshake(#[from] git_protocol::fetch::handshake::Error), + Handshake(#[from] git_protocol::handshake::Error), #[error("The object format {format:?} as used by the remote is unsupported")] UnknownObjectFormat { format: BString }, #[error(transparent)] - ListRefs(#[from] git_protocol::fetch::refs::Error), + ListRefs(#[from] git_protocol::ls_refs::Error), #[error(transparent)] Transport(#[from] git_protocol::transport::client::Error), #[error(transparent)] @@ -65,11 +72,15 @@ where /// /// Due to management of the transport, it's cleanest to only use it for a single interaction. Thus it's consumed along with /// the connection. + /// + /// ### Configuration + /// + /// - `gitoxide.userAgent` is read to obtain the application user agent for git servers and for HTTP servers as well. #[allow(clippy::result_large_err)] #[git_protocol::maybe_async::maybe_async] pub async fn ref_map(mut self, options: Options) -> Result { let res = self.ref_map_inner(options).await; - git_protocol::fetch::indicate_end_of_interaction(&mut self.transport) + git_protocol::indicate_end_of_interaction(&mut self.transport) .await .ok(); res @@ -135,6 +146,7 @@ where extra_parameters: Vec<(String, Option)>, ) -> Result { let mut credentials_storage; + let url = self.transport.to_url(); let authenticate = match self.authenticate.as_mut() { Some(f) => f, None => { @@ -142,14 +154,25 @@ where .remote .url(Direction::Fetch) .map(ToOwned::to_owned) - .unwrap_or_else(|| { - git_url::parse(self.transport.to_url().as_bytes().into()) - .expect("valid URL to be provided by transport") - }); + .unwrap_or_else(|| git_url::parse(url.as_ref()).expect("valid URL to be provided by transport")); credentials_storage = self.configured_credentials(url)?; &mut credentials_storage } }; + + if self.transport_options.is_none() { + self.transport_options = + self.remote + .repo + .transport_options(url.as_ref()) + .map_err(|err| Error::GatherTransportConfig { + source: err, + url: url.into_owned(), + })?; + } + if let Some(config) = self.transport_options.as_ref() { + self.transport.configure(&**config)?; + } let mut outcome = git_protocol::fetch::handshake(&mut self.transport, authenticate, extra_parameters, &mut self.progress) .await?; @@ -157,11 +180,12 @@ where Some(refs) => refs, None => { let specs = &self.remote.fetch_specs; - git_protocol::fetch::refs( + let agent_feature = self.remote.repo.config.user_agent_tuple(); + git_protocol::ls_refs( &mut self.transport, - outcome.server_protocol_version, &outcome.capabilities, - |_capabilities, arguments, _features| { + move |_capabilities, arguments, features| { + features.push(agent_feature); if filter_by_prefix { let mut seen = HashSet::new(); for spec in specs { @@ -176,7 +200,7 @@ where } } } - Ok(git_protocol::fetch::delegate::LsRefsAction::Continue) + Ok(git_protocol::ls_refs::Action::Continue) }, &mut self.progress, ) @@ -191,7 +215,7 @@ where #[allow(clippy::result_large_err)] fn extract_object_format( _repo: &crate::Repository, - outcome: &git_protocol::fetch::handshake::Outcome, + outcome: &git_protocol::handshake::Outcome, ) -> Result { use bstr::ByteSlice; let object_hash = diff --git a/git-repository/src/remote/fetch.rs b/git-repository/src/remote/fetch.rs index fe1ad0d7d1f..d82a70b82e7 100644 --- a/git-repository/src/remote/fetch.rs +++ b/git-repository/src/remote/fetch.rs @@ -10,6 +10,16 @@ pub(crate) enum DryRun { No, } +/// How to deal with refs when cloning or fetching. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] +pub(crate) enum WritePackedRefs { + /// Normal operation, i.e. don't use packed-refs at all for writing. + Never, + /// Put ref updates straight into the `packed-refs` file, without creating loose refs first or dealing with them in any way. + Only, +} + /// Information about the relationship between our refspecs, and remote references with their local counterparts. #[derive(Default, Debug, Clone)] pub struct RefMap { @@ -18,11 +28,11 @@ pub struct RefMap { /// Information about the fixes applied to the `mapping` due to validation and sanitization. pub fixes: Vec, /// All refs advertised by the remote. - pub remote_refs: Vec, + pub remote_refs: Vec, /// Additional information provided by the server as part of the handshake. /// /// Note that the `refs` field is always `None` as the refs are placed in `remote_refs`. - pub handshake: git_protocol::fetch::handshake::Outcome, + pub handshake: git_protocol::handshake::Outcome, /// The kind of hash used for all data sent by the server, if understood by this client implementation. /// /// It was extracted from the `handshake` as advertised by the server. @@ -35,7 +45,7 @@ pub enum Source { /// An object id, as the matched ref-spec was an object id itself. ObjectId(git_hash::ObjectId), /// The remote reference that matched the ref-specs name. - Ref(git_protocol::fetch::Ref), + Ref(git_protocol::handshake::Ref), } impl Source { @@ -54,10 +64,10 @@ impl Source { match self { Source::ObjectId(_) => None, Source::Ref(r) => match r { - git_protocol::fetch::Ref::Unborn { full_ref_name, .. } - | git_protocol::fetch::Ref::Symbolic { full_ref_name, .. } - | git_protocol::fetch::Ref::Direct { full_ref_name, .. } - | git_protocol::fetch::Ref::Peeled { full_ref_name, .. } => Some(full_ref_name.as_ref()), + git_protocol::handshake::Ref::Unborn { full_ref_name, .. } + | git_protocol::handshake::Ref::Symbolic { full_ref_name, .. } + | git_protocol::handshake::Ref::Direct { full_ref_name, .. } + | git_protocol::handshake::Ref::Peeled { full_ref_name, .. } => Some(full_ref_name.as_ref()), }, } } diff --git a/git-repository/src/repository/config.rs b/git-repository/src/repository/config/mod.rs similarity index 98% rename from git-repository/src/repository/config.rs rename to git-repository/src/repository/config/mod.rs index c0565f74073..8ce0df414bc 100644 --- a/git-repository/src/repository/config.rs +++ b/git-repository/src/repository/config/mod.rs @@ -33,6 +33,9 @@ impl crate::Repository { } } +#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] +mod transport; + mod remote { use std::{borrow::Cow, collections::BTreeSet}; diff --git a/git-repository/src/repository/config/transport.rs b/git-repository/src/repository/config/transport.rs new file mode 100644 index 00000000000..f6c29bb7370 --- /dev/null +++ b/git-repository/src/repository/config/transport.rs @@ -0,0 +1,161 @@ +use crate::bstr::BStr; +use std::any::Any; + +impl crate::Repository { + /// Produce configuration suitable for `url`, as differentiated by its protocol/scheme, to be passed to a transport instance via + /// [configure()][git_transport::client::TransportWithoutIO::configure()] (via `&**config` to pass the contained `Any` and not the `Box`). + /// `None` is returned if there is no known configuration. + /// + /// Note that the caller may cast the instance themselves to modify it before passing it on. + pub fn transport_options<'a>( + &self, + url: impl Into<&'a BStr>, + ) -> Result>, crate::config::transport::Error> { + let url = git_url::parse(url.into())?; + use git_url::Scheme::*; + + match &url.scheme { + Http | Https => { + #[cfg(not(feature = "blocking-http-transport"))] + { + Ok(None) + } + #[cfg(feature = "blocking-http-transport")] + { + use crate::bstr::ByteVec; + use crate::config::cache::util::{ApplyLeniency, ApplyLeniencyDefault}; + use git_transport::client::http; + use std::borrow::Cow; + use std::convert::{TryFrom, TryInto}; + + fn try_cow_to_string( + v: Cow<'_, BStr>, + lenient: bool, + key: &'static str, + ) -> Result, crate::config::transport::Error> { + Vec::from(v.into_owned()) + .into_string() + .map(Some) + .map_err(|err| crate::config::transport::Error::IllformedUtf8 { source: err, key }) + .with_leniency(lenient) + } + + fn integer( + config: &git_config::File<'static>, + lenient: bool, + key: &'static str, + kind: &'static str, + filter: fn(&git_config::file::Metadata) -> bool, + default: T, + ) -> Result + where + T: TryFrom, + { + Ok(integer_opt(config, lenient, key, kind, filter)?.unwrap_or(default)) + } + fn integer_opt( + config: &git_config::File<'static>, + lenient: bool, + key: &'static str, + kind: &'static str, + mut filter: fn(&git_config::file::Metadata) -> bool, + ) -> Result, crate::config::transport::Error> + where + T: TryFrom, + { + let git_config::parse::Key { + section_name, + subsection_name, + value_name, + } = git_config::parse::key(key).expect("valid key statically known"); + config + .integer_filter(section_name, subsection_name, value_name, &mut filter) + .transpose() + .map_err(|err| crate::config::transport::Error::ConfigValue { source: err, key }) + .with_leniency(lenient)? + .map(|integer| { + integer + .try_into() + .map_err(|_| crate::config::transport::Error::InvalidInteger { + actual: integer, + key, + kind, + }) + }) + .transpose() + .with_leniency(lenient) + } + let mut opts = http::Options::default(); + let config = &self.config.resolved; + let mut trusted_only = self.filter_config_section(); + let lenient = self.config.lenient_config; + opts.extra_headers = { + let mut headers = Vec::new(); + for header in config + .strings_filter("http", None, "extraHeader", &mut trusted_only) + .unwrap_or_default() + .into_iter() + .map(|v| try_cow_to_string(v, lenient, "http.extraHeader")) + { + let header = header?; + if let Some(header) = header { + headers.push(header); + } + } + if let Some(empty_pos) = headers.iter().rev().position(|h| h.is_empty()) { + headers.drain(..headers.len() - empty_pos); + } + headers + }; + + if let Some(follow_redirects) = + config.string_filter("http", None, "followRedirects", &mut trusted_only) + { + opts.follow_redirects = if follow_redirects.as_ref() == "initial" { + http::options::FollowRedirects::Initial + } else if git_config::Boolean::try_from(follow_redirects) + .map_err(|err| crate::config::transport::Error::ConfigValue { + source: err, + key: "http.followRedirects", + }) + .with_lenient_default(lenient)? + .0 + { + http::options::FollowRedirects::All + } else { + http::options::FollowRedirects::None + }; + } + + opts.low_speed_time_seconds = + integer(config, lenient, "http.lowSpeedTime", "u64", trusted_only, 0)?; + opts.low_speed_limit_bytes_per_second = + integer(config, lenient, "http.lowSpeedLimit", "u32", trusted_only, 0)?; + opts.proxy = config + .string_filter("http", None, "proxy", &mut trusted_only) + .and_then(|v| try_cow_to_string(v, lenient, "http.proxy").transpose()) + .transpose()? + .map(|mut proxy| { + if !proxy.trim().is_empty() && !proxy.contains("://") { + proxy.insert_str(0, "http://"); + proxy + } else { + proxy + } + }); + opts.connect_timeout = + integer_opt(config, lenient, "gitoxide.http.connectTimeout", "u64", trusted_only)? + .map(std::time::Duration::from_millis); + opts.user_agent = config + .string_filter("http", None, "userAgent", &mut trusted_only) + .and_then(|v| try_cow_to_string(v, lenient, "http.userAgent").transpose()) + .transpose()? + .or_else(|| Some(crate::env::agent().into())); + + Ok(Some(Box::new(opts))) + } + } + File | Git | Ssh | Ext(_) => Ok(None), + } + } +} diff --git a/git-repository/tests/clone/mod.rs b/git-repository/tests/clone/mod.rs index 2fd49a9882b..cb679219404 100644 --- a/git-repository/tests/clone/mod.rs +++ b/git-repository/tests/clone/mod.rs @@ -63,6 +63,21 @@ mod blocking_io { ); assert_eq!(out.ref_map.mappings.len(), 14); + let packed_refs = repo + .refs + .cached_packed_buffer()? + .expect("packed refs should be present"); + assert_eq!( + packed_refs.iter()?.count(), + 14, + "all non-symbolic refs should be stored" + ); + assert_eq!( + repo.refs.loose_iter()?.count(), + 2, + "HEAD and an actual symbolic ref we received" + ); + match out.status { git_repository::remote::fetch::Status::Change { update_refs, .. } => { for edit in &update_refs.edits { @@ -78,7 +93,9 @@ mod blocking_io { assert!(repo.objects.contains(id), "part of the fetched pack"); } } - let r = repo.find_reference(edit.name.as_ref()).expect("created"); + let r = repo + .find_reference(edit.name.as_ref()) + .unwrap_or_else(|_| panic!("didn't find created reference: {:?}", edit)); if r.name().category().expect("known") != git_ref::Category::Tag { assert!(r .name() @@ -107,16 +124,6 @@ mod blocking_io { "it points to the local tracking branch of what the remote actually points to" ); - let packed_refs = repo - .refs - .cached_packed_buffer()? - .expect("packed refs should be present"); - assert_eq!( - packed_refs.iter()?.count(), - 14, - "all non-symbolic refs should be stored" - ); - let head = repo.head()?; { let mut logs = head.log_iter(); diff --git a/git-repository/tests/fixtures/generated-archives/make_config_repos.tar.xz b/git-repository/tests/fixtures/generated-archives/make_config_repos.tar.xz new file mode 100644 index 00000000000..3d5422fd8e0 --- /dev/null +++ b/git-repository/tests/fixtures/generated-archives/make_config_repos.tar.xz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c75c9f7d9f687dc688ba9d86fed546d98da793ee84d43a52d002d6ba2d79d082 +size 9940 diff --git a/git-repository/tests/fixtures/make_config_repos.sh b/git-repository/tests/fixtures/make_config_repos.sh new file mode 100644 index 00000000000..a5b40e88598 --- /dev/null +++ b/git-repository/tests/fixtures/make_config_repos.sh @@ -0,0 +1,28 @@ +set -eu -o pipefail + +git init http-config +(cd http-config + git config http.extraHeader "ExtraHeader: value1" + git config --add http.extraHeader "" + git config --add http.extraHeader "ExtraHeader: value2" + git config --add http.extraHeader "ExtraHeader: value3" + git config http.followRedirects initial + git config http.lowSpeedLimit 5k + git config http.lowSpeedTime 10 + git config http.postBuffer 8k + git config http.proxy http://localhost:9090 + git config http.proxyAuthMethod anyauth + git config http.userAgent agentJustForHttp + git config gitoxide.http.connectTimeout 60k +) + +git init http-proxy-empty +(cd http-proxy-empty + git config http.proxy localhost:9090 + git config --add http.proxy "" # a value override disabling it later +) + +git init http-proxy-auto-prefix +(cd http-proxy-auto-prefix + git config http.proxy localhost:9090 # http:// is prefixed automatically +) diff --git a/git-repository/tests/fixtures/make_fetch_repos.sh b/git-repository/tests/fixtures/make_fetch_repos.sh index 65807432aa3..9bb598ea918 100644 --- a/git-repository/tests/fixtures/make_fetch_repos.sh +++ b/git-repository/tests/fixtures/make_fetch_repos.sh @@ -1,5 +1,6 @@ set -eu -o pipefail +# IMPORTANT: keep this repo small as it's used for writes, hence will be executed for each writer! git clone --bare "${1:?First argument is the complex base repo from make_remote_repos.sh/base}" base git clone --shared base clone-as-base-with-changes @@ -19,8 +20,7 @@ git clone --shared base two-origins ) git clone --shared base worktree-root -( - cd worktree-root +(cd worktree-root git worktree add ../wt-a git worktree add ../prev/wt-a-nested diff --git a/git-repository/tests/id/mod.rs b/git-repository/tests/id/mod.rs index eb555b8f1ee..e38888cbdc1 100644 --- a/git-repository/tests/id/mod.rs +++ b/git-repository/tests/id/mod.rs @@ -44,6 +44,17 @@ fn prefix() -> crate::Result { Ok(()) } +#[test] +fn display_and_debug() -> crate::Result { + let repo = crate::basic_repo()?; + let id = repo.head_id()?; + assert_eq!( + format!("{} {:?}", id, id), + "3189cd3cb0af8586c39a838aa3e54fd72a872a41 Sha1(3189cd3cb0af8586c39a838aa3e54fd72a872a41)" + ); + Ok(()) +} + mod ancestors { use git_traverse::commit; diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 82581b8d6bd..dd0ae2fb528 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -1,36 +1,43 @@ -use crate::remote::base_repo_path; -use git_repository as git; - -pub(crate) fn repo_path(name: &str) -> std::path::PathBuf { - let dir = - git_testtools::scripted_fixture_repo_read_only_with_args("make_fetch_repos.sh", [base_repo_path()]).unwrap(); - dir.join(name) -} - -pub(crate) fn repo_rw(name: &str) -> (git::Repository, git_testtools::tempfile::TempDir) { - let dir = git_testtools::scripted_fixture_repo_writable_with_args( - "make_fetch_repos.sh", - [base_repo_path()], - git_testtools::Creation::ExecuteScript, - ) - .unwrap(); - let repo = git::open_opts(dir.path().join(name), git::open::Options::isolated()).unwrap(); - (repo, dir) -} - #[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] mod blocking_and_async_io { use git_repository as git; use git_repository::remote::Direction::Fetch; use std::sync::atomic::AtomicBool; - use super::{repo_path, repo_rw}; use crate::remote::{into_daemon_remote_if_async, spawn_git_daemon_if_async}; use git_features::progress; use git_protocol::maybe_async; use git_repository::remote::fetch; use git_testtools::hex_to_id; + pub(crate) fn base_repo_path() -> String { + git::path::realpath( + git_testtools::scripted_fixture_repo_read_only("make_remote_repos.sh") + .unwrap() + .join("base"), + ) + .unwrap() + .to_string_lossy() + .into_owned() + } + + pub(crate) fn repo_path(name: &str) -> std::path::PathBuf { + let dir = git_testtools::scripted_fixture_repo_read_only_with_args("make_fetch_repos.sh", [base_repo_path()]) + .unwrap(); + dir.join(name) + } + + pub(crate) fn repo_rw(name: &str) -> (git::Repository, git_testtools::tempfile::TempDir) { + let dir = git_testtools::scripted_fixture_repo_writable_with_args( + "make_fetch_repos.sh", + [base_repo_path()], + git_testtools::Creation::ExecuteScript, + ) + .unwrap(); + let repo = git::open_opts(dir.path().join(name), git::open::Options::isolated()).unwrap(); + (repo, dir) + } + #[maybe_async::test( feature = "blocking-network-client", async(feature = "async-network-client-async-std", async_std::test) diff --git a/git-repository/tests/remote/mod.rs b/git-repository/tests/remote/mod.rs index a9d9ce48c4e..c9584a77c61 100644 --- a/git-repository/tests/remote/mod.rs +++ b/git-repository/tests/remote/mod.rs @@ -3,17 +3,6 @@ use std::{borrow::Cow, path::PathBuf}; use git_repository as git; use git_testtools::scripted_fixture_repo_read_only; -pub(crate) fn base_repo_path() -> String { - git::path::realpath( - git_testtools::scripted_fixture_repo_read_only("make_remote_repos.sh") - .unwrap() - .join("base"), - ) - .unwrap() - .to_string_lossy() - .into_owned() -} - pub(crate) fn repo_path(name: &str) -> PathBuf { let dir = scripted_fixture_repo_read_only("make_remote_repos.sh").unwrap(); dir.join(name) @@ -80,7 +69,7 @@ pub(crate) fn cow_str(s: &str) -> Cow { } mod connect; -mod fetch; +pub(crate) mod fetch; mod ref_map; mod save; mod name { diff --git a/git-repository/tests/repository/config/mod.rs b/git-repository/tests/repository/config/mod.rs index 7e169fa56b3..5b82895d3af 100644 --- a/git-repository/tests/repository/config/mod.rs +++ b/git-repository/tests/repository/config/mod.rs @@ -1,3 +1,5 @@ mod config_snapshot; mod identity; mod remote; +#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] +mod transport_options; diff --git a/git-repository/tests/repository/config/transport_options.rs b/git-repository/tests/repository/config/transport_options.rs new file mode 100644 index 00000000000..c94732b2ace --- /dev/null +++ b/git-repository/tests/repository/config/transport_options.rs @@ -0,0 +1,79 @@ +#[cfg(feature = "blocking-http-transport")] +mod http { + use git_repository as git; + + pub(crate) fn repo(name: &str) -> git::Repository { + let dir = git_testtools::scripted_fixture_repo_read_only("make_config_repos.sh").unwrap(); + git::open_opts(dir.join(name), git::open::Options::isolated()).unwrap() + } + + fn http_options(repo: &git::Repository) -> git_transport::client::http::Options { + let opts = repo + .transport_options("https://example.com/does/not/matter") + .expect("valid configuration") + .expect("configuration available for http"); + opts.downcast_ref::() + .expect("http options have been created") + .to_owned() + } + + #[test] + fn simple_configuration() { + let repo = repo("http-config"); + let git_transport::client::http::Options { + extra_headers, + follow_redirects, + low_speed_limit_bytes_per_second, + low_speed_time_seconds, + proxy, + proxy_auth_method, + user_agent, + connect_timeout, + backend, + } = http_options(&repo); + assert_eq!( + extra_headers, + &["ExtraHeader: value2", "ExtraHeader: value3"], + "it respects empty values to clear prior values" + ); + assert_eq!( + follow_redirects, + git_transport::client::http::options::FollowRedirects::Initial + ); + assert_eq!(low_speed_limit_bytes_per_second, 5120); + assert_eq!(low_speed_time_seconds, 10); + assert_eq!(proxy.as_deref(), Some("http://localhost:9090"),); + assert_eq!( + proxy_auth_method.as_ref(), + // Some(&git_transport::client::http::options::ProxyAuthMethod::AnyAuth) + None, + "TODO: implement auth" + ); + assert_eq!(user_agent.as_deref(), Some("agentJustForHttp")); + assert_eq!(connect_timeout, Some(std::time::Duration::from_millis(60 * 1024))); + assert!( + backend.is_none(), + "backed is never set as it's backend specific, rather custom options typically" + ) + } + + #[test] + fn empty_proxy_string_turns_it_off() { + let repo = repo("http-proxy-empty"); + + let opts = http_options(&repo); + assert_eq!( + opts.proxy.as_deref(), + Some(""), + "empty strings indicate that the proxy is to be unset by the transport" + ); + } + + #[test] + fn proxy_without_protocol_is_defaulted_to_http() { + let repo = repo("http-proxy-auto-prefix"); + + let opts = http_options(&repo); + assert_eq!(opts.proxy.as_deref(), Some("http://localhost:9090")); + } +} diff --git a/git-repository/tests/repository/mod.rs b/git-repository/tests/repository/mod.rs index 891efe34c99..6319c3c0a4a 100644 --- a/git-repository/tests/repository/mod.rs +++ b/git-repository/tests/repository/mod.rs @@ -11,7 +11,7 @@ mod worktree; #[test] fn size_in_memory() { let actual_size = std::mem::size_of::(); - let limit = 864; + let limit = 900; assert!( actual_size <= limit, "size of Repository shouldn't change without us noticing, it's meant to be cloned: should have been below {:?}, was {} (bigger on windows)", diff --git a/git-transport/src/client/async_io/traits.rs b/git-transport/src/client/async_io/traits.rs index d2120406a33..a4e818412c1 100644 --- a/git-transport/src/client/async_io/traits.rs +++ b/git-transport/src/client/async_io/traits.rs @@ -77,7 +77,7 @@ pub trait TransportV2Ext { async fn invoke<'a>( &mut self, command: &str, - capabilities: impl Iterator)> + 'a, + capabilities: impl Iterator>)> + 'a, arguments: Option + 'a>, ) -> Result, Error>; } @@ -87,14 +87,18 @@ impl TransportV2Ext for T { async fn invoke<'a>( &mut self, command: &str, - capabilities: impl Iterator)> + 'a, + capabilities: impl Iterator>)> + 'a, arguments: Option + 'a>, ) -> Result, Error> { let mut writer = self.request(WriteMode::OneLfTerminatedLinePerWriteCall, MessageKind::Flush)?; writer.write_all(format!("command={}", command).as_bytes()).await?; for (name, value) in capabilities { match value { - Some(value) => writer.write_all(format!("{}={}", name, value).as_bytes()).await, + Some(value) => { + writer + .write_all(format!("{}={}", name, value.as_ref()).as_bytes()) + .await + } None => writer.write_all(name.as_bytes()).await, }?; } diff --git a/git-transport/src/client/blocking_io/file.rs b/git-transport/src/client/blocking_io/file.rs index fd377c617e4..7ad6a167166 100644 --- a/git-transport/src/client/blocking_io/file.rs +++ b/git-transport/src/client/blocking_io/file.rs @@ -1,10 +1,11 @@ +use std::borrow::Cow; use std::{ any::Any, error::Error, process::{self, Command, Stdio}, }; -use bstr::{BString, ByteSlice}; +use bstr::{BStr, BString, ByteSlice}; use crate::{ client::{self, git, MessageKind, RequestWriter, SetServiceResponse, WriteMode}, @@ -93,8 +94,8 @@ impl client::TransportWithoutIO for SpawnProcessOnDemand { .request(write_mode, on_into_read) } - fn to_url(&self) -> String { - self.url.to_bstring().to_string() + fn to_url(&self) -> Cow<'_, BStr> { + Cow::Owned(self.url.to_bstring()) } fn connection_persists_across_multiple_requests(&self) -> bool { diff --git a/git-transport/src/client/blocking_io/http/curl/mod.rs b/git-transport/src/client/blocking_io/http/curl/mod.rs index e2504ef6a14..f84555f1ea7 100644 --- a/git-transport/src/client/blocking_io/http/curl/mod.rs +++ b/git-transport/src/client/blocking_io/http/curl/mod.rs @@ -14,6 +14,7 @@ pub struct Curl { req: SyncSender, res: Receiver, handle: Option>>, + config: http::Options, } impl Curl { @@ -48,6 +49,7 @@ impl Curl { url: url.to_owned(), headers: list, upload, + config: self.config.clone(), }) .is_err() { @@ -76,6 +78,7 @@ impl Default for Curl { handle: Some(handle), req, res, + config: http::Options::default(), } } } @@ -102,7 +105,10 @@ impl http::Http for Curl { self.make_request(url, headers, true) } - fn configure(&mut self, _config: &dyn std::any::Any) -> Result<(), Box> { + fn configure(&mut self, config: &dyn std::any::Any) -> Result<(), Box> { + if let Some(config) = config.downcast_ref::() { + self.config = config.clone(); + } Ok(()) } } diff --git a/git-transport/src/client/blocking_io/http/curl/remote.rs b/git-transport/src/client/blocking_io/http/curl/remote.rs index 1c6c18967c4..9d671edfd74 100644 --- a/git-transport/src/client/blocking_io/http/curl/remote.rs +++ b/git-transport/src/client/blocking_io/http/curl/remote.rs @@ -92,6 +92,7 @@ pub struct Request { pub url: String, pub headers: curl::easy::List, pub upload: bool, + pub config: http::Options, } pub struct Response { @@ -110,19 +111,60 @@ pub fn new() -> ( let handle = std::thread::spawn(move || -> Result<(), curl::Error> { let mut handle = Easy2::new(Handler::default()); - for Request { url, headers, upload } in req_recv { + for Request { + url, + mut headers, + upload, + config: + http::Options { + extra_headers, + follow_redirects: _, + low_speed_limit_bytes_per_second, + low_speed_time_seconds, + connect_timeout, + proxy, + proxy_auth_method: _, + user_agent, + backend: _, + }, + } in req_recv + { handle.url(&url)?; // GitHub sends 'chunked' to avoid unknown clients to choke on the data, I suppose handle.post(upload)?; + for header in extra_headers { + headers.append(&header)?; + } + if let Some(proxy) = proxy { + handle.proxy(&proxy)?; + let proxy_type = if proxy.starts_with("socks5h") { + curl::easy::ProxyType::Socks5Hostname + } else if proxy.starts_with("socks5") { + curl::easy::ProxyType::Socks5 + } else if proxy.starts_with("socks4a") { + curl::easy::ProxyType::Socks4a + } else if proxy.starts_with("socks") { + curl::easy::ProxyType::Socks4 + } else { + curl::easy::ProxyType::Http + }; + handle.proxy_type(proxy_type)?; + } + if let Some(user_agent) = user_agent { + handle.useragent(&user_agent)?; + } handle.http_headers(headers)?; handle.transfer_encoding(false)?; - handle.connect_timeout(Duration::from_secs(20))?; - // handle.proxy("http://localhost:9090")?; // DEBUG - let low_bytes_per_second = 1024; - handle.low_speed_limit(low_bytes_per_second)?; - handle.low_speed_time(Duration::from_secs(20))?; + if let Some(timeout) = connect_timeout { + handle.connect_timeout(timeout)?; + } + handle.tcp_keepalive(true)?; + if low_speed_time_seconds > 0 && low_speed_limit_bytes_per_second > 0 { + handle.low_speed_limit(low_speed_limit_bytes_per_second)?; + handle.low_speed_time(Duration::from_secs(low_speed_time_seconds))?; + } let (receive_data, receive_headers, send_body) = { let handler = handle.get_mut(); let (send, receive_data) = pipe::unidirectional(1); diff --git a/git-transport/src/client/blocking_io/http/mod.rs b/git-transport/src/client/blocking_io/http/mod.rs index 0b7bb93bdf7..9474e74c62b 100644 --- a/git-transport/src/client/blocking_io/http/mod.rs +++ b/git-transport/src/client/blocking_io/http/mod.rs @@ -1,3 +1,5 @@ +use bstr::BStr; +use std::sync::{Arc, Mutex}; use std::{ any::Any, borrow::Cow, @@ -15,15 +17,106 @@ use crate::{ #[cfg(feature = "http-client-curl")] mod curl; +/// The experimental `reqwest` backend. +/// +/// It doesn't support any of the shared http options yet, but can be seen as example on how to integrate blocking `http` backends. +/// There is also nothing that would prevent it from becoming a fully-featured HTTP backend except for demand and time. #[cfg(feature = "http-client-reqwest")] -mod reqwest; +pub mod reqwest; /// mod traits; -/// The http client configuration when using reqwest -#[cfg(feature = "http-client-reqwest")] -pub type Options = reqwest::Options; +/// +pub mod options { + /// Possible settings for the `http.followRedirects` configuration option. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum FollowRedirects { + /// Follow only the first redirect request, most suitable for typical git requests. + Initial, + /// Follow all redirect requests from the server unconditionally + All, + /// Follow no redirect request. + None, + } + + impl Default for FollowRedirects { + fn default() -> Self { + FollowRedirects::Initial + } + } + + /// The way to configure a proxy for authentication if a username is present in the configured proxy. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum ProxyAuthMethod { + /// Automatically pick a suitable authentication method. + AnyAuth, + ///HTTP basic authentication. + Basic, + /// Http digest authentication to prevent a password to be passed in clear text. + Digest, + /// GSS negotiate authentication. + Negotiate, + /// NTLM authentication + Ntlm, + } + + impl Default for ProxyAuthMethod { + fn default() -> Self { + ProxyAuthMethod::AnyAuth + } + } +} + +/// Options to configure curl requests. +// TODO: testing most of these fields requires a lot of effort, unless special flags to introspect ongoing requests are added. +#[derive(Default, Debug, Clone)] +pub struct Options { + /// Headers to be added to every request. + /// They are applied unconditionally and are expected to be valid as they occour in an HTTP request, like `header: value`, without newlines. + /// + /// Refers to `http.extraHeader` multi-var. + pub extra_headers: Vec, + /// How to handle redirects. + /// + /// Refers to `http.followRedirects`. + pub follow_redirects: options::FollowRedirects, + /// Used in conjunction with `low_speed_time_seconds`, any non-0 value signals the amount of bytes per second at least to avoid + /// aborting the connection. + /// + /// Refers to `http.lowSpeedLimit`. + pub low_speed_limit_bytes_per_second: u32, + /// Used in conjunction with `low_speed_bytes_per_second`, any non-0 value signals the amount seconds the minimal amount + /// of bytes per second isn't reached. + /// + /// Refers to `http.lowSpeedTime`. + pub low_speed_time_seconds: u64, + /// A curl-style proxy declaration of the form `[protocol://][user[:password]@]proxyhost[:port]`. + /// + /// Refers to `http.proxy`. + pub proxy: Option, + /// The way to authenticate against the proxy if the `proxy` field contains a username. + /// + /// Refers to `http.proxyAuthMethod`. + pub proxy_auth_method: Option, + /// The `HTTP` `USER_AGENT` string presented to an `HTTP` server, notably not the user agent present to the `git` server. + /// + /// If not overridden, it defaults to the user agent provided by `curl`, which is a deviation from how `git` handles this. + /// Thus it's expected from the callers to set it to their application, or use higher-level crates which make it easy to do this + /// more correctly. + /// + /// Using the correct user-agent might affect how the server treats the request. + /// + /// Refers to `http.userAgent`. + pub user_agent: Option, + /// The amount of time we wait until aborting a connection attempt. + /// + /// If `None`, this typically defaults to 2 minutes to 5 minutes. + /// Refers to `gitoxide.http.connectTimeout`. + pub connect_timeout: Option, + /// Backend specific options, if available. + pub backend: Option>>, +} /// The actual http client implementation, using curl #[cfg(feature = "http-client-curl")] @@ -168,8 +261,8 @@ impl client::TransportWithoutIO for Transport { )) } - fn to_url(&self) -> String { - self.url.clone() + fn to_url(&self) -> Cow<'_, BStr> { + Cow::Borrowed(self.url.as_str().into()) } fn supported_protocol_versions(&self) -> &[Protocol] { diff --git a/git-transport/src/client/blocking_io/http/reqwest.rs b/git-transport/src/client/blocking_io/http/reqwest.rs deleted file mode 100644 index dee44efce2d..00000000000 --- a/git-transport/src/client/blocking_io/http/reqwest.rs +++ /dev/null @@ -1,258 +0,0 @@ -pub use crate::client::http::reqwest::remote::Options; - -pub struct Remote { - /// A worker thread which performs the actual request. - handle: Option>>, - /// A channel to send requests (work) to the worker thread. - request: std::sync::mpsc::SyncSender, - /// A channel to receive the result of the prior request. - response: std::sync::mpsc::Receiver, - /// A mechanism for configuring the remote. - config: Options, -} - -mod remote { - use std::{ - any::Any, - convert::TryFrom, - io::Write, - str::FromStr, - sync::{Arc, Mutex}, - }; - - use git_features::io::pipe; - - use crate::client::{http, http::reqwest::Remote}; - - #[derive(Debug, thiserror::Error)] - pub enum Error { - #[error(transparent)] - Reqwest(#[from] reqwest::Error), - #[error("Request configuration failed")] - ConfigureRequest(#[from] Box), - } - - impl Default for Remote { - fn default() -> Self { - let (req_send, req_recv) = std::sync::mpsc::sync_channel(0); - let (res_send, res_recv) = std::sync::mpsc::sync_channel(0); - let handle = std::thread::spawn(move || -> Result<(), Error> { - for Request { - url, - headers, - upload, - config, - } in req_recv - { - // We may error while configuring, which is expected as part of the internal protocol. The error will be - // received and the sender of the request might restart us. - let client = reqwest::blocking::ClientBuilder::new() - .connect_timeout(std::time::Duration::from_secs(20)) - .build()?; - let mut req_builder = if upload { client.post(url) } else { client.get(url) }.headers(headers); - let (post_body_tx, post_body_rx) = pipe::unidirectional(0); - if upload { - req_builder = req_builder.body(reqwest::blocking::Body::new(post_body_rx)); - } - let mut req = req_builder.build()?; - let (mut response_body_tx, response_body_rx) = pipe::unidirectional(0); - let (mut headers_tx, headers_rx) = pipe::unidirectional(0); - if res_send - .send(Response { - headers: headers_rx, - body: response_body_rx, - upload_body: post_body_tx, - }) - .is_err() - { - // This means our internal protocol is violated as the one who sent the request isn't listening anymore. - // Shut down as something is off. - break; - } - if let Some(mutex) = config.configure_request { - let mut configure_request = mutex.lock().expect("our thread cannot ordinarily panic"); - configure_request(&mut req)?; - } - let mut res = match client.execute(req).and_then(|res| res.error_for_status()) { - Ok(res) => res, - Err(err) => { - let (kind, err) = match err.status() { - Some(status) => { - let kind = if status == reqwest::StatusCode::UNAUTHORIZED { - std::io::ErrorKind::PermissionDenied - } else { - std::io::ErrorKind::Other - }; - (kind, format!("Received HTTP status {}", status.as_str())) - } - None => (std::io::ErrorKind::Other, err.to_string()), - }; - let err = Err(std::io::Error::new(kind, err)); - headers_tx.channel.send(err).ok(); - continue; - } - }; - - let send_headers = { - let headers = res.headers(); - move || -> std::io::Result<()> { - for (name, value) in headers { - headers_tx.write_all(name.as_str().as_bytes())?; - headers_tx.write_all(b":")?; - headers_tx.write_all(value.as_bytes())?; - headers_tx.write_all(b"\n")?; - } - // Make sure this is an FnOnce closure to signal the remote reader we are done. - drop(headers_tx); - Ok(()) - } - }; - - // We don't have to care if anybody is receiving the header, as a matter of fact we cannot fail sending them. - // Thus an error means the receiver failed somehow, but might also have decided not to read headers at all. Fine with us. - send_headers().ok(); - - // reading the response body is streaming and may fail for many reasons. If so, we send the error over the response - // body channel and that's all we can do. - if let Err(err) = std::io::copy(&mut res, &mut response_body_tx) { - response_body_tx.channel.send(Err(err)).ok(); - } - } - Ok(()) - }); - - Remote { - handle: Some(handle), - request: req_send, - response: res_recv, - config: Options::default(), - } - } - } - - /// utilities - impl Remote { - fn make_request( - &mut self, - url: &str, - headers: impl IntoIterator>, - upload: bool, - ) -> Result, http::Error> { - let mut header_map = reqwest::header::HeaderMap::new(); - for header_line in headers { - let header_line = header_line.as_ref(); - let colon_pos = header_line - .find(':') - .expect("header line must contain a colon to separate key and value"); - let header_name = &header_line[..colon_pos]; - let value = &header_line[colon_pos + 1..]; - - match reqwest::header::HeaderName::from_str(header_name) - .ok() - .zip(reqwest::header::HeaderValue::try_from(value.trim()).ok()) - { - Some((key, val)) => header_map.insert(key, val), - None => continue, - }; - } - self.request - .send(Request { - url: url.to_owned(), - headers: header_map, - upload, - config: self.config.clone(), - }) - .expect("the remote cannot be down at this point"); - - let Response { - headers, - body, - upload_body, - } = match self.response.recv() { - Ok(res) => res, - Err(_) => { - let err = self - .handle - .take() - .expect("always present") - .join() - .expect("no panic") - .expect_err("no receiver means thread is down with init error"); - *self = Self::default(); - return Err(http::Error::InitHttpClient { source: Box::new(err) }); - } - }; - - Ok(http::PostResponse { - post_body: upload_body, - headers, - body, - }) - } - } - - impl http::Http for Remote { - type Headers = pipe::Reader; - type ResponseBody = pipe::Reader; - type PostBody = pipe::Writer; - - fn get( - &mut self, - url: &str, - headers: impl IntoIterator>, - ) -> Result, http::Error> { - self.make_request(url, headers, false).map(Into::into) - } - - fn post( - &mut self, - url: &str, - headers: impl IntoIterator>, - ) -> Result, http::Error> { - self.make_request(url, headers, true) - } - - fn configure(&mut self, config: &dyn Any) -> Result<(), Box> { - if let Some(config) = config.downcast_ref::() { - self.config = config.clone(); - } - Ok(()) - } - } - - /// Options to configure the reqwest HTTP handler. - #[derive(Default, Clone)] - pub struct Options { - /// A function to configure the request that is about to be made. - pub configure_request: Option< - Arc< - Mutex< - dyn FnMut( - &mut reqwest::blocking::Request, - ) -> Result<(), Box> - + Send - + Sync - + 'static, - >, - >, - >, - } - - pub struct Request { - pub url: String, - pub headers: reqwest::header::HeaderMap, - pub upload: bool, - pub config: Options, - } - - /// A link to a thread who provides data for the contained readers. - /// The expected order is: - /// - write `upload_body` - /// - read `headers` to end - /// - read `body` to hend - pub struct Response { - pub headers: pipe::Reader, - pub body: pipe::Reader, - pub upload_body: pipe::Writer, - } -} diff --git a/git-transport/src/client/blocking_io/http/reqwest/mod.rs b/git-transport/src/client/blocking_io/http/reqwest/mod.rs new file mode 100644 index 00000000000..ff53742718a --- /dev/null +++ b/git-transport/src/client/blocking_io/http/reqwest/mod.rs @@ -0,0 +1,27 @@ +/// An implementation for HTTP requests via `reqwest`. +pub struct Remote { + /// A worker thread which performs the actual request. + handle: Option>>, + /// A channel to send requests (work) to the worker thread. + request: std::sync::mpsc::SyncSender, + /// A channel to receive the result of the prior request. + response: std::sync::mpsc::Receiver, + /// A mechanism for configuring the remote. + config: crate::client::http::Options, +} + +/// Options to configure the reqwest HTTP handler. +#[derive(Default)] +pub struct Options { + /// A function to configure the request that is about to be made. + pub configure_request: Option< + Box< + dyn FnMut(&mut reqwest::blocking::Request) -> Result<(), Box> + + Send + + Sync + + 'static, + >, + >, +} + +mod remote; diff --git a/git-transport/src/client/blocking_io/http/reqwest/remote.rs b/git-transport/src/client/blocking_io/http/reqwest/remote.rs new file mode 100644 index 00000000000..d0341bd1705 --- /dev/null +++ b/git-transport/src/client/blocking_io/http/reqwest/remote.rs @@ -0,0 +1,222 @@ +use std::{any::Any, convert::TryFrom, io::Write, str::FromStr}; + +use git_features::io::pipe; + +use crate::client::{http, http::reqwest::Remote}; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Reqwest(#[from] reqwest::Error), + #[error("Request configuration failed")] + ConfigureRequest(#[from] Box), +} + +impl Default for Remote { + fn default() -> Self { + let (req_send, req_recv) = std::sync::mpsc::sync_channel(0); + let (res_send, res_recv) = std::sync::mpsc::sync_channel(0); + let handle = std::thread::spawn(move || -> Result<(), Error> { + for Request { + url, + headers, + upload, + config, + } in req_recv + { + // We may error while configuring, which is expected as part of the internal protocol. The error will be + // received and the sender of the request might restart us. + let client = reqwest::blocking::ClientBuilder::new() + .connect_timeout(std::time::Duration::from_secs(20)) + .build()?; + let mut req_builder = if upload { client.post(url) } else { client.get(url) }.headers(headers); + let (post_body_tx, post_body_rx) = pipe::unidirectional(0); + if upload { + req_builder = req_builder.body(reqwest::blocking::Body::new(post_body_rx)); + } + let mut req = req_builder.build()?; + let (mut response_body_tx, response_body_rx) = pipe::unidirectional(0); + let (mut headers_tx, headers_rx) = pipe::unidirectional(0); + if res_send + .send(Response { + headers: headers_rx, + body: response_body_rx, + upload_body: post_body_tx, + }) + .is_err() + { + // This means our internal protocol is violated as the one who sent the request isn't listening anymore. + // Shut down as something is off. + break; + } + if let Some(ref mut request_options) = config.backend.as_ref().and_then(|backend| backend.lock().ok()) { + if let Some(options) = request_options.downcast_mut::() { + if let Some(configure_request) = &mut options.configure_request { + configure_request(&mut req)?; + } + } + } + let mut res = match client.execute(req).and_then(|res| res.error_for_status()) { + Ok(res) => res, + Err(err) => { + let (kind, err) = match err.status() { + Some(status) => { + let kind = if status == reqwest::StatusCode::UNAUTHORIZED { + std::io::ErrorKind::PermissionDenied + } else { + std::io::ErrorKind::Other + }; + (kind, format!("Received HTTP status {}", status.as_str())) + } + None => (std::io::ErrorKind::Other, err.to_string()), + }; + let err = Err(std::io::Error::new(kind, err)); + headers_tx.channel.send(err).ok(); + continue; + } + }; + + let send_headers = { + let headers = res.headers(); + move || -> std::io::Result<()> { + for (name, value) in headers { + headers_tx.write_all(name.as_str().as_bytes())?; + headers_tx.write_all(b":")?; + headers_tx.write_all(value.as_bytes())?; + headers_tx.write_all(b"\n")?; + } + // Make sure this is an FnOnce closure to signal the remote reader we are done. + drop(headers_tx); + Ok(()) + } + }; + + // We don't have to care if anybody is receiving the header, as a matter of fact we cannot fail sending them. + // Thus an error means the receiver failed somehow, but might also have decided not to read headers at all. Fine with us. + send_headers().ok(); + + // reading the response body is streaming and may fail for many reasons. If so, we send the error over the response + // body channel and that's all we can do. + if let Err(err) = std::io::copy(&mut res, &mut response_body_tx) { + response_body_tx.channel.send(Err(err)).ok(); + } + } + Ok(()) + }); + + Remote { + handle: Some(handle), + request: req_send, + response: res_recv, + config: http::Options::default(), + } + } +} + +/// utilities +impl Remote { + fn make_request( + &mut self, + url: &str, + headers: impl IntoIterator>, + upload: bool, + ) -> Result, http::Error> { + let mut header_map = reqwest::header::HeaderMap::new(); + for header_line in headers { + let header_line = header_line.as_ref(); + let colon_pos = header_line + .find(':') + .expect("header line must contain a colon to separate key and value"); + let header_name = &header_line[..colon_pos]; + let value = &header_line[colon_pos + 1..]; + + match reqwest::header::HeaderName::from_str(header_name) + .ok() + .zip(reqwest::header::HeaderValue::try_from(value.trim()).ok()) + { + Some((key, val)) => header_map.insert(key, val), + None => continue, + }; + } + self.request + .send(Request { + url: url.to_owned(), + headers: header_map, + upload, + config: self.config.clone(), + }) + .expect("the remote cannot be down at this point"); + + let Response { + headers, + body, + upload_body, + } = match self.response.recv() { + Ok(res) => res, + Err(_) => { + let err = self + .handle + .take() + .expect("always present") + .join() + .expect("no panic") + .expect_err("no receiver means thread is down with init error"); + *self = Self::default(); + return Err(http::Error::InitHttpClient { source: Box::new(err) }); + } + }; + + Ok(http::PostResponse { + post_body: upload_body, + headers, + body, + }) + } +} + +impl http::Http for Remote { + type Headers = pipe::Reader; + type ResponseBody = pipe::Reader; + type PostBody = pipe::Writer; + + fn get( + &mut self, + url: &str, + headers: impl IntoIterator>, + ) -> Result, http::Error> { + self.make_request(url, headers, false).map(Into::into) + } + + fn post( + &mut self, + url: &str, + headers: impl IntoIterator>, + ) -> Result, http::Error> { + self.make_request(url, headers, true) + } + + fn configure(&mut self, config: &dyn Any) -> Result<(), Box> { + if let Some(config) = config.downcast_ref::() { + self.config = config.clone(); + } + Ok(()) + } +} + +pub(crate) struct Request { + pub url: String, + pub headers: reqwest::header::HeaderMap, + pub upload: bool, + pub config: http::Options, +} + +/// A link to a thread who provides data for the contained readers. +/// The expected order is: +/// - write `upload_body` +/// - read `headers` to end +/// - read `body` to hend +pub(crate) struct Response { + pub headers: pipe::Reader, + pub body: pipe::Reader, + pub upload_body: pipe::Writer, +} diff --git a/git-transport/src/client/blocking_io/traits.rs b/git-transport/src/client/blocking_io/traits.rs index 19ca27f9334..0b1bae1cc1b 100644 --- a/git-transport/src/client/blocking_io/traits.rs +++ b/git-transport/src/client/blocking_io/traits.rs @@ -70,7 +70,7 @@ pub trait TransportV2Ext { fn invoke<'a>( &mut self, command: &str, - capabilities: impl Iterator)>, + capabilities: impl Iterator>)> + 'a, arguments: Option>, ) -> Result, Error>; } @@ -79,14 +79,14 @@ impl TransportV2Ext for T { fn invoke<'a>( &mut self, command: &str, - capabilities: impl Iterator)>, + capabilities: impl Iterator>)> + 'a, arguments: Option>, ) -> Result, Error> { let mut writer = self.request(WriteMode::OneLfTerminatedLinePerWriteCall, MessageKind::Flush)?; writer.write_all(format!("command={}", command).as_bytes())?; for (name, value) in capabilities { match value { - Some(value) => writer.write_all(format!("{}={}", name, value).as_bytes()), + Some(value) => writer.write_all(format!("{}={}", name, value.as_ref()).as_bytes()), None => writer.write_all(name.as_bytes()), }?; } diff --git a/git-transport/src/client/git/async_io.rs b/git-transport/src/client/git/async_io.rs index b8965183497..cc8a55d4af5 100644 --- a/git-transport/src/client/git/async_io.rs +++ b/git-transport/src/client/git/async_io.rs @@ -1,7 +1,8 @@ +use std::borrow::Cow; use std::error::Error; use async_trait::async_trait; -use bstr::BString; +use bstr::{BStr, BString, ByteVec}; use futures_io::{AsyncRead, AsyncWrite}; use futures_lite::AsyncWriteExt; use git_packetline::PacketLineRef; @@ -28,14 +29,14 @@ where on_into_read, )) } - fn to_url(&self) -> String { + fn to_url(&self) -> Cow<'_, BStr> { self.custom_url.as_ref().map_or_else( || { - let mut possibly_lossy_url = self.path.to_string(); + let mut possibly_lossy_url = self.path.clone(); possibly_lossy_url.insert_str(0, "file://"); - possibly_lossy_url + Cow::Owned(possibly_lossy_url) }, - |url| url.clone(), + |url| Cow::Borrowed(url.as_ref()), ) } diff --git a/git-transport/src/client/git/blocking_io.rs b/git-transport/src/client/git/blocking_io.rs index d3744839220..0490f2b94c9 100644 --- a/git-transport/src/client/git/blocking_io.rs +++ b/git-transport/src/client/git/blocking_io.rs @@ -1,6 +1,7 @@ +use std::borrow::Cow; use std::{any::Any, error::Error, io::Write}; -use bstr::BString; +use bstr::{BStr, BString, ByteVec}; use git_packetline::PacketLineRef; use crate::{ @@ -26,14 +27,14 @@ where )) } - fn to_url(&self) -> String { + fn to_url(&self) -> Cow<'_, BStr> { self.custom_url.as_ref().map_or_else( || { - let mut possibly_lossy_url = self.path.to_string(); + let mut possibly_lossy_url = self.path.clone(); possibly_lossy_url.insert_str(0, "file://"); - possibly_lossy_url + Cow::Owned(possibly_lossy_url) }, - |url| url.clone(), + |url| Cow::Borrowed(url.as_ref()), ) } diff --git a/git-transport/src/client/git/mod.rs b/git-transport/src/client/git/mod.rs index 2aaa6f42210..3f549e34b3d 100644 --- a/git-transport/src/client/git/mod.rs +++ b/git-transport/src/client/git/mod.rs @@ -22,7 +22,7 @@ pub struct Connection { pub(in crate::client) virtual_host: Option<(String, Option)>, pub(in crate::client) desired_version: Protocol, supported_versions: [Protocol; 1], - custom_url: Option, + custom_url: Option, pub(in crate::client) mode: ConnectMode, } @@ -37,7 +37,7 @@ impl Connection { /// The URL is required as parameter for authentication helpers which are called in transports /// that support authentication. Even though plain git transports don't support that, this /// may well be the case in custom transports. - pub fn custom_url(mut self, url: Option) -> Self { + pub fn custom_url(mut self, url: Option) -> Self { self.custom_url = url; self } diff --git a/git-transport/src/client/traits.rs b/git-transport/src/client/traits.rs index 91b593b0b69..47dcdb14c5f 100644 --- a/git-transport/src/client/traits.rs +++ b/git-transport/src/client/traits.rs @@ -1,3 +1,5 @@ +use bstr::BStr; +use std::borrow::Cow; use std::{ any::Any, ops::{Deref, DerefMut}, @@ -27,9 +29,7 @@ pub trait TransportWithoutIO { fn request(&mut self, write_mode: WriteMode, on_into_read: MessageKind) -> Result, Error>; /// Returns the canonical URL pointing to the destination of this transport. - /// Please note that local paths may not be represented correctly, as they will go through a potentially lossy - /// unicode conversion. - fn to_url(&self) -> String; + fn to_url(&self) -> Cow<'_, BStr>; /// If the actually advertised server version is contained in the returned slice or empty, continue as normal, /// assume the server's protocol version is desired or acceptable. @@ -67,7 +67,7 @@ impl TransportWithoutIO for Box { self.deref_mut().request(write_mode, on_into_read) } - fn to_url(&self) -> String { + fn to_url(&self) -> Cow<'_, BStr> { self.deref().to_url() } @@ -94,7 +94,7 @@ impl TransportWithoutIO for &mut T { self.deref_mut().request(write_mode, on_into_read) } - fn to_url(&self) -> String { + fn to_url(&self) -> Cow<'_, BStr> { self.deref().to_url() } diff --git a/git-transport/tests/client/blocking_io/http/mock.rs b/git-transport/tests/client/blocking_io/http/mock.rs index 9fc2b7ad66e..5bbf6291ac7 100644 --- a/git-transport/tests/client/blocking_io/http/mock.rs +++ b/git-transport/tests/client/blocking_io/http/mock.rs @@ -106,6 +106,6 @@ pub fn serve_and_connect( path ); let client = git_transport::client::http::connect(&url, version); - assert_eq!(url, client.to_url()); + assert_eq!(url, client.to_url().as_ref()); Ok((server, client)) } diff --git a/git-transport/tests/client/blocking_io/http/mod.rs b/git-transport/tests/client/blocking_io/http/mod.rs index a508c31e52c..364115dea0c 100644 --- a/git-transport/tests/client/blocking_io/http/mod.rs +++ b/git-transport/tests/client/blocking_io/http/mod.rs @@ -451,7 +451,11 @@ Git-Protocol: version=2 ); server.next_read_and_respond_with(fixture_bytes("v2/http-fetch.response")); - let mut res = c.invoke("fetch", Vec::new().into_iter(), None::>)?; + let mut res = c.invoke( + "fetch", + Vec::<(_, Option<&str>)>::new().into_iter(), + None::>, + )?; let mut line = String::new(); res.read_line(&mut line)?; assert_eq!(line, "packfile\n"); diff --git a/git-transport/tests/client/git.rs b/git-transport/tests/client/git.rs index b8d2d10e0b0..0ca41c86cdd 100644 --- a/git-transport/tests/client/git.rs +++ b/git-transport/tests/client/git.rs @@ -33,9 +33,9 @@ async fn handshake_v1_and_request() -> crate::Result { "tcp connections are stateful" ); let c = c.custom_url(Some("anything".into())); - assert_eq!(c.to_url(), "anything"); + assert_eq!(c.to_url().as_ref(), "anything"); let mut c = c.custom_url(None); - assert_eq!(c.to_url(), "file:///foo.git"); + assert_eq!(c.to_url().as_ref(), "file:///foo.git"); let mut res = c.handshake(Service::UploadPack, &[]).await?; assert_eq!(res.actual_protocol, Protocol::V1); assert_eq!( diff --git a/gitoxide-core/src/pack/receive.rs b/gitoxide-core/src/pack/receive.rs index 869a6a98eaf..147c5485f07 100644 --- a/gitoxide-core/src/pack/receive.rs +++ b/gitoxide-core/src/pack/receive.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::{ io, path::PathBuf, @@ -11,7 +12,8 @@ use git_repository::{ odb::pack, protocol, protocol::{ - fetch::{Action, Arguments, LsRefsAction, Ref, Response}, + fetch::{Action, Arguments, Response}, + handshake::Ref, transport, transport::client::Capabilities, }, @@ -51,15 +53,15 @@ impl protocol::fetch::DelegateBlocking for CloneDelegate { &mut self, server: &Capabilities, arguments: &mut Vec, - _features: &mut Vec<(&str, Option<&str>)>, - ) -> io::Result { + _features: &mut Vec<(&str, Option>)>, + ) -> io::Result { if server.contains("ls-refs") { arguments.extend(FILTER.iter().map(|r| format!("ref-prefix {}", r).into())); } Ok(if self.wanted_refs.is_empty() { - LsRefsAction::Continue + ls_refs::Action::Continue } else { - LsRefsAction::Skip + ls_refs::Action::Skip }) } @@ -67,7 +69,7 @@ impl protocol::fetch::DelegateBlocking for CloneDelegate { &mut self, version: transport::Protocol, server: &Capabilities, - _features: &mut Vec<(&str, Option<&str>)>, + _features: &mut Vec<(&str, Option>)>, _refs: &[Ref], ) -> io::Result { if !self.wanted_refs.is_empty() && !remote_supports_ref_in_want(server) { @@ -115,12 +117,8 @@ impl protocol::fetch::DelegateBlocking for CloneDelegate { mod blocking_io { use std::{io, io::BufRead, path::PathBuf}; - use git_repository::{ - bstr::BString, - protocol, - protocol::fetch::{Ref, Response}, - Progress, - }; + use git_repository as git; + use git_repository::{bstr::BString, protocol, protocol::fetch::Response, protocol::handshake::Ref, Progress}; use super::{receive_pack_blocking, CloneDelegate, Context}; use crate::net; @@ -172,6 +170,7 @@ mod blocking_io { protocol::credentials::builtin, progress, protocol::FetchConnection::TerminateOnSuccessfulCompletion, + git::env::agent(), )?; Ok(()) } @@ -179,6 +178,7 @@ mod blocking_io { #[cfg(feature = "blocking-client")] pub use blocking_io::receive; +use git_repository::protocol::ls_refs; #[cfg(feature = "async-client")] mod async_io { @@ -190,12 +190,14 @@ mod async_io { bstr::{BString, ByteSlice}, odb::pack, protocol, - protocol::fetch::{Ref, Response}, + protocol::fetch::Response, + protocol::handshake::Ref, Progress, }; use super::{print, receive_pack_blocking, write_raw_refs, CloneDelegate, Context}; use crate::{net, OutputFormat}; + use git_repository as git; #[async_trait(?Send)] impl protocol::fetch::Delegate for CloneDelegate { @@ -245,6 +247,7 @@ mod async_io { protocol::credentials::builtin, progress, protocol::FetchConnection::TerminateOnSuccessfulCompletion, + git::env::agent(), )) }) .await?; diff --git a/gitoxide-core/src/repository/remote.rs b/gitoxide-core/src/repository/remote.rs index f13f3c579f9..0d7f03c6a70 100644 --- a/gitoxide-core/src/repository/remote.rs +++ b/gitoxide-core/src/repository/remote.rs @@ -3,7 +3,7 @@ mod refs_impl { use anyhow::bail; use git_repository as git; use git_repository::{ - protocol::fetch, + protocol::handshake, refspec::{match_group::validate::Fix, RefSpec}, remote::fetch::Source, }; @@ -227,21 +227,21 @@ mod refs_impl { }, } - impl From for JsonRef { - fn from(value: fetch::Ref) -> Self { + impl From for JsonRef { + fn from(value: handshake::Ref) -> Self { match value { - fetch::Ref::Unborn { full_ref_name, target } => JsonRef::Unborn { + handshake::Ref::Unborn { full_ref_name, target } => JsonRef::Unborn { path: full_ref_name.to_string(), target: target.to_string(), }, - fetch::Ref::Direct { + handshake::Ref::Direct { full_ref_name: path, object, } => JsonRef::Direct { path: path.to_string(), object: object.to_string(), }, - fetch::Ref::Symbolic { + handshake::Ref::Symbolic { full_ref_name: path, target, object, @@ -250,7 +250,7 @@ mod refs_impl { target: target.to_string(), object: object.to_string(), }, - fetch::Ref::Peeled { + handshake::Ref::Peeled { full_ref_name: path, tag, object, @@ -263,30 +263,30 @@ mod refs_impl { } } - pub(crate) fn print_ref(mut out: impl std::io::Write, r: &fetch::Ref) -> std::io::Result<&git::hash::oid> { + pub(crate) fn print_ref(mut out: impl std::io::Write, r: &handshake::Ref) -> std::io::Result<&git::hash::oid> { match r { - fetch::Ref::Direct { + handshake::Ref::Direct { full_ref_name: path, object, } => write!(&mut out, "{} {}", object, path).map(|_| object.as_ref()), - fetch::Ref::Peeled { + handshake::Ref::Peeled { full_ref_name: path, tag, object, } => write!(&mut out, "{} {} object:{}", tag, path, object).map(|_| tag.as_ref()), - fetch::Ref::Symbolic { + handshake::Ref::Symbolic { full_ref_name: path, target, object, } => write!(&mut out, "{} {} symref-target:{}", object, path, target).map(|_| object.as_ref()), - fetch::Ref::Unborn { full_ref_name, target } => { + handshake::Ref::Unborn { full_ref_name, target } => { static NULL: git::hash::ObjectId = git::hash::ObjectId::null(git::hash::Kind::Sha1); write!(&mut out, "unborn {} symref-target:{}", full_ref_name, target).map(|_| NULL.as_ref()) } } } - pub(crate) fn print(mut out: impl std::io::Write, refs: &[fetch::Ref]) -> std::io::Result<()> { + pub(crate) fn print(mut out: impl std::io::Write, refs: &[handshake::Ref]) -> std::io::Result<()> { for r in refs { print_ref(&mut out, r)?; writeln!(out)?; diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index bfffea78bf1..f2f1b1e8eec 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -6,20 +6,18 @@ use tabled::{Style, TableIteratorExt, Tabled}; #[derive(Clone)] enum Usage { - NotApplicable { - reason: &'static str, - }, - NotPlanned { - reason: &'static str, - }, - Planned { - note: Option<&'static str>, - }, + /// It's not reasonable to implement it as the prerequisites don't apply. + NotApplicable { reason: &'static str }, + /// We have no intention to implement it, but that can change if there is demand. + NotPlanned { reason: &'static str }, + /// We definitely want to implement this configuration value. + Planned { note: Option<&'static str> }, + /// The configuration is already effective and used (at least) in the given module `name`. InModule { name: &'static str, deviation: Option<&'static str>, }, - /// Needs analysis + /// Needs analysis, unclear how it works or what it does. Puzzled, } use Usage::*; @@ -73,7 +71,15 @@ impl Tabled for Record { fn fields(&self) -> Vec { let mut tokens = self.config.split('.'); - let mut buf = vec![tokens.next().expect("present").bold().to_string()]; + let mut buf = vec![{ + let name = tokens.next().expect("present"); + if name == "gitoxide" { + name.bold().green() + } else { + name.bold() + } + .to_string() + }]; buf.extend(tokens.map(ToOwned::to_owned)); vec![self.usage.icon().into(), buf.join("."), self.usage.to_string()] @@ -321,7 +327,7 @@ static GIT_CONFIG: &[Record] = &[ }, Record { config: "diff.algorithm", - usage: InModule {name: "config::cache::access", deviation: Some("'patience' diff is not implemented and can default to 'histogram' if lenient config is used")}, + usage: InModule {name: "config::cache::access", deviation: Some("'patience' diff is not implemented and can default to 'histogram' if lenient config is used, and defaults to histogram if unset for fastest and best results")}, }, Record { config: "extensions.objectFormat", @@ -544,6 +550,154 @@ static GIT_CONFIG: &[Record] = &[ config: "index.version", usage: Planned { note: Some("once V4 indices can be written, we need to be able to set a desired version. For now we write the smallest possible index version only.") }, }, + Record { + config: "http.proxy", + usage: InModule { name: "repository::config::transport", deviation: Some("ignores strings with illformed UTF-8") } + }, + Record { + config: "http.extraHeader", + usage: InModule { name: "repository::config::transport", deviation: Some("ignores strings with illformed UTF-8") } + }, + Record { + config: "http.proxyAuthMethod", + usage: Planned { note: None }, + }, + Record { + config: "http.proxySSLCert", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.proxySSLKey", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.proxySSLCertPasswordProtected", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.proxySSLCAInfo", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.emptyAuth", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.delegation", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.cookieFile", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.saveCookies", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.version", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.curloptResolve", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslVersion", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslCipherList", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslCipherList", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslVerify", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslCert", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslKey", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslCertPasswordProtected", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslCertPasswordProtected", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslCAInfo", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslCAPath", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslBackend", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.schannelCheckRevoke", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.schannelUseSSLCAInfo", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.pinnedPubkey", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.sslTry", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.maxRequests", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.minSessions", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.postBuffer", + usage: Planned { note: Some("relevant when implementing push, we should understand how memory allocation works when streaming") } + }, + Record { + config: "http.lowSpeedLimit", + usage: InModule { name: "repository::config::transport", deviation: Some("fails on negative values") } + }, + Record { + config: "http.lowSpeedTime", + usage: InModule { name: "repository::config::transport", deviation: Some("fails on negative values") } + }, + Record { + config: "http.userAgent", + usage: InModule { name: "repository::config::transport", deviation: Some("ignores strings with illformed UTF-8") } + }, + Record { + config: "http.noEPSV", + usage: NotPlanned { reason: "on demand" } + }, + Record { + config: "http.followRedirects", + usage: InModule { name: "repository::config::transport", deviation: None } + }, + Record { + config: "http..*", + usage: Planned { note: Some("it's a vital part of git configuration. It's unclear how to get a baseline from git for this one.") } + }, Record { config: "sparse.expectFilesOutsideOfPatterns", usage: NotPlanned { reason: "todo" }, @@ -559,6 +713,38 @@ static GIT_CONFIG: &[Record] = &[ usage: Planned { note: Some("required for big monorepos, and typically used in conjunction with sparse indices") } + }, + Record { + config: "remote..proxy", + usage: Planned { + note: None + } + }, + Record { + config: "remote..proxyAuthMethod", + usage: Planned { + note: None + } + }, + Record { + config: "gitoxide.userAgent", + usage: InModule { + name: "remote::connection", + deviation: None + } + }, + Record { + config: "gitoxide.http.noProxy", + usage: NotPlanned { + reason: "on demand, without it it's not possible to implement environment overrides via `no_proxy` or `NO_PROXY` for a list of hostnames or `*`" + } + }, + Record { + config: "gitoxide.http.connectTimeout", + usage: InModule { + name: "repository::config::transport", + deviation: Some("entirely new, and in milliseconds like all other timeout suffixed variables in the git config") + } } ]; @@ -572,7 +758,7 @@ pub fn show_progress() -> anyhow::Result<()> { println!("{}", sorted.table().with(Style::blank())); println!( - "\nTotal records: {} ({perfect_icon} = {perfect}, {deviation_icon} = {deviation}, {planned_icon} = {planned})", + "\nTotal records: {} ({perfect_icon} = {perfect}, {deviation_icon} = {deviation}, {planned_icon} = {planned}, {ondemand_icon} = {ondemand}, {not_applicable_icon} = {not_applicable})", GIT_CONFIG.len(), perfect_icon = InModule { name: "", @@ -586,6 +772,8 @@ pub fn show_progress() -> anyhow::Result<()> { .icon(), planned_icon = Planned { note: None }.icon(), planned = GIT_CONFIG.iter().filter(|e| matches!(e.usage, Planned { .. })).count(), + ondemand_icon = NotPlanned { reason: "" }.icon(), + not_applicable_icon = NotApplicable { reason: "" }.icon(), perfect = GIT_CONFIG .iter() .filter(|e| matches!(e.usage, InModule { deviation, .. } if deviation.is_none())) @@ -593,6 +781,14 @@ pub fn show_progress() -> anyhow::Result<()> { deviation = GIT_CONFIG .iter() .filter(|e| matches!(e.usage, InModule { deviation, .. } if deviation.is_some())) + .count(), + ondemand = GIT_CONFIG + .iter() + .filter(|e| matches!(e.usage, NotPlanned { .. })) + .count(), + not_applicable = GIT_CONFIG + .iter() + .filter(|e| matches!(e.usage, NotApplicable { .. })) .count() ); Ok(())