merge main

Cargo.toml

@@ -23,37 +23,37 @@ tag-prefix = ""
 lto = true
 
 [workspace.dependencies]
-anyhow = "1.0.92"
+anyhow = "1.0.94"
 archspec = "0.1.3"
 assert_matches = "1.5.0"
-async-compression = { version = "0.4.17", features = [
+async-compression = { version = "0.4.18", features = [
     "gzip",
     "tokio",
     "bzip2",
     "zstd",
 ] }
 async-fd-lock = "0.2.0"
-fs4 = "0.11.0"
+fs4 = "0.12.0"
 async-trait = "0.1.83"
-axum = { version = "0.7.7", default-features = false, features = [
+axum = { version = "0.7.9", default-features = false, features = [
     "tokio",
     "http1",
 ] }
 base64 = "0.22.1"
-bindgen = "0.69.5"
+bindgen = "0.71.1"
 blake2 = "0.10.6"
-bytes = "1.8.0"
-bzip2 = "0.4.4"
+bytes = "1.9.0"
+bzip2 = "0.5.0"
 cache_control = "0.2.0"
 cfg-if = "1.0"
-chrono = { version = "0.4.38", default-features = false, features = [
+chrono = { version = "0.4.39", default-features = false, features = [
     "std",
     "serde",
     "alloc",
 ] }
-clap = { version = "4.5.20", features = ["derive"] }
-cmake = "0.1.51"
-console = { version = "0.15.8", features = ["windows-console-colors"] }
+clap = { version = "4.5.23", features = ["derive"] }
+cmake = "0.1.52"
+console = { version = "0.15.10", features = ["windows-console-colors"] }
 criterion = "0.5"
 dashmap = "6.1.0"
 difference = "2.0.0"
@@ -66,28 +66,28 @@ fslock = "0.2.1"
 futures = "0.3.31"
 futures-util = "0.3.31"
 fxhash = "0.2.1"
-# lots of other crates are still stuck on older version
+# lots of other crates are still stuck on older version which breaks `deserialize`
 generic-array = "0.14.7"
 getrandom = { version = "0.2.15", default-features = false }
 glob = "0.3.1"
-google-cloud-auth = { version = "0.17.1", default-features = false }
+google-cloud-auth = { version = "0.17.2", default-features = false }
 google-cloud-token = "0.1.2"
 hex = "0.4.3"
 hex-literal = "0.4.1"
-http = "1.1"
+http = "1.2"
 http-cache-semantics = "2.1.0"
 humansize = "2.1.3"
 humantime = "2.1.0"
-indexmap = "2.6.0"
-indicatif = "0.17.8"
+indexmap = "2.7.0"
+indicatif = "0.17.9"
 insta = { version = "1.41.1" }
 itertools = "0.13.0"
 json-patch = "3.0.1"
 keyring = "3.6.1"
 lazy-regex = "3.3.0"
 lazy_static = "1.5.0"
 libc = { version = "0.2" }
-libloading = "0.8.5"
+libloading = "0.8.6"
 libz-sys = { version = "1.1.20", default-features = false }
 md-5 = "0.10.6"
 memchr = "2.7.4"
@@ -98,15 +98,16 @@ num_cpus = "1.16.0"
 once_cell = "1.20.2"
 ouroboros = "0.18.4"
 parking_lot = "0.12.3"
-pathdiff = "0.2.2"
-pep440_rs = { version = "0.7.2" }
+pathdiff = "0.2.3"
+pep440_rs = { version = "0.7.3" }
 pep508_rs = { version = "0.9.1" }
 percent-encoding = "2.3.1"
 pin-project-lite = "0.2.15"
 plist = "1"
 purl = { version = "0.1.3", features = ["serde"] }
 quote = "1.0.37"
 rand = "0.8.5"
+rayon = "1.10.0"
 reflink-copy = "0.1.20"
 regex = "1.11.1"
 reqwest = { version = "0.12.9", default-features = false }
@@ -117,9 +118,9 @@ retry-policies = { version = "0.4.0", default-features = false }
 rmp-serde = { version = "1.3.0" }
 rstest = { version = "0.23.0" }
 rstest_reuse = "0.7.0"
-simd-json = { version = "0.14.2", features = ["serde_impl"] }
-serde = { version = "1.0.214" }
-serde_json = { version = "1.0.132" }
+simd-json = { version = "0.14.3", features = ["serde_impl"] }
+serde = { version = "1.0.216" }
+serde_json = { version = "1.0.133" }
 serde_repr = "0.1"
 serde-value = "0.7.0"
 serde_with = "3.11.0"
@@ -136,28 +137,28 @@ smallvec = { version = "1.13.2", features = [
 ] }
 strum = { version = "0.26.3", features = ["derive"] }
 superslice = "1.0.0"
-syn = "2.0.86"
-sysinfo = "0.32.0"
-tar = "0.4.42"
+syn = "2.0.90"
+sysinfo = "0.33.0"
+tar = "0.4.43"
 tempdir = "0.3.7"
-tempfile = "3.13.0"
+tempfile = "3.14.0"
 temp-env = "0.3.6"
 test-log = "0.2.16"
-thiserror = "1.0"
-tokio = { version = "1.41.0", default-features = false }
-tokio-stream = "0.1.16"
-tokio-util = "0.7.12"
-tower = { version = "0.5.1", default-features = false }
-tower-http = { version = "0.6.1", default-features = false }
-tracing = "0.1.40"
-tracing-subscriber = { version = "0.3.18", default-features = false }
+thiserror = "2.0"
+tokio = { version = "1.42.0", default-features = false }
+tokio-stream = "0.1.17"
+tokio-util = "0.7.13"
+tower = { version = "0.5.2", default-features = false }
+tower-http = { version = "0.6.2", default-features = false }
+tracing = "0.1.41"
+tracing-subscriber = { version = "0.3.19", default-features = false }
 tracing-test = { version = "0.2.5" }
 trybuild = { version = "1.0.101" }
-typed-path = { version = "0.9.3" }
-url = { version = "2.5.2" }
+typed-path = { version = "0.10.0" }
+url = { version = "2.5.4" }
 uuid = { version = "1.11.0", default-features = false }
 unicode-normalization = { version = "0.1.24" }
 walkdir = "2.5.0"
 windows-sys = { version = "0.59.0", default-features = false }
-zip = { version = "2.2.0", default-features = false }
+zip = { version = "2.2.2", default-features = false }
 zstd = { version = "0.13.2", default-features = false }

crates/file_url/CHANGELOG.md

@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.1](https://github.com/conda/rattler/compare/file_url-v0.2.0...file_url-v0.2.1) - 2024-12-17
+
+### Other
+
+- update everything (#987)
+
 ## [0.2.0](https://github.com/conda/rattler/compare/file_url-v0.1.7...file_url-v0.2.0) - 2024-11-30
 
 ### Added

crates/file_url/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "file_url"
-version = "0.2.0"
+version = "0.2.1"
 edition.workspace = true
 authors = ["Bas Zalmstra <zalmstra.bas@gmail.com>"]
 description = "Helper functions to work with file:// urls"
@@ -13,7 +13,7 @@ url = { workspace = true }
 percent-encoding = { workspace = true }
 itertools = { workspace = true }
 typed-path = { workspace = true }
-thiserror = "1.0.66"
+thiserror = "2.0.7"
 
 [dev-dependencies]
 rstest = { workspace = true }

crates/rattler-bin/Cargo.toml

@@ -27,13 +27,13 @@ clap = { workspace = true, features = ["derive"] }
 console = { workspace = true, features = ["windows-console-colors"] }
 indicatif = { workspace = true }
 once_cell = { workspace = true }
-rattler = { path="../rattler", version = "0.28.6", default-features = false, features = ["indicatif"] }
-rattler_conda_types = { path="../rattler_conda_types", version = "0.29.4", default-features = false }
-rattler_networking = { path="../rattler_networking", version = "0.21.8", default-features = false, features = ["gcs"] }
-rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.21.26", default-features = false, features = ["gateway"] }
-rattler_solve = { path="../rattler_solve", version = "1.2.5", default-features = false, features = ["resolvo", "libsolv_c"] }
-rattler_virtual_packages = { path="../rattler_virtual_packages", version = "1.1.12", default-features = false }
-rattler_cache = { path="../rattler_cache", version = "0.2.14", default-features = false }
+rattler = { path="../rattler", version = "0.28.8", default-features = false, features = ["indicatif"] }
+rattler_conda_types = { path="../rattler_conda_types", version = "0.29.6", default-features = false }
+rattler_networking = { path="../rattler_networking", version = "0.21.9", default-features = false, features = ["gcs"] }
+rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.21.28", default-features = false, features = ["gateway"] }
+rattler_solve = { path="../rattler_solve", version = "1.3.0", default-features = false, features = ["resolvo", "libsolv_c"] }
+rattler_virtual_packages = { path="../rattler_virtual_packages", version = "1.1.14", default-features = false }
+rattler_cache = { path="../rattler_cache", version = "0.3.0", default-features = false }
 reqwest = { workspace = true }
 reqwest-middleware = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

crates/rattler-bin/src/commands/create.rs

@@ -27,7 +27,7 @@ use rattler_solve::{
     libsolv_c::{self},
     resolvo, SolverImpl, SolverTask,
 };
-use reqwest::Client;
+use reqwest::{Client, Url};
 
 use crate::global_multi_progress;
 
@@ -136,7 +136,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         .collect::<Result<Vec<_>, _>>()?;
 
     // Determine the packages that are currently installed in the environment.
-    let installed_packages = PrefixRecord::collect_from_prefix(&target_prefix)?;
+    let installed_packages = PrefixRecord::collect_from_prefix::<PrefixRecord>(&target_prefix)?;
 
     // For each channel/subdirectory combination, download and cache the
     // `repodata.json` that should be available from the corresponding Url. The
@@ -165,11 +165,16 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         ))
         .with_client(download_client.clone())
         .with_channel_config(rattler_repodata_gateway::ChannelConfig {
-            default: SourceConfig {
-                sharded_enabled: false,
-                ..SourceConfig::default()
-            },
-            ..rattler_repodata_gateway::ChannelConfig::default()
+            default: SourceConfig::default(),
+            per_channel: [(
+                Url::parse("https://prefix.dev")?,
+                SourceConfig {
+                    sharded_enabled: true,
+                    ..SourceConfig::default()
+                },
+            )]
+            .into_iter()
+            .collect(),
         })
         .finish();
 

crates/rattler/CHANGELOG.md

@@ -6,6 +6,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.28.8](https://github.com/conda/rattler/compare/rattler-v0.28.7...rattler-v0.28.8) - 2024-12-17
+
+### Added
+
+- speed up `PrefixRecord` loading (#984)
+- improve performance when linking files using `rayon` (#985)
+
+## [0.28.7](https://github.com/conda/rattler/compare/rattler-v0.28.6...rattler-v0.28.7) - 2024-12-13
+
+### Fixed
+
+- check reflink support before linking (#979)
+
 ## [0.28.6](https://github.com/conda/rattler/compare/rattler-v0.28.5...rattler-v0.28.6) - 2024-12-12
 
 ### Other

crates/rattler/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rattler"
-version = "0.28.6"
+version = "0.28.8"
 edition.workspace = true
 authors = ["Bas Zalmstra <zalmstra.bas@gmail.com>"]
 description = "Rust library to install conda environments"
@@ -32,13 +32,14 @@ memchr = { workspace = true }
 memmap2 = { workspace = true }
 once_cell = { workspace = true }
 parking_lot = { workspace = true }
-rattler_cache = { path = "../rattler_cache", version = "0.2.14", default-features = false }
-rattler_conda_types = { path = "../rattler_conda_types", version = "0.29.4", default-features = false }
 rattler_menuinst = { path = "../rattler_menuinst", version = "0.1.0", default-features = false }
-rattler_digest = { path = "../rattler_digest", version = "1.0.3", default-features = false }
-rattler_networking = { path = "../rattler_networking", version = "0.21.8", default-features = false }
-rattler_shell = { path = "../rattler_shell", version = "0.22.9", default-features = false }
-rattler_package_streaming = { path = "../rattler_package_streaming", version = "0.22.17", default-features = false, features = ["reqwest"] }
+rattler_cache = { path = "../rattler_cache", version = "0.3.0", default-features = false }
+rattler_conda_types = { path = "../rattler_conda_types", version = "0.29.6", default-features = false }
+rattler_digest = { path = "../rattler_digest", version = "1.0.4", default-features = false }
+rattler_networking = { path = "../rattler_networking", version = "0.21.9", default-features = false }
+rattler_shell = { path = "../rattler_shell", version = "0.22.11", default-features = false }
+rattler_package_streaming = { path = "../rattler_package_streaming", version = "0.22.19", default-features = false, features = ["reqwest"] }
+rayon = { workspace = true }
 reflink-copy = { workspace = true }
 regex = { workspace = true }
 reqwest = { workspace = true, features = ["stream", "json", "gzip"] }

crates/rattler/src/install/clobber_registry.rs

@@ -696,7 +696,8 @@ mod tests {
                 ],
             );
 
-            let prefix_records = PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
+            let prefix_records: Vec<PrefixRecord> =
+                PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
 
             for record in prefix_records {
                 if record.repodata_record.package_record.name.as_normalized() == "clobber-1" {
@@ -878,7 +879,8 @@ mod tests {
 
         println!("== RUNNING UPDATE");
 
-        let mut prefix_records = PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
+        let mut prefix_records: Vec<PrefixRecord> =
+            PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
         prefix_records.sort_by(|a, b| {
             a.repodata_record
                 .package_record
@@ -974,7 +976,8 @@ mod tests {
             &["clobber.txt", "another-clobber.txt"],
         );
 
-        let mut prefix_records = PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
+        let mut prefix_records: Vec<PrefixRecord> =
+            PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
         prefix_records.sort_by(|a, b| {
             a.repodata_record
                 .package_record
@@ -1063,7 +1066,8 @@ mod tests {
             ],
         );
 
-        let mut prefix_records = PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
+        let mut prefix_records: Vec<PrefixRecord> =
+            PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
         prefix_records.sort_by(|a, b| {
             a.repodata_record
                 .package_record
@@ -1232,7 +1236,8 @@ mod tests {
         )
         .await;
 
-        let prefix_records = PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
+        let prefix_records: Vec<PrefixRecord> =
+            PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
 
         // remove one of the clobbering files
         let transaction = transaction::Transaction::<PrefixRecord, RepoDataRecord> {
@@ -1299,7 +1304,8 @@ mod tests {
         )
         .await;
 
-        let prefix_records = PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
+        let prefix_records: Vec<PrefixRecord> =
+            PrefixRecord::collect_from_prefix(target_prefix.path()).unwrap();
 
         // remove one of the clobbering files
         let transaction = transaction::Transaction::<PrefixRecord, RepoDataRecord> {

crates/rattler/src/install/driver.rs

@@ -30,7 +30,7 @@ use crate::install::link_script::LinkScriptError;
 /// system has available.
 pub struct InstallDriver {
     io_concurrency_semaphore: Option<Arc<Semaphore>>,
-    clobber_registry: Arc<Mutex<ClobberRegistry>>,
+    pub(crate) clobber_registry: Arc<Mutex<ClobberRegistry>>,
     execute_link_scripts: bool,
 }
 

crates/rattler/src/install/installer/indicatif.rs

@@ -375,8 +375,8 @@ impl<F: ProgressFormatter> IndicatifReporterInner<F> {
             .iter()
             .map(|&idx| (self.package_sizes[idx], &self.package_names[idx]));
 
-        let largest_package = package_iter.max_by_key(|(size, _)| *size);
-        if let Some((_, first)) = largest_package {
+        let smallest_package = package_iter.min_by_key(|(size, _)| *size);
+        if let Some((_, first)) = smallest_package {
             msg.push_str(first);
         }