From 58e0d7c60df9dcd2e90b1b2a764d96673c20cbed Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Sat, 9 Nov 2024 05:19:16 +0200 Subject: [PATCH] Update concurrency parameters --- .../commands/cluster/farmer.rs | 7 +++--- .../commands/cluster/plotter.rs | 10 ++++---- .../src/bin/subspace-farmer/commands/farm.rs | 23 ++++++++++--------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs index f2179e71a8..506c864583 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs @@ -100,12 +100,13 @@ pub(super) struct FarmerArgs { farming_thread_pool_size: Option, /// How many sectors a will be plotted concurrently per farm. /// - /// Defaults to 4, but can be decreased if there is a large number of farms available to - /// decrease peak memory usage, especially with slow disks. + /// Defaults to 2, but can be decreased if there is a large number of farms available to + /// decrease peak memory usage, especially with slow disks, or slightly increased to utilize all + /// compute available in case of a single farm. /// /// Increasing this value is not recommended and can result in excessive RAM usage due to more /// sectors being stuck in-flight if writes to farm disk are too slow. - #[arg(long, default_value = "4")] + #[arg(long, default_value = "2")] max_plotting_sectors_per_farm: NonZeroUsize, /// Disable farm locking, for example if file system doesn't support it #[arg(long)] diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs index 3dfc065636..26927d8262 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs @@ -88,7 +88,7 @@ struct CpuPlottingOptions { #[derive(Debug, Parser)] struct CudaPlottingOptions { /// How many sectors farmer will download concurrently during plotting with CUDA GPUs. - /// Limits memory usage of the plotting process. Defaults to the number of CUDA GPUs + 1, + /// Limits memory usage of the plotting process. Defaults to the number of CUDA GPUs * 3, /// to download future sectors ahead of time. /// /// Increasing this value will cause higher memory usage. @@ -105,8 +105,8 @@ struct CudaPlottingOptions { #[cfg(feature = "rocm")] #[derive(Debug, Parser)] struct RocmPlottingOptions { - /// How many sectors the farmer will download concurrently during plotting with ROCm GPUs. - /// Limits memory usage of the plotting process. Defaults to number of ROCm GPUs + 1, + /// How many sectors farmer will download concurrently during plotting with ROCm GPUs. + /// Limits memory usage of the plotting process. Defaults to the number of ROCm GPUs * 3, /// to download future sectors ahead of time. /// /// Increasing this value will cause higher memory usage. @@ -388,7 +388,7 @@ where let cuda_downloading_semaphore = Arc::new(Semaphore::new( cuda_sector_downloading_concurrency .map(|cuda_sector_downloading_concurrency| cuda_sector_downloading_concurrency.get()) - .unwrap_or(cuda_devices.len() + 1), + .unwrap_or(cuda_devices.len() * 3), )); Ok(Some( @@ -470,7 +470,7 @@ where let rocm_downloading_semaphore = Arc::new(Semaphore::new( rocm_sector_downloading_concurrency .map(|rocm_sector_downloading_concurrency| rocm_sector_downloading_concurrency.get()) - .unwrap_or(rocm_devices.len() + 1), + .unwrap_or(rocm_devices.len() * 3), )); Ok(Some( diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs index e7900b6188..a9da959518 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs @@ -145,9 +145,9 @@ struct CpuPlottingOptions { #[cfg(feature = "cuda")] #[derive(Debug, Parser)] struct CudaPlottingOptions { - /// How many sectors a farmer will download concurrently during plotting with a CUDA GPU. - /// Limits memory usage of the plotting process. Defaults to the number of CUDA GPUs found - /// + 1 to download future sector ahead of time. + /// How many sectors farmer will download concurrently during plotting with CUDA GPUs. + /// Limits memory usage of the plotting process. Defaults to the number of CUDA GPUs * 3, + /// to download future sectors ahead of time. /// /// Increasing this value will cause higher memory usage. #[arg(long)] @@ -163,9 +163,9 @@ struct CudaPlottingOptions { #[cfg(feature = "rocm")] #[derive(Debug, Parser)] struct RocmPlottingOptions { - /// How many sectors a farmer will download concurrently during plotting with a ROCm GPU. - /// Limits memory usage of the plotting process. Defaults to the number of ROCm GPUs found - /// + 1 to download future sector ahead of time. + /// How many sectors farmer will download concurrently during plotting with ROCm GPUs. + /// Limits memory usage of the plotting process. Defaults to the number of ROCm GPUs * 3, + /// to download future sectors ahead of time. /// /// Increasing this value will cause higher memory usage. #[arg(long)] @@ -253,12 +253,13 @@ pub(crate) struct FarmingArgs { rocm_plotting_options: RocmPlottingOptions, /// How many sectors a will be plotted concurrently per farm. /// - /// Defaults to 4, but can be decreased if there is a large number of farms available to - /// decrease peak memory usage, especially with slow disks. + /// Defaults to 2, but can be decreased if there is a large number of farms available to + /// decrease peak memory usage, especially with slow disks, or slightly increased to utilize all + /// compute available in case of a single farm. /// /// Increasing this value is not recommended and can result in excessive RAM usage due to more /// sectors being stuck in-flight if writes to farm disk are too slow. - #[arg(long, default_value = "4")] + #[arg(long, default_value = "2")] max_plotting_sectors_per_farm: NonZeroUsize, /// Enable plot cache. /// @@ -961,7 +962,7 @@ where let cpu_downloading_semaphore = Arc::new(Semaphore::new( cpu_sector_downloading_concurrency .map(|cpu_sector_downloading_concurrency| cpu_sector_downloading_concurrency.get()) - .unwrap_or(plotting_thread_pool_core_indices.len() + 1), + .unwrap_or(plotting_thread_pool_core_indices.len() * 3), )); let cpu_record_encoding_concurrency = cpu_record_encoding_concurrency.unwrap_or_else(|| { @@ -1059,7 +1060,7 @@ where let cuda_downloading_semaphore = Arc::new(Semaphore::new( cuda_sector_downloading_concurrency .map(|cuda_sector_downloading_concurrency| cuda_sector_downloading_concurrency.get()) - .unwrap_or(cuda_devices.len() + 1), + .unwrap_or(cuda_devices.len() * 3), )); Ok(Some(