diff --git a/core/src/validator.rs b/core/src/validator.rs index f41ff1a554fdc2..11c2f2f2d8bc77 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -108,8 +108,10 @@ use { solana_vote_program::vote_state, std::{ collections::{HashMap, HashSet}, + fs, net::SocketAddr, path::{Path, PathBuf}, + process, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, Arc, RwLock, @@ -2182,6 +2184,17 @@ fn get_stake_percent_in_gossip(bank: &Bank, cluster_info: &ClusterInfo, log: boo fn cleanup_accounts_paths(config: &ValidatorConfig) { for accounts_path in &config.account_paths { move_and_async_delete_path(accounts_path); + if !accounts_path.exists() { + // Keep the empty directory + fs::create_dir_all(accounts_path).unwrap_or_else(|err| { + eprintln!( + "Failed to create accounts directory {}: {}", + accounts_path.display(), + err + ); + process::exit(1); + }); + } } if let Some(ref shrink_paths) = config.account_shrink_paths { for accounts_path in shrink_paths { diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 896a8baad9cac7..e38a3e45216a51 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -15,7 +15,7 @@ use { snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_config::SnapshotConfig, snapshot_hash::{FullSnapshotHash, IncrementalSnapshotHash, StartingSnapshotHashes}, - snapshot_utils, + snapshot_utils::{self, BankSnapshotType, SnapshotFrom}, }, solana_sdk::genesis_config::GenesisConfig, std::{ @@ -97,21 +97,27 @@ pub fn load_bank_forks( "Initializing bank snapshot path: {}", snapshot_config.bank_snapshots_dir.display() ); - fs::create_dir_all(&snapshot_config.bank_snapshots_dir) - .expect("Couldn't create snapshot directory"); - if snapshot_utils::get_highest_full_snapshot_archive_info( - &snapshot_config.full_snapshot_archives_dir, - ) - .is_some() - { + if snapshot_config.snapshot_from == SnapshotFrom::Dir { true } else { - warn!( - "No snapshot package found in directory: {:?}; will load from genesis", - &snapshot_config.full_snapshot_archives_dir - ); - false + let _ = fs::remove_dir_all(&snapshot_config.bank_snapshots_dir); + fs::create_dir_all(&snapshot_config.bank_snapshots_dir) + .expect("Couldn't create snapshot directory"); + + if snapshot_utils::get_highest_full_snapshot_archive_info( + &snapshot_config.full_snapshot_archives_dir, + ) + .is_some() + { + true + } else { + warn!( + "No snapshot package found in directory: {:?}; will load from genesis", + &snapshot_config.full_snapshot_archives_dir + ); + false + } } } else { info!("Snapshots disabled; will load from genesis"); @@ -198,10 +204,92 @@ fn bank_forks_from_snapshot( process::exit(1); } - // Given that we are going to boot from an archive, the accountvecs held in the snapshot dirs for fast-boot should - // be released. They will be released by the account_background_service anyway. But in the case of the account_paths - // using memory-mounted file system, they are not released early enough to give space for the new append-vecs from - // the archives, causing the out-of-memory problem. So, purge the snapshot dirs upfront before loading from the archive. + // In both the SnapshotFrom::Dir and SnapshotFrom::ARCHIVE cases, the unneeded snapshot dirs left by the + // previous validator processes are purged here upfront before booting and constructing a new bank. This frees + // up the file system space, giving room to the new accounts files. + // The new snapshot dirs generated by the current validator process will be purged separately in background by + // the account_background_service (ABS). + + // In the case of the memory-mounted account_paths and booting from an archive, purging the snapshot dirs upfront + // is critical to have enough memory space for the new append-vecs from the archive. + + if snapshot_config.snapshot_from == SnapshotFrom::Dir { + // Purge all the PRE snapshot dirs, and all the POST ones except the highest one. + snapshot_utils::purge_old_bank_snapshots( + &snapshot_config.bank_snapshots_dir, + 0, + Some(BankSnapshotType::Pre), + ); + snapshot_utils::purge_old_bank_snapshots( + &snapshot_config.bank_snapshots_dir, + 1, + Some(BankSnapshotType::Post), + ); + + match snapshot_utils::bank_from_latest_snapshot_dir( + &snapshot_config.bank_snapshots_dir, + genesis_config, + &process_options.runtime_config, + &account_paths, + process_options.debug_keys.clone(), + None, + process_options.account_indexes.clone(), + process_options.limit_load_slot_count_from_snapshot, + process_options.shrink_ratio, + process_options.verify_index, + process_options.accounts_db_config.clone(), + accounts_update_notifier.clone(), + exit, + ) { + Ok(bank) => { + if let Some(ref shrink_paths) = shrink_paths { + bank.set_shrink_paths(shrink_paths.to_vec()); + } + + if let Some(full_snapshot_archive_info) = + snapshot_utils::get_highest_full_snapshot_archive_info( + &snapshot_config.full_snapshot_archives_dir, + ) + { + let full_snapshot_hash = FullSnapshotHash(( + full_snapshot_archive_info.slot(), + *full_snapshot_archive_info.hash(), + )); + let incremental_snapshot_hash = + snapshot_utils::get_highest_incremental_snapshot_archive_info( + &snapshot_config.incremental_snapshot_archives_dir, + full_snapshot_archive_info.slot(), + ) + .map(|incremental_snapshot_archive_info| { + IncrementalSnapshotHash(( + full_snapshot_archive_info.slot(), + *incremental_snapshot_archive_info.hash(), + )) + }); + let starting_snapshot_hashes = StartingSnapshotHashes { + full: full_snapshot_hash, + incremental: incremental_snapshot_hash, + }; + let bank_forks = BankForks::new(bank); + return ( + Arc::new(RwLock::new(bank_forks)), + Some(starting_snapshot_hashes), + ); + } else { + info!( + "No snapshot archive found in directory: {:?}", + &snapshot_config.full_snapshot_archives_dir + ); + } + } + Err(err) => { + error!("Failed to load bank from snapshot dir: {:?}", err); + } + } + info!("Falling back to loading from snapshot archive"); + } + + // Purge all the snapshot dirs. snapshot_utils::purge_old_bank_snapshots(&snapshot_config.bank_snapshots_dir, 0, None); let (deserialized_bank, full_snapshot_archive_info, incremental_snapshot_archive_info) = diff --git a/runtime/src/snapshot_config.rs b/runtime/src/snapshot_config.rs index 8849cf795ca9e9..19bb309b022119 100644 --- a/runtime/src/snapshot_config.rs +++ b/runtime/src/snapshot_config.rs @@ -1,5 +1,5 @@ use { - crate::snapshot_utils::{self, ArchiveFormat, SnapshotVersion}, + crate::snapshot_utils::{self, ArchiveFormat, SnapshotFrom, SnapshotVersion}, solana_sdk::clock::Slot, std::{num::NonZeroUsize, path::PathBuf}, }; @@ -43,6 +43,9 @@ pub struct SnapshotConfig { // Thread niceness adjustment for snapshot packager service pub packager_thread_niceness_adj: i8, + + /// The snapshot source + pub snapshot_from: SnapshotFrom, } impl Default for SnapshotConfig { @@ -64,6 +67,7 @@ impl Default for SnapshotConfig { snapshot_utils::DEFAULT_MAX_INCREMENTAL_SNAPSHOT_ARCHIVES_TO_RETAIN, accounts_hash_debug_verify: false, packager_thread_niceness_adj: 0, + snapshot_from: SnapshotFrom::Archive, } } } diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index bd40a4e5b11a25..fa729d3ca421c0 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -1758,11 +1758,12 @@ pub fn bank_from_latest_snapshot_dir( accounts_update_notifier: Option, exit: &Arc, ) -> Result { - info!("Loading bank from snapshot dir"); let bank_snapshot = get_highest_bank_snapshot_post(&bank_snapshots_dir).ok_or_else(|| { SnapshotError::NoSnapshotSlotDir(bank_snapshots_dir.as_ref().to_path_buf()) })?; + info!("Loading bank from snapshot dir {:?}", bank_snapshot.slot); + let (bank, timings) = bank_from_snapshot_dir( account_paths, &bank_snapshot, diff --git a/validator/src/cli.rs b/validator/src/cli.rs index a7a12324a03ab0..078bd7d7a503a2 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -299,6 +299,15 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .takes_value(true) .help("Use DIR as separate location for incremental snapshot archives [default: --snapshots value]"), ) + .arg( + Arg::with_name("boot_from_local_state") + .long("boot-from-local-state") + .takes_value(false) + // Hide it because it is for internal usage only at this point. Once this path has been verified via + // internal usage, this argument will be removed, and this path will be set as the default. + .hidden(hidden_unless_forced()) + .help("At boot, reuses account storage files already on disk instead of extracting them from a snapshot archive"), + ) .arg( Arg::with_name("tower") .long("tower") diff --git a/validator/src/main.rs b/validator/src/main.rs index 4603b2d472e6da..df9421dbaecdde 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -42,9 +42,7 @@ use { }, runtime_config::RuntimeConfig, snapshot_config::{SnapshotConfig, SnapshotUsage}, - snapshot_utils::{ - self, create_all_accounts_run_and_snapshot_dirs, ArchiveFormat, SnapshotVersion, - }, + snapshot_utils::{self, ArchiveFormat, SnapshotFrom, SnapshotVersion}, }, solana_sdk::{ clock::{Slot, DEFAULT_S_PER_SLOT}, @@ -1401,10 +1399,12 @@ pub fn main() { }); let (account_run_paths, account_snapshot_paths) = - create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap_or_else(|err| { - eprintln!("Error: {err:?}"); - exit(1); - }); + snapshot_utils::create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap_or_else( + |err| { + eprintln!("Error: {err:?}"); + exit(1); + }, + ); // From now on, use run/ paths in the same way as the previous account_paths. validator_config.account_paths = account_run_paths; @@ -1509,6 +1509,11 @@ pub fn main() { (Slot::MAX, Slot::MAX) }; + let snapshot_from = match matches.is_present("boot_from_local_state") { + true => SnapshotFrom::Dir, + false => SnapshotFrom::Archive, + }; + validator_config.snapshot_config = SnapshotConfig { usage: if full_snapshot_archive_interval_slots == Slot::MAX { SnapshotUsage::LoadOnly @@ -1526,6 +1531,7 @@ pub fn main() { maximum_incremental_snapshot_archives_to_retain, accounts_hash_debug_verify: validator_config.accounts_db_test_hash_calculation, packager_thread_niceness_adj: snapshot_packager_niceness_adj, + snapshot_from, }; validator_config.accounts_hash_interval_slots =