ledger-tool: verify: add --verify-slots and --verify-slots-details

This adds:

    --record-slots <FILENAME>
	Write the slot hashes to this file.

    --record-slots-config hash-only|bank
	Store the bank (=accounts) json file, or not.

    --verify-slots <FILENAME>
        Verify slot hashes against this file.

The first case can be used to dump a list of (slot, hash) to a json file
during a replay. The second case can be used to check slot hashes against
previously recorded values.

This is useful for debugging consensus failures, eg:

    # on good commit/branch
    ledger-tool verify --record-slots good.json --record-slots-config=bank

    # on bad commit or potentially consensus breaking branch
    ledger-tool verify --verify-slots good.json

On a hash mismatch an error will be logged with the expected hash vs the
computed hash.

ledger-tool/src/main.rs

@@ -41,6 +41,7 @@ use {
     solana_ledger::{
         blockstore::{create_new_ledger, Blockstore},
         blockstore_options::{AccessType, LedgerColumnOptions},
+        blockstore_processor::ProcessSlotCallback,
         use_snapshot_archives_at_startup,
     },
     solana_measure::{measure, measure::Measure},
@@ -88,7 +89,7 @@ use {
         str::FromStr,
         sync::{
             atomic::{AtomicBool, Ordering},
-            Arc, RwLock,
+            Arc, Mutex, RwLock,
         },
     },
 };
@@ -1060,6 +1061,31 @@ fn main() {
                              information that went into computing the completed bank's bank hash. \
                              The file will be written within <LEDGER_DIR>/bank_hash_details/",
                         ),
+                )
+                .arg(
+                    Arg::with_name("record_slots")
+                        .long("record-slots")
+                        .default_value("slots.json")
+                        // .default_value() does not work with .conflicts_with() in clap 2.33
+                        // .conflicts_with("verify_slots")
+                        // https://github.com/clap-rs/clap/issues/1605#issuecomment-722326915
+                        .value_name("FILENAME")
+                        .help("Record slots to a file"),
+                )
+                .arg(
+                    Arg::with_name("verify_slots")
+                        .long("verify-slots")
+                        .default_value("slots.json")
+                        .value_name("FILENAME")
+                        .help("Verify slots match contents of file"),
+                )
+                .arg(
+                    Arg::with_name("record_slots_config")
+                        .long("record-slots-config")
+                        .default_value("hash-only")
+                        .possible_values(&["hash-only", "accounts"])
+                        .requires("record_slots")
+                        .help("In the slot recording, include bank details or not"),
                 ),
         )
         .subcommand(
@@ -1621,7 +1647,105 @@ fn main() {
                         },
                     );
 
-                    let process_options = parse_process_options(&ledger_path, arg_matches);
+                    let mut process_options = parse_process_options(&ledger_path, arg_matches);
+
+                    let (slot_callback, record_slots_file, recorded_slots) = if arg_matches
+                        .occurrences_of("record_slots")
+                        > 0
+                    {
+                        if arg_matches.occurrences_of("verify_slots") > 0 {
+                            panic!(
+                                "--verify-slots and --record-slots cannot be used at the same time"
+                            )
+                        }
+
+                        let filename = Path::new(arg_matches.value_of_os("record_slots").unwrap());
+
+                        let file = File::create(filename).unwrap_or_else(|err| {
+                            eprintln!("Unable to write to file: {}: {:#}", filename.display(), err);
+                            exit(1);
+                        });
+
+                        let include_bank =
+                            match arg_matches.value_of("record_slots_config").unwrap() {
+                                "hash-only" => false,
+                                "accounts" => true,
+                                _ => unreachable!(),
+                            };
+
+                        let slot_hashes = Arc::new(Mutex::new(Vec::new()));
+
+                        let slot_callback = Arc::new({
+                            let slots = Arc::clone(&slot_hashes);
+                            move |bank: &Bank| {
+                                let slot_details = if include_bank {
+                                    bank_hash_details::BankHashSlotDetails::try_from(bank).unwrap()
+                                } else {
+                                    bank_hash_details::BankHashSlotDetails {
+                                        slot: bank.slot(),
+                                        bank_hash: bank.hash().to_string(),
+                                        ..Default::default()
+                                    }
+                                };
+
+                                slots.lock().unwrap().push(slot_details);
+                            }
+                        });
+
+                        (
+                            Some(slot_callback as ProcessSlotCallback),
+                            Some(file),
+                            Some(slot_hashes),
+                        )
+                    } else if arg_matches.occurrences_of("verify_slots") > 0 {
+                        if arg_matches.occurrences_of("record_slots") > 0 {
+                            panic!(
+                                "--verify-slots and --record-slots cannot be used at the same time"
+                            )
+                        }
+
+                        let filename = Path::new(arg_matches.value_of_os("verify_slots").unwrap());
+
+                        let file = File::open(filename).unwrap_or_else(|err| {
+                            eprintln!("Unable to read file: {}: {err:#}", filename.display());
+                            exit(1);
+                        });
+
+                        let reader = std::io::BufReader::new(file);
+
+                        let details: bank_hash_details::BankHashDetails =
+                            serde_json::from_reader(reader).unwrap_or_else(|err| {
+                                eprintln!("Error loading slots file: {err:#}");
+                                exit(1);
+                            });
+
+                        let slots = Arc::new(Mutex::new(details.bank_hash_details));
+
+                        let slot_callback = Arc::new(move |bank: &Bank| {
+                            let bank_hash_details::BankHashSlotDetails {
+                                slot: expected_slot,
+                                bank_hash: expected_hash,
+                                ..
+                            } = slots.lock().unwrap().remove(0);
+                            if bank.slot() != expected_slot
+                                || bank.hash().to_string() != expected_hash
+                            {
+                                error!("Expected slot: {expected_slot} hash: {expected_hash} got slot: {} hash: {}",
+                                bank.slot(), bank.hash());
+                            } else {
+                                info!(
+                                    "Expected slot: {expected_slot} hash: {expected_hash} correct"
+                                );
+                            }
+                        });
+
+                        (Some(slot_callback as ProcessSlotCallback), None, None)
+                    } else {
+                        (None, None, None)
+                    };
+
+                    process_options.slot_callback = slot_callback;
+
                     let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
                     let write_bank_file = arg_matches.is_present("write_bank_file");
                     let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
@@ -1653,6 +1777,21 @@ fn main() {
                             })
                             .ok();
                     }
+
+                    if let Some(recorded_slots_file) = record_slots_file {
+                        if let Ok(recorded_slots) = recorded_slots.clone().unwrap().lock() {
+                            let bank_hashes =
+                                bank_hash_details::BankHashDetails::new(recorded_slots.to_vec());
+
+                            // writing the json file ends up with a syscall for each number, comma, indentation etc.
+                            // use BufWriter to speed things up
+
+                            let writer = std::io::BufWriter::new(recorded_slots_file);
+
+                            serde_json::to_writer_pretty(writer, &bank_hashes).unwrap();
+                        }
+                    }
+
                     exit_signal.store(true, Ordering::Relaxed);
                     system_monitor_service.join().unwrap();
                 }

ledger/src/blockstore_processor.rs

@@ -676,15 +676,17 @@ pub enum BlockstoreProcessorError {
     RootBankWithMismatchedCapitalization(Slot),
 }
 
-/// Callback for accessing bank state while processing the blockstore
-pub type ProcessCallback = Arc<dyn Fn(&Bank) + Sync + Send>;
+/// Callback for accessing bank state after each slot is confirmed while
+/// processing the blockstore
+pub type ProcessSlotCallback = Arc<dyn Fn(&Bank) + Sync + Send>;
 
 #[derive(Default, Clone)]
 pub struct ProcessOptions {
     /// Run PoH, transaction signature and other transaction verifications on the entries.
     pub run_verification: bool,
     pub full_leader_cache: bool,
     pub halt_at_slot: Option<Slot>,
+    pub slot_callback: Option<ProcessSlotCallback>,
     pub new_hard_forks: Option<Vec<Slot>>,
     pub debug_keys: Option<Arc<HashSet<Pubkey>>>,
     pub account_indexes: AccountSecondaryIndexes,
@@ -1810,6 +1812,11 @@ fn process_single_slot(
         result?
     }
     bank.freeze(); // all banks handled by this routine are created from complete slots
+
+    if let Some(slot_callback) = &opts.slot_callback {
+        slot_callback(bank);
+    }
+
     if blockstore.is_primary_access() {
         blockstore.insert_bank_hash(bank.slot(), bank.hash(), false);
     }

runtime/src/bank/bank_hash_details.rs

@@ -22,7 +22,7 @@ use {
 };
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-pub(crate) struct BankHashDetails {
+pub struct BankHashDetails {
     /// The client version
     pub version: String,
     /// The encoding format for account data buffers
@@ -66,17 +66,35 @@ impl BankHashDetails {
 }
 
 /// The components that go into a bank hash calculation for a single bank/slot.
-#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-pub(crate) struct BankHashSlotDetails {
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, Default)]
+pub struct BankHashSlotDetails {
     pub slot: Slot,
     pub bank_hash: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
     pub parent_bank_hash: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
     pub accounts_delta_hash: String,
+    #[serde(skip_serializing_if = "u64_is_zero")]
+    #[serde(default)]
     pub signature_count: u64,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
     pub last_blockhash: String,
+    #[serde(skip_serializing_if = "bankhashaccounts_is_empty")]
+    #[serde(default)]
     pub accounts: BankHashAccounts,
 }
 
+fn u64_is_zero(val: &u64) -> bool {
+    *val == 0
+}
+
+fn bankhashaccounts_is_empty(accounts: &BankHashAccounts) -> bool {
+    accounts.accounts.is_empty()
+}
+
 impl BankHashSlotDetails {
     pub fn new(
         slot: Slot,
@@ -141,8 +159,8 @@ impl TryFrom<&Bank> for BankHashSlotDetails {
 
 /// Wrapper around a Vec<_> to facilitate custom Serialize/Deserialize trait
 /// implementations.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub(crate) struct BankHashAccounts {
+#[derive(Clone, Debug, Eq, PartialEq, Default)]
+pub struct BankHashAccounts {
     pub accounts: Vec<PubkeyHashAccount>,
 }
 
@@ -257,7 +275,12 @@ pub fn write_bank_hash_details_file(bank: &Bank) -> std::result::Result<(), Stri
         _ = std::fs::create_dir_all(parent_dir);
         let file = std::fs::File::create(&path)
             .map_err(|err| format!("Unable to create file at {}: {err}", path.display()))?;
-        serde_json::to_writer_pretty(file, &details)
+
+        // writing the json file ends up with a syscall for each number, comma, indentation etc.
+        // use BufWriter to speed things up
+        let writer = std::io::BufWriter::new(file);
+
+        serde_json::to_writer_pretty(writer, &details)
             .map_err(|err| format!("Unable to write file at {}: {err}", path.display()))?;
     }
     Ok(())