[TieredStorage] Make HotStorageReader use AccountOffset type (solana-…

…labs#33964)

[TieredStorage] Improve param naming of IndexBlockFormat (solana-labs#34033)
[TieredStorage] HotStorageReader::get_account_offset (solana-labs#34031)
[TieredStorage] Rename owners_offset to owners_block_offset (solana-labs#34047)
[TieredStorage] HotStorageReader::get_account_address (solana-labs#34032)
[TieredStorage] OwnersBlock (solana-labs#34052)
[TieredStorage] HotStorageReader::get_owner_address (solana-labs#34053)
[TieredStorage] Define OwnerOffset as u32 (solana-labs#34105)
[TieredStorage] Use OwnerOffset type in TieredAccountMeta (solana-labs#34106)
Refactors TieredStorageFile read/write methods (solana-labs#34147)
[TieredStorage] Make IndexBlock persist u32 offsets (solana-labs#34133)
[TieredStorage] Make IndexOffset use u32 (solana-labs#34152)
Move MatchAccountOwnerError from append_vec to accounts_file (solana-labs#34187)
[TieredStorage] Make AccountOffset use u32 (solana-labs#34151)
[TieredStorage] Allow HotStorage to handle more account data (solana-labs#34155)
[TieredStorage] Make AccountOffset a trait, introduce HotAccountOffset (solana-labs#34335)
[TieredStorage]  Improve comments for HOT_ACCOUNT_ALIGNMENT (solana-labs#34404)
[TieredStorage] Unit-tests for checking invalid HotAccountOffset (solana-labs#34376)
[TieredStorage] Boundary check for accessing hot account meta (solana-labs#34349)
[TieredStorage] boundary check for get_account_address() (solana-labs#34529)
Sanitizes tiered storage footer after reading from disk (solana-labs#34200)
Adds read/write/get_pod() fns to tiered storage (solana-labs#34415)
Uses consistent error types in tiered storage (solana-labs#34110)
[TieredStorage] Boundary check for get_account_offset() (solana-labs#34531)
[TieredStorage] HotStorageReader::account_matches_owners (solana-labs#34350)
[TieredStorage] Fix typos in index.rs (solana-labs#34546)
[TieredStorage] HotAccountsReader::get_account (solana-labs#34499)
[TieredStorage] Rename AddressAndBlockOffsetOnly to AddressesThenOffsets (solana-labs#34658)
[TieredStorage] HotStorageWriter::new() (solana-labs#34659)
[TieredStorage] Include executable field into AccountMetaFlags (solana-labs#34724)
[TieredStorage] Code refactoring for OwnersBlock (solana-labs#34854)
[TieredStorage] In-memory struct for writing OwnersBlock (solana-labs#34853)
[TieredStorage] writing hot account blocks and index blocks (solana-labs#34828)
[TieredStorage] Use RENT_EXEMPT_RENT_EPOCH in HotStorageWriter (solana-labs#34950)
[TieredStorage] Write owners block for HotAccountStorage (solana-labs#34927)
[TieredStorage] Avoid AccountHash copy in AccountMetaOptionalFields (solana-labs#34969)
[TieredStorage] Correct the HotStorage API for account_matches_owners (solana-labs#34967)
[TS] Add get_account() and account_matches_owner() to TieredStorageReader (solana-labs#34968)
[TieredStorage] Have HotStorageWriter::write_account() return Vec<StoredAccountInfo> (solana-labs#34929)
[TieredStorage] Use IndexOffset in TieredStorageMeta and get_account() (solana-labs#35046)
[TieredStorage] TieredStorageReader:: and HotStorageReader:: accounts() (solana-labs#35031)
[TieredStorage] Enable hot-storage in TieredStorage::write_accounts() (solana-labs#35049)
[TieredStorage] Put commonly used test functions into test_utils.rs (solana-labs#35065)

accounts-db/Cargo.toml

@@ -24,6 +24,7 @@ fnv = { workspace = true }
 fs-err = { workspace = true }
 im = { workspace = true, features = ["rayon", "serde"] }
 index_list = { workspace = true }
+indexmap = { workspace = true }
 itertools = { workspace = true }
 lazy_static = { workspace = true }
 log = { workspace = true }

accounts-db/src/account_storage/meta.rs

@@ -142,7 +142,7 @@ impl<'storage> StoredAccountMeta<'storage> {
     pub fn offset(&self) -> usize {
         match self {
             Self::AppendVec(av) => av.offset(),
-            Self::Hot(hot) => hot.index(),
+            Self::Hot(hot) => hot.index().0 as usize,
         }
     }
 

accounts-db/src/accounts_db.rs

@@ -31,7 +31,7 @@ use {
             AccountStorage, AccountStorageStatus, ShrinkInProgress,
         },
         accounts_cache::{AccountsCache, CachedAccount, SlotCache},
-        accounts_file::{AccountsFile, AccountsFileError},
+        accounts_file::{AccountsFile, AccountsFileError, MatchAccountOwnerError},
         accounts_hash::{
             AccountHash, AccountsDeltaHash, AccountsHash, AccountsHashKind, AccountsHasher,
             CalcAccountsHashConfig, CalculateHashIntermediate, HashStats, IncrementalAccountsHash,
@@ -54,8 +54,7 @@ use {
             get_ancient_append_vec_capacity, is_ancient, AccountsToStore, StorageSelector,
         },
         append_vec::{
-            aligned_stored_size, AppendVec, MatchAccountOwnerError, APPEND_VEC_MMAPPED_FILES_OPEN,
-            STORE_META_OVERHEAD,
+            aligned_stored_size, AppendVec, APPEND_VEC_MMAPPED_FILES_OPEN, STORE_META_OVERHEAD,
         },
         cache_hash_data::{CacheHashData, CacheHashDataFileReference},
         contains::Contains,

accounts-db/src/accounts_file.rs

@@ -4,7 +4,7 @@ use {
             StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredAccountMeta,
         },
         accounts_hash::AccountHash,
-        append_vec::{AppendVec, AppendVecError, MatchAccountOwnerError},
+        append_vec::{AppendVec, AppendVecError},
         storable_accounts::StorableAccounts,
         tiered_storage::error::TieredStorageError,
     },
@@ -40,6 +40,14 @@ pub enum AccountsFileError {
     TieredStorageError(#[from] TieredStorageError),
 }
 
+#[derive(Error, Debug, PartialEq, Eq)]
+pub enum MatchAccountOwnerError {
+    #[error("The account owner does not match with the provided list")]
+    NoMatch,
+    #[error("Unable to load the account")]
+    UnableToLoad,
+}
+
 pub type Result<T> = std::result::Result<T, AccountsFileError>;
 
 #[derive(Debug)]

accounts-db/src/append_vec.rs

@@ -10,7 +10,7 @@ use {
             AccountMeta, StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo,
             StoredAccountMeta, StoredMeta, StoredMetaWriteVersion,
         },
-        accounts_file::{AccountsFileError, Result, ALIGN_BOUNDARY_OFFSET},
+        accounts_file::{AccountsFileError, MatchAccountOwnerError, Result, ALIGN_BOUNDARY_OFFSET},
         accounts_hash::AccountHash,
         storable_accounts::StorableAccounts,
         u64_align,
@@ -96,14 +96,6 @@ impl<'append_vec> Iterator for AppendVecAccountsIter<'append_vec> {
     }
 }
 
-#[derive(Error, Debug, PartialEq, Eq)]
-pub enum MatchAccountOwnerError {
-    #[error("The account owner does not match with the provided list")]
-    NoMatch,
-    #[error("Unable to load the account")]
-    UnableToLoad,
-}
-
 /// References to account data stored elsewhere. Getting an `Account` requires cloning
 /// (see `StoredAccountMeta::clone_account()`).
 #[derive(PartialEq, Eq, Debug)]

accounts-db/src/tiered_storage.rs

@@ -8,7 +8,9 @@ pub mod hot;
 pub mod index;
 pub mod meta;
 pub mod mmap_utils;
+pub mod owners;
 pub mod readable;
+mod test_utils;
 pub mod writer;
 
 use {
@@ -18,24 +20,25 @@ use {
         storable_accounts::StorableAccounts,
     },
     error::TieredStorageError,
-    footer::{AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat},
+    footer::{AccountBlockFormat, AccountMetaFormat},
+    hot::{HotStorageWriter, HOT_FORMAT},
     index::IndexBlockFormat,
+    owners::OwnersBlockFormat,
     readable::TieredStorageReader,
     solana_sdk::account::ReadableAccount,
     std::{
         borrow::Borrow,
-        fs::OpenOptions,
+        fs::{self, OpenOptions},
         path::{Path, PathBuf},
         sync::OnceLock,
     },
-    writer::TieredStorageWriter,
 };
 
 pub type TieredStorageResult<T> = Result<T, TieredStorageError>;
 
 /// The struct that defines the formats of all building blocks of a
 /// TieredStorage.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct TieredStorageFormat {
     pub meta_entry_size: usize,
     pub account_meta_format: AccountMetaFormat,
@@ -52,8 +55,11 @@ pub struct TieredStorage {
 
 impl Drop for TieredStorage {
     fn drop(&mut self) {
-        if let Err(err) = fs_err::remove_file(&self.path) {
-            panic!("TieredStorage failed to remove backing storage file: {err}");
+        if let Err(err) = fs::remove_file(&self.path) {
+            panic!(
+                "TieredStorage failed to remove backing storage file '{}': {err}",
+                self.path.display(),
+            );
         }
     }
 }
@@ -110,19 +116,23 @@ impl TieredStorage {
             ));
         }
 
-        let result = {
-            let writer = TieredStorageWriter::new(&self.path, format)?;
-            writer.write_accounts(accounts, skip)
-        };
+        if format == &HOT_FORMAT {
+            let result = {
+                let writer = HotStorageWriter::new(&self.path)?;
+                writer.write_accounts(accounts, skip)
+            };
 
-        // panic here if self.reader.get() is not None as self.reader can only be
-        // None since we have passed `is_read_only()` check previously, indicating
-        // self.reader is not yet set.
-        self.reader
-            .set(TieredStorageReader::new_from_path(&self.path)?)
-            .unwrap();
+            // panic here if self.reader.get() is not None as self.reader can only be
+            // None since we have passed `is_read_only()` check previously, indicating
+            // self.reader is not yet set.
+            self.reader
+                .set(TieredStorageReader::new_from_path(&self.path)?)
+                .unwrap();
 
-        result
+            return result;
+        }
+
+        Err(TieredStorageError::UnknownFormat(self.path.to_path_buf()))
     }
 
     /// Returns the underlying reader of the TieredStorage.  None will be
@@ -151,19 +161,20 @@ impl TieredStorage {
 mod tests {
     use {
         super::*,
-        crate::account_storage::meta::{StoredMeta, StoredMetaWriteVersion},
+        crate::account_storage::meta::StoredMetaWriteVersion,
         footer::{TieredStorageFooter, TieredStorageMagicNumber},
         hot::HOT_FORMAT,
-        solana_accounts_db::rent_collector::RENT_EXEMPT_RENT_EPOCH,
+        index::IndexOffset,
         solana_sdk::{
-            account::{Account, AccountSharedData},
-            clock::Slot,
-            hash::Hash,
-            pubkey::Pubkey,
+            account::AccountSharedData, clock::Slot, hash::Hash, pubkey::Pubkey,
             system_instruction::MAX_PERMITTED_DATA_LENGTH,
         },
-        std::mem::ManuallyDrop,
+        std::{
+            collections::{HashMap, HashSet},
+            mem::ManuallyDrop,
+        },
         tempfile::tempdir,
+        test_utils::{create_test_account, verify_test_account},
     };
 
     impl TieredStorage {
@@ -196,6 +207,7 @@ mod tests {
                 Err(TieredStorageError::AttemptToUpdateReadOnly(_)),
             ) => {}
             (Err(TieredStorageError::Unsupported()), Err(TieredStorageError::Unsupported())) => {}
+            (Ok(_), Ok(_)) => {}
             // we don't expect error type mis-match or other error types here
             _ => {
                 panic!("actual: {result:?}, expected: {expected_result:?}");
@@ -224,10 +236,7 @@ mod tests {
             assert_eq!(tiered_storage.path(), tiered_storage_path);
             assert_eq!(tiered_storage.file_size().unwrap(), 0);
 
-            // Expect the result to be TieredStorageError::Unsupported as the feature
-            // is not yet fully supported, but we can still check its partial results
-            // in the test.
-            write_zero_accounts(&tiered_storage, Err(TieredStorageError::Unsupported()));
+            write_zero_accounts(&tiered_storage, Ok(vec![]));
         }
 
         let tiered_storage_readonly = TieredStorage::new_readonly(&tiered_storage_path).unwrap();
@@ -252,10 +261,7 @@ mod tests {
         let tiered_storage_path = temp_dir.path().join("test_write_accounts_twice");
 
         let tiered_storage = TieredStorage::new_writable(&tiered_storage_path);
-        // Expect the result to be TieredStorageError::Unsupported as the feature
-        // is not yet fully supported, but we can still check its partial results
-        // in the test.
-        write_zero_accounts(&tiered_storage, Err(TieredStorageError::Unsupported()));
+        write_zero_accounts(&tiered_storage, Ok(vec![]));
         // Expect AttemptToUpdateReadOnly error as write_accounts can only
         // be invoked once.
         write_zero_accounts(
@@ -273,15 +279,15 @@ mod tests {
         let tiered_storage_path = temp_dir.path().join("test_remove_on_drop");
         {
             let tiered_storage = TieredStorage::new_writable(&tiered_storage_path);
-            write_zero_accounts(&tiered_storage, Err(TieredStorageError::Unsupported()));
+            write_zero_accounts(&tiered_storage, Ok(vec![]));
         }
         // expect the file does not exists as it has been removed on drop
         assert!(!tiered_storage_path.try_exists().unwrap());
 
         {
             let tiered_storage =
                 ManuallyDrop::new(TieredStorage::new_writable(&tiered_storage_path));
-            write_zero_accounts(&tiered_storage, Err(TieredStorageError::Unsupported()));
+            write_zero_accounts(&tiered_storage, Ok(vec![]));
         }
         // expect the file exists as we have ManuallyDrop this time.
         assert!(tiered_storage_path.try_exists().unwrap());
@@ -301,29 +307,6 @@ mod tests {
         assert!(!tiered_storage_path.try_exists().unwrap());
     }
 
-    /// Create a test account based on the specified seed.
-    fn create_account(seed: u64) -> (StoredMeta, AccountSharedData) {
-        let data_byte = seed as u8;
-        let account = Account {
-            lamports: seed,
-            data: std::iter::repeat(data_byte).take(seed as usize).collect(),
-            owner: Pubkey::new_unique(),
-            executable: seed % 2 > 0,
-            rent_epoch: if seed % 3 > 0 {
-                seed
-            } else {
-                RENT_EXEMPT_RENT_EPOCH
-            },
-        };
-
-        let stored_meta = StoredMeta {
-            write_version_obsolete: StoredMetaWriteVersion::default(),
-            pubkey: Pubkey::new_unique(),
-            data_len: seed,
-        };
-        (stored_meta, AccountSharedData::from(account))
-    }
-
     /// The helper function for all write_accounts tests.
     /// Currently only supports hot accounts.
     fn do_test_write_accounts(
@@ -333,7 +316,7 @@ mod tests {
     ) {
         let accounts: Vec<_> = account_data_sizes
             .iter()
-            .map(|size| create_account(*size))
+            .map(|size| create_test_account(*size))
             .collect();
 
         let account_refs: Vec<_> = accounts
@@ -363,34 +346,27 @@ mod tests {
         let tiered_storage = TieredStorage::new_writable(tiered_storage_path);
         _ = tiered_storage.write_accounts(&storable_accounts, 0, &format);
 
-        verify_hot_storage(&tiered_storage, &accounts, format);
-    }
-
-    /// Verify the generated tiered storage in the test.
-    fn verify_hot_storage(
-        tiered_storage: &TieredStorage,
-        expected_accounts: &[(StoredMeta, AccountSharedData)],
-        expected_format: TieredStorageFormat,
-    ) {
         let reader = tiered_storage.reader().unwrap();
-        assert_eq!(reader.num_accounts(), expected_accounts.len());
-
-        let footer = reader.footer();
-        let expected_footer = TieredStorageFooter {
-            account_meta_format: expected_format.account_meta_format,
-            owners_block_format: expected_format.owners_block_format,
-            index_block_format: expected_format.index_block_format,
-            account_block_format: expected_format.account_block_format,
-            account_entry_count: expected_accounts.len() as u32,
-            // Hash is not yet implemented, so we bypass the check
-            hash: footer.hash,
-            ..TieredStorageFooter::default()
-        };
+        let num_accounts = storable_accounts.len();
+        assert_eq!(reader.num_accounts(), num_accounts);
 
-        // TODO(yhchiang): verify account meta and data once the reader side
-        // is implemented in a separate PR.
+        let mut expected_accounts_map = HashMap::new();
+        for i in 0..num_accounts {
+            let (account, address, account_hash, _write_version) = storable_accounts.get(i);
+            expected_accounts_map.insert(address, (account, account_hash));
+        }
 
-        assert_eq!(*footer, expected_footer);
+        let mut index_offset = IndexOffset(0);
+        let mut verified_accounts = HashSet::new();
+        while let Some((stored_meta, next)) = reader.get_account(index_offset).unwrap() {
+            if let Some((account, account_hash)) = expected_accounts_map.get(stored_meta.pubkey()) {
+                verify_test_account(&stored_meta, *account, stored_meta.pubkey(), account_hash);
+                verified_accounts.insert(stored_meta.pubkey());
+            }
+            index_offset = next;
+        }
+        assert!(!verified_accounts.is_empty());
+        assert_eq!(verified_accounts.len(), expected_accounts_map.len())
     }
 
     #[test]