Step Finance Patch

core/src/banking_stage/committer.rs

@@ -9,7 +9,7 @@ use {
         accounts::TransactionLoadResult,
         bank::{
             Bank, CommitTransactionCounts, TransactionBalancesSet, TransactionExecutionResult,
-            TransactionResults,
+            TransactionResults, TransactionDatumSet
         },
         bank_utils,
         prioritization_fee_cache::PrioritizationFeeCache,
@@ -32,6 +32,7 @@ pub enum CommitTransactionDetails {
 #[derive(Default)]
 pub(super) struct PreBalanceInfo {
     pub native: Vec<Vec<u64>>,
+    pub datum: Vec<Vec<Option<Vec<u8>>>>,
     pub token: Vec<Vec<TransactionTokenBalance>>,
     pub mint_decimals: HashMap<Pubkey, u8>,
 }
@@ -142,7 +143,7 @@ impl Committer {
     ) {
         if let Some(transaction_status_sender) = &self.transaction_status_sender {
             let txs = batch.sanitized_transactions().to_vec();
-            let post_balances = bank.collect_balances(batch);
+            let (post_balances, post_datum) = bank.collect_balances_and_datum(batch);
             let post_token_balances =
                 collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals);
             let mut transaction_index = starting_transaction_index.unwrap_or_default();
@@ -167,6 +168,10 @@ impl Committer {
                     std::mem::take(&mut pre_balance_info.native),
                     post_balances,
                 ),
+                TransactionDatumSet::new(
+                    std::mem::take(&mut pre_balance_info.datum), 
+                    post_datum
+                ),
                 TransactionTokenBalancesSet::new(
                     std::mem::take(&mut pre_balance_info.token),
                     post_token_balances,

core/src/banking_stage/consumer.rs

@@ -537,7 +537,7 @@ impl Consumer {
             // If the extra meta-data services are enabled for RPC, collect the
             // pre-balances for native and token programs.
             if transaction_status_sender_enabled {
-                pre_balance_info.native = bank.collect_balances(batch);
+                (pre_balance_info.native, pre_balance_info.datum) = bank.collect_balances_and_datum(batch);
                 pre_balance_info.token =
                     collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals)
             }

ledger/src/blockstore_processor.rs

@@ -28,7 +28,7 @@ use {
         accounts_index::AccountSecondaryIndexes,
         accounts_update_notifier_interface::AccountsUpdateNotifier,
         bank::{
-            Bank, TransactionBalancesSet, TransactionExecutionDetails, TransactionExecutionResult,
+            Bank, TransactionBalancesSet, TransactionDatumSet, TransactionExecutionDetails, TransactionExecutionResult,
             TransactionResults,
         },
         bank_forks::BankForks,
@@ -155,7 +155,7 @@ fn execute_batch(
         vec![]
     };
 
-    let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions(
+    let (tx_results, balances, datum) = batch.bank().load_execute_and_commit_transactions(
         batch,
         MAX_PROCESSING_AGE,
         transaction_status_sender.is_some(),
@@ -201,6 +201,7 @@ fn execute_batch(
             transactions,
             execution_results,
             balances,
+            datum,
             token_balances,
             rent_debits,
             transaction_indexes.to_vec(),
@@ -1744,6 +1745,7 @@ pub struct TransactionStatusBatch {
     pub transactions: Vec<SanitizedTransaction>,
     pub execution_results: Vec<Option<TransactionExecutionDetails>>,
     pub balances: TransactionBalancesSet,
+    pub datum: TransactionDatumSet,
     pub token_balances: TransactionTokenBalancesSet,
     pub rent_debits: Vec<RentDebits>,
     pub transaction_indexes: Vec<usize>,
@@ -1761,6 +1763,7 @@ impl TransactionStatusSender {
         transactions: Vec<SanitizedTransaction>,
         execution_results: Vec<TransactionExecutionResult>,
         balances: TransactionBalancesSet,
+        datum: TransactionDatumSet,
         token_balances: TransactionTokenBalancesSet,
         rent_debits: Vec<RentDebits>,
         transaction_indexes: Vec<usize>,
@@ -1780,6 +1783,7 @@ impl TransactionStatusSender {
                     })
                     .collect(),
                 balances,
+                datum,
                 token_balances,
                 rent_debits,
                 transaction_indexes,

rpc-client/src/mock_sender.rs

@@ -226,6 +226,8 @@ impl RpcSender for MockSender {
                             fee: 0,
                             pre_balances: vec![499999999999999950, 50, 1],
                             post_balances: vec![499999999999999950, 50, 1],
+                            pre_datum: None,
+                            post_datum: None,
                             inner_instructions: OptionSerializer::None,
                             log_messages: OptionSerializer::None,
                             pre_token_balances: OptionSerializer::None,

rpc/src/transaction_status_service.rs

@@ -70,6 +70,7 @@ impl TransactionStatusService {
                 bank,
                 transactions,
                 execution_results,
+                datum,
                 balances,
                 token_balances,
                 rent_debits,
@@ -81,6 +82,8 @@ impl TransactionStatusService {
                     execution_result,
                     pre_balances,
                     post_balances,
+                    pre_datum,
+                    post_datum,
                     pre_token_balances,
                     post_token_balances,
                     rent_debits,
@@ -90,6 +93,8 @@ impl TransactionStatusService {
                     execution_results,
                     balances.pre_balances,
                     balances.post_balances,
+                    datum.pre_datum,
+                    datum.post_datum,
                     token_balances.pre_token_balances,
                     token_balances.post_token_balances,
                     rent_debits,
@@ -159,6 +164,8 @@ impl TransactionStatusService {
                             fee,
                             pre_balances,
                             post_balances,
+                            pre_datum: Some(pre_datum),
+                            post_datum: Some(post_datum),
                             inner_instructions,
                             log_messages,
                             pre_token_balances,

runtime/src/bank.rs

@@ -211,6 +211,8 @@ pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0;
 
 pub const MAX_LEADER_SCHEDULE_STAKES: Epoch = 5;
 
+pub const STEP_TX_DATUM_MAX_SIZE: usize = 1_000_000;
+
 #[derive(Default)]
 struct RentMetrics {
     hold_range_us: AtomicU64,
@@ -422,6 +424,22 @@ impl TransactionBalancesSet {
 }
 pub type TransactionBalances = Vec<Vec<u64>>;
 
+pub struct TransactionDatumSet {
+    pub pre_datum: TransactionDatum,
+    pub post_datum: TransactionDatum,
+}
+
+impl TransactionDatumSet {
+    pub fn new(pre_datum: TransactionDatum, post_datum: TransactionDatum) -> Self {
+        assert_eq!(pre_datum.len(), post_datum.len());
+        Self {
+            pre_datum,
+            post_datum,
+        }
+    }
+}
+pub type TransactionDatum = Vec<Vec<Option<Vec<u8>>>>;
+
 /// An ordered list of compiled instructions that were invoked during a
 /// transaction instruction
 pub type InnerInstructions = Vec<InnerInstruction>;
@@ -799,6 +817,7 @@ impl PartialEq for Bank {
             loaded_programs_cache: _,
             check_program_modification_slot: _,
             epoch_reward_status: _,
+            dataum_excluded_programs: _,
             // Ignore new fields explicitly if they do not impact PartialEq.
             // Adding ".." will remove compile-time checks that if a new field
             // is added to the struct, this PartialEq is accordingly updated.
@@ -1071,6 +1090,8 @@ pub struct Bank {
     bank_freeze_or_destruction_incremented: AtomicBool,
 
     epoch_reward_status: EpochRewardStatus,
+    /// programs that will not have their account data sent to geyser
+    pub dataum_excluded_programs: (HashSet<Pubkey>, HashSet<Pubkey>),
 }
 
 struct VoteWithStakeDelegations {
@@ -1292,6 +1313,7 @@ impl Bank {
             loaded_programs_cache: Arc::<RwLock<LoadedPrograms>>::default(),
             check_program_modification_slot: false,
             epoch_reward_status: EpochRewardStatus::default(),
+            dataum_excluded_programs: Self::get_dataum_excluded_programs(),
         };
 
         bank.bank_created();
@@ -1605,6 +1627,7 @@ impl Bank {
             loaded_programs_cache: parent.loaded_programs_cache.clone(),
             check_program_modification_slot: false,
             epoch_reward_status: parent.epoch_reward_status.clone(),
+            dataum_excluded_programs: Self::get_dataum_excluded_programs(),
         };
 
         let (_, ancestors_time_us) = measure_us!({
@@ -1755,6 +1778,39 @@ impl Bank {
         );
     }
 
+    /// programs that we don't need the data for
+    /// both the owner and the account itself are compared
+    fn get_dataum_excluded_programs() -> (HashSet<Pubkey>, HashSet<Pubkey>) {
+        let mut token_set = HashSet::<Pubkey>::new();
+        //token
+        token_set.insert(Pubkey::try_from("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap());
+
+        //my logic for detecting and only including mints doesn't apply to 2022
+        //token2022
+        //token_set.insert(Pubkey::try_from("TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb").unwrap());
+
+        let mut other_set = HashSet::<Pubkey>::new();
+        //bpf loader
+        other_set.insert(Pubkey::try_from("BPFLoaderUpgradeab1e11111111111111111111111").unwrap());
+        //voting
+        other_set.insert(Pubkey::try_from("Vote111111111111111111111111111111111111111").unwrap());
+        //address lookups
+        other_set.insert(Pubkey::try_from("AddressLookupTab1e1111111111111111111111111").unwrap());
+        //system
+        other_set.insert(Pubkey::try_from("11111111111111111111111111111111").unwrap());
+        //sysvars
+        other_set.insert(Pubkey::try_from("SysvarEpochSchedu1e111111111111111111111111").unwrap());
+        other_set.insert(Pubkey::try_from("SysvarFees111111111111111111111111111111111").unwrap());
+        other_set.insert(Pubkey::try_from("Sysvar1nstructions1111111111111111111111111").unwrap());
+        other_set.insert(Pubkey::try_from("SysvarRecentB1ockHashes11111111111111111111").unwrap());
+        other_set.insert(Pubkey::try_from("SysvarRent111111111111111111111111111111111").unwrap());
+        other_set.insert(Pubkey::try_from("SysvarS1otHashes111111111111111111111111111").unwrap());
+        other_set.insert(Pubkey::try_from("SysvarStakeHistory1111111111111111111111111").unwrap());
+        other_set.insert(Pubkey::try_from("SysvarEpochRewards1111111111111111111111111").unwrap());
+        other_set.insert(Pubkey::try_from("SysvarLastRestartS1ot1111111111111111111111").unwrap());
+        (token_set, other_set)
+    }
+
     pub fn byte_limit_for_scans(&self) -> Option<usize> {
         self.rc
             .accounts
@@ -1947,6 +2003,7 @@ impl Bank {
             loaded_programs_cache: Arc::<RwLock<LoadedPrograms>>::default(),
             check_program_modification_slot: false,
             epoch_reward_status: EpochRewardStatus::default(),
+            dataum_excluded_programs: Self::get_dataum_excluded_programs(),
         };
         bank.bank_created();
 
@@ -3913,6 +3970,23 @@ impl Bank {
         balances
     }
 
+    pub fn collect_balances_and_datum(&self, batch: &TransactionBatch) -> (TransactionBalances, TransactionDatum) {
+        let mut balances: TransactionBalances = vec![];
+        let mut datum: TransactionDatum = vec![];
+        for transaction in batch.sanitized_transactions() {
+            let mut transaction_balances: Vec<u64> = vec![];
+            let mut transaction_datum: Vec<Option<Vec<u8>>> = vec![];
+            for account_key in transaction.message().account_keys().iter() {
+                let balance_and_data = self.get_balance_and_data(account_key);
+                transaction_balances.push(balance_and_data.0);
+                transaction_datum.push(balance_and_data.1);
+            }
+            balances.push(transaction_balances);
+            datum.push(transaction_datum);
+        }
+        (balances, datum)
+    }
+
     fn program_modification_slot(&self, pubkey: &Pubkey) -> Result<Slot> {
         let program = self
             .get_account_with_fixed_root(pubkey)
@@ -5802,11 +5876,11 @@ impl Bank {
         enable_return_data_recording: bool,
         timings: &mut ExecuteTimings,
         log_messages_bytes_limit: Option<usize>,
-    ) -> (TransactionResults, TransactionBalancesSet) {
-        let pre_balances = if collect_balances {
-            self.collect_balances(batch)
+    ) -> (TransactionResults, TransactionBalancesSet, TransactionDatumSet) {
+        let (pre_balances, pre_datum) = if collect_balances {
+            self.collect_balances_and_datum(batch)
         } else {
-            vec![]
+            (vec![], vec![])
         };
 
         let LoadAndExecuteTransactionsOutput {
@@ -5846,14 +5920,15 @@ impl Bank {
             },
             timings,
         );
-        let post_balances = if collect_balances {
-            self.collect_balances(batch)
+        let (post_balances, post_datum) = if collect_balances {
+            self.collect_balances_and_datum(batch)
         } else {
-            vec![]
+            (vec![], vec![])
         };
         (
             results,
             TransactionBalancesSet::new(pre_balances, post_balances),
+            TransactionDatumSet::new(pre_datum, post_datum),
         )
     }
 
@@ -5972,13 +6047,43 @@ impl Bank {
     pub fn read_balance(account: &AccountSharedData) -> u64 {
         account.lamports()
     }
+
+    pub fn read_data(&self, key: &Pubkey, account: &AccountSharedData) -> Option<Vec<u8>> {
+        let data = account.data();
+        let owner = account.owner();
+        //no data for
+        //large accounts
+        if data.len() > STEP_TX_DATUM_MAX_SIZE 
+            //executable accounts
+            || account.executable() 
+            //exclude token MINT accounts
+            || (self.dataum_excluded_programs.0.contains(owner) && data.len() == 82)
+            //exclude by owner
+            || self.dataum_excluded_programs.1.contains(owner)
+            //exclude by key
+            || self.dataum_excluded_programs.1.contains(key) {
+            None
+        } else {
+            Some(data.to_vec())
+        }
+    }
     /// Each program would need to be able to introspect its own state
     /// this is hard-coded to the Budget language
     pub fn get_balance(&self, pubkey: &Pubkey) -> u64 {
         self.get_account(pubkey)
             .map(|x| Self::read_balance(&x))
             .unwrap_or(0)
     }
+    pub fn get_balance_and_data(&self, pubkey: &Pubkey) -> (u64, Option<Vec<u8>>) {
+        self.get_account(pubkey)
+            .map(|x| {
+                (
+                    Self::read_balance(&x),
+                    Self::read_data(self, pubkey, &x),
+                )
+            })
+            .unwrap_or((0, None))
+    }
 
     /// Compute all the parents of the bank in order
     pub fn parents(&self) -> Vec<Arc<Bank>> {

storage-bigtable/src/lib.rs

@@ -243,6 +243,8 @@ impl From<StoredConfirmedBlockTransactionStatusMeta> for TransactionStatusMeta {
             fee,
             pre_balances,
             post_balances,
+            pre_datum: None,
+            post_datum: None,
             inner_instructions: None,
             log_messages: None,
             pre_token_balances: None,

storage-proto/src/convert.rs

@@ -369,6 +369,7 @@ impl From<TransactionStatusMeta> for generated::TransactionStatusMeta {
             loaded_addresses,
             return_data,
             compute_units_consumed,
+            ..
         } = value;
         let err = match status {
             Ok(()) => None,
@@ -522,6 +523,8 @@ impl TryFrom<generated::TransactionStatusMeta> for TransactionStatusMeta {
             fee,
             pre_balances,
             post_balances,
+            pre_datum: None,
+            post_datum: None,
             inner_instructions,
             log_messages,
             pre_token_balances,

storage-proto/src/lib.rs

@@ -200,6 +200,8 @@ impl From<StoredTransactionStatusMeta> for TransactionStatusMeta {
             fee,
             pre_balances,
             post_balances,
+            pre_datum: None,
+            post_datum: None,
             inner_instructions,
             log_messages,
             pre_token_balances: pre_token_balances
@@ -231,6 +233,7 @@ impl TryFrom<TransactionStatusMeta> for StoredTransactionStatusMeta {
             loaded_addresses,
             return_data,
             compute_units_consumed,
+            ..
         } = value;
 
         if !loaded_addresses.is_empty() {