From b1976e689cb335c2014693c1aa0b19180dd8da0d Mon Sep 17 00:00:00 2001 From: Emma Zhong Date: Wed, 21 Aug 2024 13:34:49 -0700 Subject: [PATCH] [indexer] index protocol configs and feature flags (#18450) ## Description This PR adds two tables `protocol_configs` and `feature_flags` that get populated at indexer startup time and every epoch change time if protocol version has changed. ## Test plan Tested locally against devnet. --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [x] Indexer: adds two new indexer tables that stores protocol configs and features flags of different versions. - [ ] JSON-RPC: - [x] GraphQL: uses the stored data to query for protocol configs instead of native configs stored in the binary. - [ ] CLI: - [ ] Rust SDK: --- crates/sui-benchmark/tests/simtest.rs | 2 +- .../tests/epoch/protocol_configs.exp | 40 ++++++ .../tests/epoch/protocol_configs.move | 32 +++++ .../src/types/protocol_config.rs | 114 ++++++++++------- .../pg/2023-08-19-044052_epochs/up.sql | 19 +++ crates/sui-indexer/src/handlers/committer.rs | 13 ++ crates/sui-indexer/src/indexer.rs | 8 ++ crates/sui-indexer/src/models/epoch.rs | 18 ++- crates/sui-indexer/src/schema/mod.rs | 6 + crates/sui-indexer/src/schema/mysql.rs | 16 +++ crates/sui-indexer/src/schema/pg.rs | 37 ++++-- crates/sui-indexer/src/store/indexer_store.rs | 7 ++ .../sui-indexer/src/store/pg_indexer_store.rs | 116 +++++++++++++++++- 13 files changed, 366 insertions(+), 62 deletions(-) create mode 100644 crates/sui-graphql-e2e-tests/tests/epoch/protocol_configs.exp create mode 100644 crates/sui-graphql-e2e-tests/tests/epoch/protocol_configs.move diff --git a/crates/sui-benchmark/tests/simtest.rs b/crates/sui-benchmark/tests/simtest.rs index f3a29e01ba079..b7d2e189da3a2 100644 --- a/crates/sui-benchmark/tests/simtest.rs +++ b/crates/sui-benchmark/tests/simtest.rs @@ -550,7 +550,7 @@ mod test { .build() .await, ); - test_simulated_load(test_cluster, 10).await; + test_simulated_load(test_cluster, 30).await; let checkpoint_files = std::fs::read_dir(path) .map(|entries| { diff --git a/crates/sui-graphql-e2e-tests/tests/epoch/protocol_configs.exp b/crates/sui-graphql-e2e-tests/tests/epoch/protocol_configs.exp new file mode 100644 index 0000000000000..e237e6000cd9a --- /dev/null +++ b/crates/sui-graphql-e2e-tests/tests/epoch/protocol_configs.exp @@ -0,0 +1,40 @@ +processed 4 tasks + +init: +C: object(0,0) + +task 1, line 6: +//# create-checkpoint +Checkpoint created: 1 + +task 2, lines 8-19: +//# run-graphql +Response: { + "data": { + "protocolConfig": { + "protocolVersion": 51, + "config": { + "value": "128" + }, + "featureFlag": { + "value": true + } + } + } +} + +task 3, lines 21-32: +//# run-graphql +Response: { + "data": { + "protocolConfig": { + "protocolVersion": 8, + "config": { + "value": null + }, + "featureFlag": { + "value": false + } + } + } +} diff --git a/crates/sui-graphql-e2e-tests/tests/epoch/protocol_configs.move b/crates/sui-graphql-e2e-tests/tests/epoch/protocol_configs.move new file mode 100644 index 0000000000000..901326e4969a3 --- /dev/null +++ b/crates/sui-graphql-e2e-tests/tests/epoch/protocol_configs.move @@ -0,0 +1,32 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 51 --simulator --accounts C + +//# create-checkpoint + +//# run-graphql +{ + protocolConfig { + protocolVersion + config(key: "max_move_identifier_len") { + value + } + featureFlag(key: "enable_coin_deny_list") { + value + } + } +} + +//# run-graphql +{ + protocolConfig(protocolVersion: 8) { + protocolVersion + config(key: "max_move_identifier_len") { + value + } + featureFlag(key: "enable_coin_deny_list") { + value + } + } +} diff --git a/crates/sui-graphql-rpc/src/types/protocol_config.rs b/crates/sui-graphql-rpc/src/types/protocol_config.rs index ce60c22892d39..f4f8fd9caf771 100644 --- a/crates/sui-graphql-rpc/src/types/protocol_config.rs +++ b/crates/sui-graphql-rpc/src/types/protocol_config.rs @@ -1,15 +1,15 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use std::collections::BTreeMap; + use async_graphql::*; use diesel::{ExpressionMethods, QueryDsl}; -use sui_indexer::schema::{chain_identifier, epochs}; -use sui_protocol_config::{ProtocolConfig as NativeProtocolConfig, ProtocolVersion}; +use sui_indexer::schema::{epochs, feature_flags, protocol_configs}; use crate::{ data::{Db, DbConnection, QueryExecutor}, error::Error, - types::chain_identifier::ChainIdentifier, }; use super::uint53::UInt53; @@ -30,7 +30,9 @@ pub(crate) struct ProtocolConfigFeatureFlag { #[derive(Clone, Debug)] pub(crate) struct ProtocolConfigs { - native: NativeProtocolConfig, + version: u64, + configs: BTreeMap>, + feature_flags: BTreeMap, } /// Constants that control how the chain operates. @@ -41,15 +43,15 @@ impl ProtocolConfigs { /// The protocol is not required to change on every epoch boundary, so the protocol version /// tracks which change to the protocol these configs are from. async fn protocol_version(&self) -> UInt53 { - self.native.version.as_u64().into() + self.version.into() } /// List all available feature flags and their values. Feature flags are a form of boolean /// configuration that are usually used to gate features while they are in development. Once a /// flag has been enabled, it is rare for it to be disabled. async fn feature_flags(&self) -> Vec { - self.native - .feature_map() + self.feature_flags + .clone() .into_iter() .map(|(key, value)| ProtocolConfigFeatureFlag { key, value }) .collect() @@ -58,31 +60,24 @@ impl ProtocolConfigs { /// List all available configurations and their values. These configurations can take any value /// (but they will all be represented in string form), and do not include feature flags. async fn configs(&self) -> Vec { - self.native - .attr_map() + self.configs + .clone() .into_iter() - .map(|(key, value)| ProtocolConfigAttr { - key, - value: value.map(|v| v.to_string()), - }) + .map(|(key, value)| ProtocolConfigAttr { key, value }) .collect() } /// Query for the value of the configuration with name `key`. async fn config(&self, key: String) -> Option { - self.native - .attr_map() - .get(&key) - .map(|value| ProtocolConfigAttr { - key, - value: value.as_ref().map(|v| v.to_string()), - }) + self.configs.get(&key).map(|value| ProtocolConfigAttr { + key, + value: value.as_ref().map(|v| v.to_string()), + }) } /// Query for the state of the feature flag with name `key`. async fn feature_flag(&self, key: String) -> Option { - self.native - .feature_map() + self.feature_flags .get(&key) .map(|value| ProtocolConfigFeatureFlag { key, value: *value }) } @@ -90,36 +85,61 @@ impl ProtocolConfigs { impl ProtocolConfigs { pub(crate) async fn query(db: &Db, protocol_version: Option) -> Result { - use chain_identifier::dsl as c; use epochs::dsl as e; + use feature_flags::dsl as f; + use protocol_configs::dsl as p; + + let version = if let Some(version) = protocol_version { + version + } else { + let latest_version: i64 = db + .execute(move |conn| { + conn.first(move || { + e::epochs + .select(e::protocol_version) + .order_by(e::epoch.desc()) + }) + }) + .await + .map_err(|e| { + Error::Internal(format!( + "Failed to fetch latest protocol version in db: {e}" + )) + })?; + latest_version as u64 + }; + + // TODO: This could be optimized by fetching all configs and flags in a single query. + let configs: BTreeMap> = db + .execute(move |conn| { + conn.results(move || { + p::protocol_configs + .select((p::config_name, p::config_value)) + .filter(p::protocol_version.eq(version as i64)) + }) + }) + .await + .map_err(|e| Error::Internal(format!("Failed to fetch protocol configs in db: {e}")))? + .into_iter() + .collect(); - let (latest_version, digest_bytes): (i64, Option>) = db + let feature_flags: BTreeMap = db .execute(move |conn| { - conn.first(move || { - e::epochs - .select(( - e::protocol_version, - c::chain_identifier - .select(c::checkpoint_digest) - .single_value(), - )) - .order_by(e::epoch.desc()) + conn.results(move || { + f::feature_flags + .select((f::flag_name, f::flag_value)) + .filter(f::protocol_version.eq(version as i64)) }) }) .await - .map_err(|e| Error::Internal(format!("Failed to fetch system details: {e}")))?; - - let native = NativeProtocolConfig::get_for_version_if_supported( - protocol_version.unwrap_or(latest_version as u64).into(), - ChainIdentifier::from_bytes(digest_bytes.unwrap_or_default())?.chain(), - ) - .ok_or_else(|| { - Error::ProtocolVersionUnsupported( - ProtocolVersion::MIN.as_u64(), - ProtocolVersion::MAX.as_u64(), - ) - })?; - - Ok(ProtocolConfigs { native }) + .map_err(|e| Error::Internal(format!("Failed to fetch feature flags in db: {e}")))? + .into_iter() + .collect(); + + Ok(ProtocolConfigs { + version, + configs, + feature_flags, + }) } } diff --git a/crates/sui-indexer/migrations/pg/2023-08-19-044052_epochs/up.sql b/crates/sui-indexer/migrations/pg/2023-08-19-044052_epochs/up.sql index 4a0a17289ccec..5b540121cb849 100644 --- a/crates/sui-indexer/migrations/pg/2023-08-19-044052_epochs/up.sql +++ b/crates/sui-indexer/migrations/pg/2023-08-19-044052_epochs/up.sql @@ -26,3 +26,22 @@ CREATE TABLE epochs -- of the epoch epoch_commitments bytea ); + +-- Table storing the protocol configs for each protocol version. +-- Examples include gas schedule, transaction limits, etc. +CREATE TABLE protocol_configs +( + protocol_version BIGINT NOT NULL, + config_name TEXT NOT NULL, + config_value TEXT, + PRIMARY KEY(protocol_version, config_name) +); + +-- Table storing the feature flags for each protocol version. +CREATE TABLE feature_flags +( + protocol_version BIGINT NOT NULL, + flag_name TEXT NOT NULL, + flag_value BOOLEAN NOT NULL, + PRIMARY KEY(protocol_version, flag_name) +); diff --git a/crates/sui-indexer/src/handlers/committer.rs b/crates/sui-indexer/src/handlers/committer.rs index 58d6fe94e1f7c..f4e5504893f5c 100644 --- a/crates/sui-indexer/src/handlers/committer.rs +++ b/crates/sui-indexer/src/handlers/committer.rs @@ -162,6 +162,8 @@ async fn commit_checkpoints( .expect("Persisting data into DB should not fail."); } + let is_epoch_end = epoch.is_some(); + // handle partitioning on epoch boundary if let Some(epoch_data) = epoch { state @@ -184,6 +186,17 @@ async fn commit_checkpoints( ); }) .expect("Persisting data into DB should not fail."); + + if is_epoch_end { + // The epoch has advanced so we update the configs for the new protocol version, if it has changed. + let chain_id = state + .get_chain_identifier() + .await + .expect("Failed to get chain identifier") + .expect("Chain identifier should have been indexed at this point"); + let _ = state.persist_protocol_configs_and_feature_flags(chain_id); + } + let elapsed = guard.stop_and_record(); commit_notifier diff --git a/crates/sui-indexer/src/indexer.rs b/crates/sui-indexer/src/indexer.rs index 3b71cf795d37f..f38a7a1de140d 100644 --- a/crates/sui-indexer/src/indexer.rs +++ b/crates/sui-indexer/src/indexer.rs @@ -122,6 +122,14 @@ impl Indexer { spawn_monitored_task!(pruner.start(CancellationToken::new())); } + // If we already have chain identifier indexed (i.e. the first checkpoint has been indexed), + // then we persist protocol configs for protocol versions not yet in the db. + // Otherwise, we would do the persisting in `commit_checkpoint` while the first cp is + // being indexed. + if let Some(chain_id) = store.get_chain_identifier().await? { + store.persist_protocol_configs_and_feature_flags(chain_id)?; + } + let cancel_clone = cancel.clone(); let (exit_sender, exit_receiver) = oneshot::channel(); // Spawn a task that links the cancellation token to the exit sender diff --git a/crates/sui-indexer/src/models/epoch.rs b/crates/sui-indexer/src/models/epoch.rs index a392fafbbd4d5..0991203d5cd02 100644 --- a/crates/sui-indexer/src/models/epoch.rs +++ b/crates/sui-indexer/src/models/epoch.rs @@ -3,9 +3,9 @@ use diesel::{Insertable, Queryable, Selectable}; -use crate::errors::IndexerError; use crate::schema::epochs; use crate::types::IndexedEpochInfo; +use crate::{errors::IndexerError, schema::feature_flags, schema::protocol_configs}; use sui_json_rpc_types::{EndOfEpochInfo, EpochInfo}; use sui_types::sui_system_state::sui_system_state_summary::SuiSystemStateSummary; @@ -33,6 +33,22 @@ pub struct StoredEpochInfo { pub epoch_commitments: Option>, } +#[derive(Queryable, Insertable, Debug, Clone, Default)] +#[diesel(table_name = protocol_configs)] +pub struct StoredProtocolConfig { + pub protocol_version: i64, + pub config_name: String, + pub config_value: Option, +} + +#[derive(Queryable, Insertable, Debug, Clone, Default)] +#[diesel(table_name = feature_flags)] +pub struct StoredFeatureFlag { + pub protocol_version: i64, + pub flag_name: String, + pub flag_value: bool, +} + #[derive(Queryable, Selectable, Clone)] #[diesel(table_name = epochs)] pub struct QueryableEpochInfo { diff --git a/crates/sui-indexer/src/schema/mod.rs b/crates/sui-indexer/src/schema/mod.rs index 8f98297ac4d80..dfba1ad7b7b43 100644 --- a/crates/sui-indexer/src/schema/mod.rs +++ b/crates/sui-indexer/src/schema/mod.rs @@ -24,11 +24,13 @@ mod inner { pub use crate::schema::pg::event_struct_name; pub use crate::schema::pg::event_struct_package; pub use crate::schema::pg::events; + pub use crate::schema::pg::feature_flags; pub use crate::schema::pg::objects; pub use crate::schema::pg::objects_history; pub use crate::schema::pg::objects_snapshot; pub use crate::schema::pg::objects_version; pub use crate::schema::pg::packages; + pub use crate::schema::pg::protocol_configs; pub use crate::schema::pg::pruner_cp_watermark; pub use crate::schema::pg::transactions; pub use crate::schema::pg::tx_calls_fun; @@ -57,11 +59,13 @@ mod inner { pub use crate::schema::mysql::event_struct_name; pub use crate::schema::mysql::event_struct_package; pub use crate::schema::mysql::events; + pub use crate::schema::mysql::feature_flags; pub use crate::schema::mysql::objects; pub use crate::schema::mysql::objects_history; pub use crate::schema::mysql::objects_snapshot; pub use crate::schema::mysql::objects_version; pub use crate::schema::mysql::packages; + pub use crate::schema::mysql::protocol_configs; pub use crate::schema::mysql::pruner_cp_watermark; pub use crate::schema::mysql::transactions; pub use crate::schema::mysql::tx_calls_fun; @@ -87,11 +91,13 @@ pub use inner::event_struct_module; pub use inner::event_struct_name; pub use inner::event_struct_package; pub use inner::events; +pub use inner::feature_flags; pub use inner::objects; pub use inner::objects_history; pub use inner::objects_snapshot; pub use inner::objects_version; pub use inner::packages; +pub use inner::protocol_configs; pub use inner::pruner_cp_watermark; pub use inner::transactions; pub use inner::tx_calls_fun; diff --git a/crates/sui-indexer/src/schema/mysql.rs b/crates/sui-indexer/src/schema/mysql.rs index e7805ae562be0..4b4799566a0d4 100644 --- a/crates/sui-indexer/src/schema/mysql.rs +++ b/crates/sui-indexer/src/schema/mysql.rs @@ -150,6 +150,14 @@ diesel::table! { } } +diesel::table! { + feature_flags (protocol_version, flag_name) { + protocol_version -> Bigint, + flag_name -> Text, + flag_value -> Bool, + } +} + diesel::table! { objects (object_id) { object_id -> Blob, @@ -236,6 +244,14 @@ diesel::table! { } } +diesel::table! { + protocol_configs (protocol_version, config_name) { + protocol_version -> Bigint, + config_name -> Text, + config_value -> Nullable, + } +} + diesel::table! { pruner_cp_watermark (checkpoint_sequence_number) { checkpoint_sequence_number -> Bigint, diff --git a/crates/sui-indexer/src/schema/pg.rs b/crates/sui-indexer/src/schema/pg.rs index 1ef8e0aed8e81..2515c98d34c06 100644 --- a/crates/sui-indexer/src/schema/pg.rs +++ b/crates/sui-indexer/src/schema/pg.rs @@ -31,14 +31,6 @@ diesel::table! { } } -diesel::table! { - pruner_cp_watermark (checkpoint_sequence_number) { - checkpoint_sequence_number -> Int8, - min_tx_sequence_number -> Int8, - max_tx_sequence_number -> Int8, - } -} - diesel::table! { display (object_type) { object_type -> Text, @@ -176,6 +168,14 @@ diesel::table! { } } +diesel::table! { + feature_flags (protocol_version, flag_name) { + protocol_version -> Int8, + flag_name -> Text, + flag_value -> Bool, + } +} + diesel::table! { objects (object_id) { object_id -> Bytea, @@ -275,6 +275,14 @@ diesel::table! { } } +diesel::table! { + protocol_configs (protocol_version, config_name) { + protocol_version -> Int8, + config_name -> Text, + config_value -> Nullable, + } +} + diesel::table! { packages (package_id, original_id, package_version) { package_id -> Bytea, @@ -285,6 +293,14 @@ diesel::table! { } } +diesel::table! { + pruner_cp_watermark (checkpoint_sequence_number) { + checkpoint_sequence_number -> Int8, + min_tx_sequence_number -> Int8, + max_tx_sequence_number -> Int8, + } +} + diesel::table! { transactions (tx_sequence_number) { tx_sequence_number -> Int8, @@ -395,7 +411,6 @@ macro_rules! for_all_tables { $action!( chain_identifier, checkpoints, - pruner_cp_watermark, display, epochs, event_emit_module, @@ -406,11 +421,13 @@ macro_rules! for_all_tables { event_struct_name, event_struct_package, events, - objects, + feature_flags, objects_history, objects_snapshot, objects_version, packages, + protocol_configs, + pruner_cp_watermark, transactions, tx_calls_fun, tx_calls_mod, diff --git a/crates/sui-indexer/src/store/indexer_store.rs b/crates/sui-indexer/src/store/indexer_store.rs index 97516929ffe24..2fd2c1531b76a 100644 --- a/crates/sui-indexer/src/store/indexer_store.rs +++ b/crates/sui-indexer/src/store/indexer_store.rs @@ -32,6 +32,13 @@ pub trait IndexerStore: Any + Clone + Sync + Send + 'static { &self, ) -> Result, IndexerError>; + async fn get_chain_identifier(&self) -> Result>, IndexerError>; + + fn persist_protocol_configs_and_feature_flags( + &self, + chain_id: Vec, + ) -> Result<(), IndexerError>; + async fn persist_objects( &self, object_changes: Vec, diff --git a/crates/sui-indexer/src/store/pg_indexer_store.rs b/crates/sui-indexer/src/store/pg_indexer_store.rs index 59e6ba3dc07ff..7a2ac3cdb8854 100644 --- a/crates/sui-indexer/src/store/pg_indexer_store.rs +++ b/crates/sui-indexer/src/store/pg_indexer_store.rs @@ -20,6 +20,7 @@ use itertools::Itertools; use tap::TapFallible; use tracing::info; +use sui_protocol_config::ProtocolConfig; use sui_types::base_types::ObjectID; use crate::db::ConnectionPool; @@ -32,6 +33,7 @@ use crate::models::checkpoints::StoredCheckpoint; use crate::models::checkpoints::StoredCpTx; use crate::models::display::StoredDisplay; use crate::models::epoch::StoredEpochInfo; +use crate::models::epoch::{StoredFeatureFlag, StoredProtocolConfig}; use crate::models::events::StoredEvent; use crate::models::obj_indices::StoredObjectVersion; use crate::models::objects::{ @@ -43,9 +45,10 @@ use crate::models::transactions::StoredTransaction; use crate::schema::{ chain_identifier, checkpoints, display, epochs, event_emit_module, event_emit_package, event_senders, event_struct_instantiation, event_struct_module, event_struct_name, - event_struct_package, events, objects, objects_history, objects_snapshot, objects_version, - packages, pruner_cp_watermark, transactions, tx_calls_fun, tx_calls_mod, tx_calls_pkg, - tx_changed_objects, tx_digests, tx_input_objects, tx_kinds, tx_recipients, tx_senders, + event_struct_package, events, feature_flags, objects, objects_history, objects_snapshot, + objects_version, packages, protocol_configs, pruner_cp_watermark, transactions, tx_calls_fun, + tx_calls_mod, tx_calls_pkg, tx_changed_objects, tx_digests, tx_input_objects, tx_kinds, + tx_recipients, tx_senders, }; use crate::types::EventIndex; use crate::types::{IndexedCheckpoint, IndexedEvent, IndexedPackage, IndexedTransaction, TxIndex}; @@ -60,6 +63,7 @@ use super::ObjectChangeToCommit; #[cfg(feature = "postgres-feature")] use diesel::upsert::excluded; +use sui_types::digests::{ChainIdentifier, CheckpointDigest}; #[macro_export] macro_rules! chunk { @@ -195,6 +199,38 @@ impl PgIndexerStore { .context("Failed reading latest epoch id from PostgresDB") } + /// Get the range of the protocol versions that need to be indexed. + pub fn get_protocol_version_index_range(&self) -> Result<(i64, i64), IndexerError> { + // We start indexing from the next protocol version after the latest one stored in the db. + let start = read_only_blocking!(&self.blocking_cp, |conn| { + protocol_configs::dsl::protocol_configs + .select(max(protocol_configs::protocol_version)) + .first::>(conn) + }) + .context("Failed reading latest protocol version from PostgresDB")? + .map_or(1, |v| v + 1); + + // We end indexing at the protocol version of the latest epoch stored in the db. + let end = read_only_blocking!(&self.blocking_cp, |conn| { + epochs::dsl::epochs + .select(max(epochs::protocol_version)) + .first::>(conn) + }) + .context("Failed reading latest epoch protocol version from PostgresDB")? + .unwrap_or(1); + Ok((start, end)) + } + + pub fn get_chain_identifier(&self) -> Result>, IndexerError> { + read_only_blocking!(&self.blocking_cp, |conn| { + chain_identifier::dsl::chain_identifier + .select(chain_identifier::checkpoint_digest) + .first::>(conn) + .optional() + }) + .context("Failed reading chain id from PostgresDB") + } + fn get_latest_checkpoint_sequence_number(&self) -> Result, IndexerError> { read_only_blocking!(&self.blocking_cp, |conn| { checkpoints::dsl::checkpoints @@ -627,6 +663,8 @@ impl PgIndexerStore { // If the first checkpoint has sequence number 0, we need to persist the digest as // chain identifier. if first_checkpoint.sequence_number == 0 { + let checkpoint_digest = first_checkpoint.checkpoint_digest.into_inner().to_vec(); + self.persist_protocol_configs_and_feature_flags(checkpoint_digest.clone())?; transactional_blocking_with_retry!( &self.blocking_cp, |conn| { @@ -1629,6 +1667,11 @@ impl IndexerStore for PgIndexerStore { .await } + async fn get_chain_identifier(&self) -> Result>, IndexerError> { + self.execute_in_blocking_worker(|this| this.get_chain_identifier()) + .await + } + async fn get_latest_object_snapshot_checkpoint_sequence_number( &self, ) -> Result, IndexerError> { @@ -2143,6 +2186,73 @@ impl IndexerStore for PgIndexerStore { fn as_any(&self) -> &dyn StdAny { self } + + /// Persist protocol configs and feature flags until the protocol version for the latest epoch + /// we have stored in the db, inclusive. + fn persist_protocol_configs_and_feature_flags( + &self, + chain_id: Vec, + ) -> Result<(), IndexerError> { + let chain_id = ChainIdentifier::from( + CheckpointDigest::try_from(chain_id).expect("Unable to convert chain id"), + ); + + let mut all_configs = vec![]; + let mut all_flags = vec![]; + + let (start_version, end_version) = self.get_protocol_version_index_range()?; + info!( + "Persisting protocol configs with start_version: {}, end_version: {}", + start_version, end_version + ); + + // Gather all protocol configs and feature flags for all versions between start and end. + for version in start_version..=end_version { + let protocol_configs = ProtocolConfig::get_for_version_if_supported( + (version as u64).into(), + chain_id.chain(), + ) + .ok_or(IndexerError::GenericError(format!( + "Unable to fetch protocol version {} and chain {:?}", + version, + chain_id.chain() + )))?; + let configs_vec = protocol_configs + .attr_map() + .into_iter() + .map(|(k, v)| StoredProtocolConfig { + protocol_version: version, + config_name: k, + config_value: v.map(|v| v.to_string()), + }) + .collect::>(); + all_configs.extend(configs_vec); + + let feature_flags = protocol_configs + .feature_map() + .into_iter() + .map(|(k, v)| StoredFeatureFlag { + protocol_version: version, + flag_name: k, + flag_value: v, + }) + .collect::>(); + all_flags.extend(feature_flags); + } + + // Now insert all of them into the db. + // TODO: right now the size of these updates is manageable but later we may consider batching. + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + insert_or_ignore_into!(protocol_configs::table, all_configs.clone(), conn); + insert_or_ignore_into!(feature_flags::table, all_flags.clone(), conn); + Ok::<(), IndexerError>(()) + }, + PG_DB_COMMIT_SLEEP_DURATION + )?; + Ok(()) + } } /// Construct deleted objects and mutated objects to commit.