From ceca0945419533ed190f70fc04842496a029eb37 Mon Sep 17 00:00:00 2001 From: kamille Date: Sat, 14 Sep 2024 13:05:01 +0800 Subject: [PATCH 1/2] add a config option to control should we compat the old to on/off layered memtable. --- src/analytic_engine/src/instance/open.rs | 1 + src/analytic_engine/src/lib.rs | 7 ++++ src/analytic_engine/src/manifest/details.rs | 1 + src/analytic_engine/src/table/data.rs | 38 ++++++++++++++----- .../src/table_meta_set_impl.rs | 4 ++ 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/analytic_engine/src/instance/open.rs b/src/analytic_engine/src/instance/open.rs index 525a97defb..220fa84c3a 100644 --- a/src/analytic_engine/src/instance/open.rs +++ b/src/analytic_engine/src/instance/open.rs @@ -122,6 +122,7 @@ impl Instance { preflush_write_buffer_size_ratio: ctx.config.preflush_write_buffer_size_ratio, manifest_snapshot_every_n_updates: ctx.config.manifest.snapshot_every_n_updates, enable_primary_key_sampling: ctx.config.enable_primary_key_sampling, + try_compat_old_layered_memtable_opts: ctx.config.try_compat_old_layered_memtable_opts, metrics_opt: ctx.config.metrics.clone(), }); let manifest = ManifestImpl::open( diff --git a/src/analytic_engine/src/lib.rs b/src/analytic_engine/src/lib.rs index e6fa2f85b5..9b2780dd3e 100644 --- a/src/analytic_engine/src/lib.rs +++ b/src/analytic_engine/src/lib.rs @@ -69,6 +69,12 @@ pub struct Config { /// Default options for table pub table_opts: TableOptions, + /// Should we try to compat the `LayeredMemtableOptions` in `TableOptions` + /// The old one use if `mutable_segment_switch_threshold` > 0 to control + /// the on/off of layered memtable(`0`:off, `>0`:on). + /// The new one use a explicit flag `enable` to do that. + pub try_compat_old_layered_memtable_opts: bool, + pub compaction: SchedulerConfig, /// sst meta cache capacity @@ -179,6 +185,7 @@ impl Default for Config { replay_batch_size: 500, max_replay_tables_per_batch: 64, table_opts: TableOptions::default(), + try_compat_old_layered_memtable_opts: false, compaction: SchedulerConfig::default(), sst_meta_cache_cap: Some(1000), sst_data_cache_cap: Some(1000), diff --git a/src/analytic_engine/src/manifest/details.rs b/src/analytic_engine/src/manifest/details.rs index e49c3d82cc..4f999d967d 100644 --- a/src/analytic_engine/src/manifest/details.rs +++ b/src/analytic_engine/src/manifest/details.rs @@ -776,6 +776,7 @@ mod tests { manifest_snapshot_every_n_updates: NonZeroUsize::new(usize::MAX).unwrap(), metrics_opt: MetricsOptions::default(), enable_primary_key_sampling: false, + try_compat_old_layered_memtable_opts: false, }, &purger, mem_size_options, diff --git a/src/analytic_engine/src/table/data.rs b/src/analytic_engine/src/table/data.rs index 99e59cba74..5f5c1e0e71 100644 --- a/src/analytic_engine/src/table/data.rs +++ b/src/analytic_engine/src/table/data.rs @@ -68,6 +68,7 @@ use crate::{ sst_util, version::{MemTableForWrite, MemTableState, SamplingMemTable, TableVersion}, }, + table_options::UpdateMode, MetricsOptions, TableOptions, }; @@ -155,6 +156,7 @@ pub struct TableConfig { pub manifest_snapshot_every_n_updates: NonZeroUsize, pub metrics_opt: MetricsOptions, pub enable_primary_key_sampling: bool, + pub try_compat_old_layered_memtable_opts: bool, } #[derive(Debug, Clone)] @@ -314,11 +316,13 @@ impl TableData { name, schema, } = desc; + let TableConfig { preflush_write_buffer_size_ratio, manifest_snapshot_every_n_updates, metrics_opt, enable_primary_key_sampling, + .. } = config; let memtable_factory: MemTableFactoryRef = match opts.memtable_type { @@ -406,6 +410,7 @@ impl TableData { manifest_snapshot_every_n_updates, metrics_opt, enable_primary_key_sampling, + try_compat_old_layered_memtable_opts, } = config; let memtable_factory: MemTableFactoryRef = match add_meta.opts.memtable_type { @@ -421,17 +426,31 @@ impl TableData { .mutable_segment_switch_threshold .0 as usize; - ensure!( - mutable_segment_switch_threshold > 0, - InvalidTableOpts { + if mutable_segment_switch_threshold > 0 { + ensure!( + add_meta.opts.update_mode != UpdateMode::Overwrite, + InvalidTableOpts { + msg: "layered memtable is enabled but update mode is Overwrite", + } + ); + + Arc::new(LayeredMemtableFactory::new( + memtable_factory, + mutable_segment_switch_threshold, + )) as _ + } else if try_compat_old_layered_memtable_opts { + // Maybe some old layered memtable opts controlling if we enable this feature + // by checking `mutable_segment_switch_threshold`(`0`:disable, `>0`:enable) + // were persisted. + // If `try_compat_old_layered_memtable_opts` is true, we will try to keep the + // same behavior. + memtable_factory as _ + } else { + return InvalidTableOpts { msg: "layered memtable is enabled but mutable_switch_threshold is 0", } - ); - - Arc::new(LayeredMemtableFactory::new( - memtable_factory, - mutable_segment_switch_threshold, - )) as _ + .fail(); + } } else { memtable_factory as _ }; @@ -1028,6 +1047,7 @@ pub mod tests { manifest_snapshot_every_n_updates: self.manifest_snapshot_every_n_updates, metrics_opt: MetricsOptions::default(), enable_primary_key_sampling: false, + try_compat_old_layered_memtable_opts: false, }, &purger, mem_size_options, diff --git a/src/analytic_engine/src/table_meta_set_impl.rs b/src/analytic_engine/src/table_meta_set_impl.rs index e14f5e4a3c..23f3f7a3f0 100644 --- a/src/analytic_engine/src/table_meta_set_impl.rs +++ b/src/analytic_engine/src/table_meta_set_impl.rs @@ -54,6 +54,7 @@ pub(crate) struct TableMetaSetImpl { pub(crate) preflush_write_buffer_size_ratio: f32, pub(crate) manifest_snapshot_every_n_updates: NonZeroUsize, pub(crate) enable_primary_key_sampling: bool, + pub(crate) try_compat_old_layered_memtable_opts: bool, pub(crate) metrics_opt: MetricsOptions, } @@ -140,6 +141,8 @@ impl TableMetaSetImpl { .manifest_snapshot_every_n_updates, metrics_opt: self.metrics_opt.clone(), enable_primary_key_sampling: self.enable_primary_key_sampling, + try_compat_old_layered_memtable_opts: self + .try_compat_old_layered_memtable_opts, }, &self.file_purger, mem_size_options, @@ -271,6 +274,7 @@ impl TableMetaSetImpl { manifest_snapshot_every_n_updates: self.manifest_snapshot_every_n_updates, metrics_opt: self.metrics_opt.clone(), enable_primary_key_sampling: self.enable_primary_key_sampling, + try_compat_old_layered_memtable_opts: self.try_compat_old_layered_memtable_opts, }, mem_size_options, allocator, From 24a46e4d70ba727b3f892241975a026b40c36e28 Mon Sep 17 00:00:00 2001 From: kamille Date: Sat, 14 Sep 2024 13:10:12 +0800 Subject: [PATCH 2/2] fix comment. --- src/analytic_engine/src/table/data.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/analytic_engine/src/table/data.rs b/src/analytic_engine/src/table/data.rs index 5f5c1e0e71..3f01362319 100644 --- a/src/analytic_engine/src/table/data.rs +++ b/src/analytic_engine/src/table/data.rs @@ -439,11 +439,11 @@ impl TableData { mutable_segment_switch_threshold, )) as _ } else if try_compat_old_layered_memtable_opts { - // Maybe some old layered memtable opts controlling if we enable this feature + // Maybe some old layered memtable opts controlling the on/off of this feature // by checking `mutable_segment_switch_threshold`(`0`:disable, `>0`:enable) // were persisted. - // If `try_compat_old_layered_memtable_opts` is true, we will try to keep the - // same behavior. + // If `try_compat_old_layered_memtable_opts` is true, we will try to follow the + // old behavior. memtable_factory as _ } else { return InvalidTableOpts {