diff --git a/src/meta/src/hummock/compaction_scheduler.rs b/src/meta/src/hummock/compaction_scheduler.rs index c4904040fdb91..026316e4d7400 100644 --- a/src/meta/src/hummock/compaction_scheduler.rs +++ b/src/meta/src/hummock/compaction_scheduler.rs @@ -540,7 +540,7 @@ where if history.len() >= HISTORY_TABLE_INFO_WINDOW_SIZE { let window_total_size = history.iter().sum::(); is_high_write_throughput = history.iter().all(|throughput| { - **throughput > self.env.opts.table_write_throughput_threshold + *throughput > self.env.opts.table_write_throughput_threshold }); is_low_write_throughput = window_total_size < (HISTORY_TABLE_INFO_WINDOW_SIZE as u64) diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs index c2d6e72a03ef9..d17f26ff1a793 100644 --- a/src/meta/src/hummock/manager/compaction_group_manager.rs +++ b/src/meta/src/hummock/manager/compaction_group_manager.rs @@ -644,29 +644,11 @@ impl HummockManager { #[named] pub async fn calculate_compaction_group_statistic(&self) -> Vec { - let mut infos = vec![]; - { + let mut infos = { let versioning_guard = read_lock!(self, versioning).await; - let table_infos = versioning_guard.version_stats.table_stats.clone(); - - for (group_id, group) in &versioning_guard.current_version.levels { - let mut group_info = TableGroupInfo { - group_id: *group_id, - ..Default::default() - }; - for table_id in &group.member_table_ids { - if let Some(stats) = table_infos.get(table_id) { - let table_size = stats.total_key_size + stats.total_value_size; - group_info - .table_statistic - .insert(*table_id, table_size as u64); - group_info.group_size += table_size as u64; - } else { - group_info.table_statistic.insert(*table_id, 0); - } - } - infos.push(group_info); - } + versioning_guard + .current_version + .calculate_compaction_group_statistic() }; let manager = self.compaction_group_manager.read().await; for info in &mut infos { diff --git a/src/meta/src/lib.rs b/src/meta/src/lib.rs index 1131081d1ee81..c72e8c465f7f0 100644 --- a/src/meta/src/lib.rs +++ b/src/meta/src/lib.rs @@ -265,7 +265,7 @@ pub fn start(opts: MetaNodeOpts) -> Pin + Send>> { max_compactor_task_multiplier: config.meta.max_compactor_task_multiplier, split_group_size_limit: config.meta.split_group_size_limit, move_table_size_limit: config.meta.move_table_size_limit, - table_write_throughput_threshold: config.meta.table_write_throughput_limit, + table_write_throughput_threshold: config.meta.table_write_throughput_threshold, min_table_split_write_throughput: config.meta.min_table_split_write_throughput, partition_vnode_count: config.meta.partition_vnode_count, do_not_config_object_storage_lifecycle: config diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index b94256f2d0835..eb0fd05613545 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -125,6 +125,8 @@ pub trait HummockVersionExt { fn level_iter bool>(&self, compaction_group_id: CompactionGroupId, f: F); fn get_object_ids(&self) -> Vec; + + fn calculate_compaction_group_statistic(&self) -> Vec; } pub type BranchedSstInfo = HashMap; @@ -218,6 +220,62 @@ impl HummockVersionExt for HummockVersion { .map(|group| group.levels.len() + 1) .unwrap_or(0) } + + fn calculate_compaction_group_statistic(&self) -> Vec { + let mut infos = vec![]; + for (group_id, group) in &self.levels { + if group.member_table_ids.len() == 1 { + continue; + } + let group_size = group + .levels + .iter() + .map(|level| level.total_file_size) + .sum::() + + group.l0.as_ref().unwrap().total_file_size; + let mut table_statistic: HashMap = HashMap::new(); + let member_table_id: HashSet = HashSet::from_iter(group.member_table_ids.clone()); + for level in &group.l0.as_ref().unwrap().sub_levels { + if level.level_type() == LevelType::Overlapping { + continue; + } + for sst in &level.table_infos { + if sst.table_ids.len() > 1 { + // do not calculate size for small state-table. + continue; + } + if !member_table_id.contains(&sst.table_ids[0]) { + continue; + } + let entry = table_statistic.entry(sst.table_ids[0]).or_default(); + *entry += sst.file_size; + } + } + + for level in &group.levels { + for sst in &level.table_infos { + if sst.table_ids.len() > 1 { + // do not calculate size for small state-table. + continue; + } + for table_id in &sst.table_ids { + if !member_table_id.contains(table_id) { + continue; + } + let entry = table_statistic.entry(*table_id).or_default(); + *entry += sst.file_size; + } + } + } + infos.push(TableGroupInfo { + group_id: *group_id, + group_size, + table_statistic, + split_by_table: false, + }); + } + infos + } } pub type SstSplitInfo = (