Skip to content

Commit

Permalink
fix format
Browse files Browse the repository at this point in the history
Signed-off-by: Little-Wallace <bupt2013211450@gmail.com>
  • Loading branch information
Little-Wallace committed Jun 14, 2023
1 parent 1e63b8f commit 35199c4
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 24 deletions.
2 changes: 1 addition & 1 deletion src/meta/src/hummock/compaction_scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ where
if history.len() >= HISTORY_TABLE_INFO_WINDOW_SIZE {
let window_total_size = history.iter().sum::<u64>();
is_high_write_throughput = history.iter().all(|throughput| {
**throughput > self.env.opts.table_write_throughput_threshold
*throughput > self.env.opts.table_write_throughput_threshold
});
is_low_write_throughput = window_total_size
< (HISTORY_TABLE_INFO_WINDOW_SIZE as u64)
Expand Down
26 changes: 4 additions & 22 deletions src/meta/src/hummock/manager/compaction_group_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -644,29 +644,11 @@ impl<S: MetaStore> HummockManager<S> {

#[named]
pub async fn calculate_compaction_group_statistic(&self) -> Vec<TableGroupInfo> {
let mut infos = vec![];
{
let mut infos = {
let versioning_guard = read_lock!(self, versioning).await;
let table_infos = versioning_guard.version_stats.table_stats.clone();

for (group_id, group) in &versioning_guard.current_version.levels {
let mut group_info = TableGroupInfo {
group_id: *group_id,
..Default::default()
};
for table_id in &group.member_table_ids {
if let Some(stats) = table_infos.get(table_id) {
let table_size = stats.total_key_size + stats.total_value_size;
group_info
.table_statistic
.insert(*table_id, table_size as u64);
group_info.group_size += table_size as u64;
} else {
group_info.table_statistic.insert(*table_id, 0);
}
}
infos.push(group_info);
}
versioning_guard
.current_version
.calculate_compaction_group_statistic()
};
let manager = self.compaction_group_manager.read().await;
for info in &mut infos {
Expand Down
2 changes: 1 addition & 1 deletion src/meta/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
max_compactor_task_multiplier: config.meta.max_compactor_task_multiplier,
split_group_size_limit: config.meta.split_group_size_limit,
move_table_size_limit: config.meta.move_table_size_limit,
table_write_throughput_threshold: config.meta.table_write_throughput_limit,
table_write_throughput_threshold: config.meta.table_write_throughput_threshold,
min_table_split_write_throughput: config.meta.min_table_split_write_throughput,
partition_vnode_count: config.meta.partition_vnode_count,
do_not_config_object_storage_lifecycle: config
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ pub trait HummockVersionExt {
fn level_iter<F: FnMut(&Level) -> bool>(&self, compaction_group_id: CompactionGroupId, f: F);

fn get_object_ids(&self) -> Vec<u64>;

fn calculate_compaction_group_statistic(&self) -> Vec<TableGroupInfo>;
}

pub type BranchedSstInfo = HashMap<CompactionGroupId, /* SST Id */ HummockSstableId>;
Expand Down Expand Up @@ -218,6 +220,62 @@ impl HummockVersionExt for HummockVersion {
.map(|group| group.levels.len() + 1)
.unwrap_or(0)
}

fn calculate_compaction_group_statistic(&self) -> Vec<TableGroupInfo> {
let mut infos = vec![];
for (group_id, group) in &self.levels {
if group.member_table_ids.len() == 1 {
continue;
}
let group_size = group
.levels
.iter()
.map(|level| level.total_file_size)
.sum::<u64>()
+ group.l0.as_ref().unwrap().total_file_size;
let mut table_statistic: HashMap<StateTableId, u64> = HashMap::new();
let member_table_id: HashSet<u32> = HashSet::from_iter(group.member_table_ids.clone());
for level in &group.l0.as_ref().unwrap().sub_levels {
if level.level_type() == LevelType::Overlapping {
continue;
}
for sst in &level.table_infos {
if sst.table_ids.len() > 1 {
// do not calculate size for small state-table.
continue;
}
if !member_table_id.contains(&sst.table_ids[0]) {
continue;
}
let entry = table_statistic.entry(sst.table_ids[0]).or_default();
*entry += sst.file_size;
}
}

for level in &group.levels {
for sst in &level.table_infos {
if sst.table_ids.len() > 1 {
// do not calculate size for small state-table.
continue;
}
for table_id in &sst.table_ids {
if !member_table_id.contains(table_id) {
continue;
}
let entry = table_statistic.entry(*table_id).or_default();
*entry += sst.file_size;
}
}
}
infos.push(TableGroupInfo {
group_id: *group_id,
group_size,
table_statistic,
split_by_table: false,
});
}
infos
}
}

pub type SstSplitInfo = (
Expand Down

0 comments on commit 35199c4

Please sign in to comment.