Skip to content

Commit

Permalink
QueryConfig clean up
Browse files Browse the repository at this point in the history
  • Loading branch information
tanjialiang committed Oct 25, 2024
1 parent c7fe8e7 commit 35b096f
Showing 1 changed file with 16 additions and 33 deletions.
49 changes: 16 additions & 33 deletions velox/core/QueryConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,20 @@ class QueryConfig {
static constexpr const char* kAbandonPartialTopNRowNumberMinPct =
"abandon_partial_topn_row_number_min_pct";

/// The maximum number of bytes to buffer in PartitionedOutput operator to
/// avoid creating tiny SerializedPages.
///
/// For PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator
/// would buffer up to that number of bytes / number of destinations for each
/// destination before producing a SerializedPage.
static constexpr const char* kMaxPartitionedOutputBufferSize =
"max_page_partitioning_buffer_size";

/// The maximum size in bytes for the task's buffered output.
///
/// The producer Drivers are blocked when the buffered size exceeds
/// this. The Drivers are resumed when the buffered size goes below
/// OutputBufferManager::kContinuePct % of this.
static constexpr const char* kMaxOutputBufferSize = "max_output_buffer_size";

/// Preferred size of batches in bytes to be returned by operators from
Expand Down Expand Up @@ -268,10 +279,6 @@ class QueryConfig {
static constexpr const char* kSpillNumPartitionBits =
"spiller_num_partition_bits";

/// !!! DEPRECATED: do not use.
static constexpr const char* kJoinSpillPartitionBits =
"join_spiller_partition_bits";

static constexpr const char* kMinSpillableReservationPct =
"min_spillable_reservation_pct";

Expand All @@ -287,7 +294,7 @@ class QueryConfig {
static constexpr const char* kPrestoArrayAggIgnoreNulls =
"presto.array_agg.ignore_nulls";

// The default number of expected items for the bloomfilter.
/// The default number of expected items for the bloomfilter.
static constexpr const char* kSparkBloomFilterExpectedNumItems =
"spark.bloom_filter.expected_num_items";

Expand Down Expand Up @@ -341,10 +348,10 @@ class QueryConfig {
static constexpr const char* kEnableExpressionEvaluationCache =
"enable_expression_evaluation_cache";

// For a given shared subexpression, the maximum distinct sets of inputs we
// cache results for. Lambdas can call the same expression with different
// inputs many times, causing the results we cache to explode in size. Putting
// a limit contains the memory usage.
/// For a given shared subexpression, the maximum distinct sets of inputs we
/// cache results for. Lambdas can call the same expression with different
/// inputs many times, causing the results we cache to explode in size.
/// Putting a limit contains the memory usage.
static constexpr const char* kMaxSharedSubexprResultsCached =
"max_shared_subexpr_results_cached";

Expand Down Expand Up @@ -474,22 +481,11 @@ class QueryConfig {
return get<uint64_t>(kMaxSpillBytes, kDefault);
}

/// Returns the maximum number of bytes to buffer in PartitionedOutput
/// operator to avoid creating tiny SerializedPages.
///
/// For PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator
/// would buffer up to that number of bytes / number of destinations for each
/// destination before producing a SerializedPage.
uint64_t maxPartitionedOutputBufferSize() const {
static constexpr uint64_t kDefault = 32UL << 20;
return get<uint64_t>(kMaxPartitionedOutputBufferSize, kDefault);
}

/// Returns the maximum size in bytes for the task's buffered output.
///
/// The producer Drivers are blocked when the buffered size exceeds
/// this. The Drivers are resumed when the buffered size goes below
/// OutputBufferManager::kContinuePct % of this.
uint64_t maxOutputBufferSize() const {
static constexpr uint64_t kDefault = 32UL << 20;
return get<uint64_t>(kMaxOutputBufferSize, kDefault);
Expand Down Expand Up @@ -640,19 +636,6 @@ class QueryConfig {
return get<uint8_t>(kSpillStartPartitionBit, kDefaultStartBit);
}

/// Returns the number of bits used to calculate the spill partition number
/// for hash join. The number of spill partitions will be power of two.
///
/// NOTE: as for now, we only support up to 8-way spill partitioning.
///
/// DEPRECATED.
uint8_t joinSpillPartitionBits() const {
constexpr uint8_t kDefaultBits = 3;
constexpr uint8_t kMaxBits = 3;
return std::min(
kMaxBits, get<uint8_t>(kJoinSpillPartitionBits, kDefaultBits));
}

/// Returns the number of bits used to calculate the spill partition number
/// for hash join and RowNumber. The number of spill partitions will be power
/// of two.
Expand Down

0 comments on commit 35b096f

Please sign in to comment.