bench.yml

#
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

data_gen:
  scale_factor: 10000
  parallel: 100
  raw_data_path: PATH_FOR_RAW_DATA # raw NDS csv data
  local_or_hdfs: hdfs
  # data generation is not a timed part in the full benchmark steps, set to "true" by default.
  # the raw_data_path is the location for already generated data when "skip" is "true"
  skip: true
load_test:
  # template to do Iceberg(or DeltaLake, use "delta" keyword template) writing, GPU disabled
  spark_template_path: convert_submit_cpu_iceberg.template
  output_path: PATH_FOR_ICEBERG_OR_DELTA_WAREHOUSE # warehouse location for Iceberg or DeltaLake
  warehouse_type: iceberg # support "iceberg" and "delta", use "iceberg" by default
  report_path: load_test.txt # execution report path
  skip: false
generate_query_stream:
  # how many streams to be generated including the stream for Power Test
  # If there're 4 streams in one Throughput test, num_streams = 4*2+1 according to Spec 4.3.2
  num_streams: 9
  # template dir contains all query templates to generate Spark compatible queries
  # yaml doesn't support resolving environment variable, please use absolute path
  query_template_dir: PATH_TO_TPCDS_HOME/query_templates
  stream_output_path: bench_streams
  skip: false
power_test: # use load test output as input to avoid duplication
  # template with both Iceberg(or DeltaLake) & GPU enabled, use Iceberg by default
  spark_template_path: power_run_gpu_iceberg.template
  report_path: power_test.csv
  property_path: properties/aqe-on.properties # property file contains some Spark configs as a addition to spark template file
  output_path: # leave it empty to use "collect" as Spark action. Otherwise it can be used for data validation
  skip: false
throughput_test: # use most parameters from power test to avoid duplication
  spark_template_path: throughput_run_gpu_iceberg.template # user can copy the one used for Power run and add resource limit
  report_base_path: throughput_report
  skip: false
maintenance_test:
  # template to do data maintenance on Iceberg(or DeltaLake, use "delta" keyword template)
  maintenance_template_path: maintenance_iceberg.template
  query_dir: data_maintenance # folder that contains all data maintenance queries
  maintenance_report_base_path: maintenance_report
  skip: false
metrics_report_path: metrics.csv