Merge remote-tracking branch 'apache/main' into prep-37

apache · Mar 28, 2024 · 1b611ea · 1b611ea
2 parents ab1e0a1 + 6f9948b
commit 1b611ea
Show file tree

Hide file tree

Showing 158 changed files with 5,799 additions and 3,497 deletions.
diff --git a/.github/workflows/pr_benchmarks.yml b/.github/workflows/pr_benchmarks.yml
@@ -28,9 +28,10 @@ jobs:
           cd benchmarks
           mkdir data
           
-          # Setup the TPC-H data set with a scale factor of 10
+          # Setup the TPC-H data sets for scale factors 1 and 10
           ./bench.sh data tpch
-          
+          ./bench.sh data tpch10
+
       - name: Generate unique result names
         run: |
           echo "HEAD_LONG_SHA=$(git log -1 --format='%H')" >> "$GITHUB_ENV" 
@@ -44,6 +45,9 @@ jobs:
           cd benchmarks
 
           ./bench.sh run tpch
+          ./bench.sh run tpch_mem
+          ./bench.sh run tpch10
+          ./bench.sh run tpch_mem10
           
           # For some reason this step doesn't seem to propagate the env var down into the script
           if [ -d "results/HEAD" ]; then
@@ -64,6 +68,9 @@ jobs:
           cd benchmarks
 
           ./bench.sh run tpch
+          ./bench.sh run tpch_mem
+          ./bench.sh run tpch10
+          ./bench.sh run tpch_mem10
           
           echo ${{ github.event.issue.number }} > pr
           

diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh
@@ -314,7 +314,7 @@ run_tpch() {
     fi
     TPCH_DIR="${DATA_DIR}/tpch_sf${SCALE_FACTOR}"
 
-    RESULTS_FILE="${RESULTS_DIR}/tpch.json"
+    RESULTS_FILE="${RESULTS_DIR}/tpch_sf${SCALE_FACTOR}.json"
     echo "RESULTS_FILE: ${RESULTS_FILE}"
     echo "Running tpch benchmark..."
     $CARGO_COMMAND --bin tpch -- benchmark datafusion --iterations 5 --path "${TPCH_DIR}" --format parquet -o ${RESULTS_FILE}
@@ -329,7 +329,7 @@ run_tpch_mem() {
     fi
     TPCH_DIR="${DATA_DIR}/tpch_sf${SCALE_FACTOR}"
 
-    RESULTS_FILE="${RESULTS_DIR}/tpch_mem.json"
+    RESULTS_FILE="${RESULTS_DIR}/tpch_mem_sf${SCALE_FACTOR}.json"
     echo "RESULTS_FILE: ${RESULTS_FILE}"
     echo "Running tpch_mem benchmark..."
     # -m means in memory