Merge branch 'master' into compatible-dropwizard-metrics

.asf.yaml

@@ -17,8 +17,26 @@
 # under the License.
 #
 
+# Please refer https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features
+
 github:
   description: "Apache Pinot - A realtime distributed OLAP datastore"
   homepage: https://pinot.apache.org/
   labels:
     - java
+  del_branch_on_merge: true
+  enabled_merge_buttons:
+    squash: true
+    merge: false
+    rebase: false
+  collaborators:
+    - sullis
+    - shenyu0127
+    - tibrewalpratik17
+    - abhioncbr
+    - zhtaoxiang
+    - shounakmk219
+    - itschrispeck
+    - soumitra-st
+    - swaminathanmanish
+    - yashmayya

.github/dependabot.yml

@@ -24,6 +24,7 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    open-pull-requests-limit: 20
 
   - package-ecosystem: "npm"
     directory: "/pinot-controller/src/main/resources"

.github/workflows/build-multi-arch-pinot-docker-image.yml

@@ -45,7 +45,7 @@ jobs:
       commit-id: ${{ steps.generate-build-info.outputs.commit-id }}
       tags: ${{ steps.generate-build-info.outputs.tags }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Generate Build Info
         id: generate-build-info
         env:
@@ -64,17 +64,17 @@ jobs:
     needs: [ generate-build-info ]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
         with:
           platforms: linux/${{ matrix.arch }}
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v3
         name: Set up Docker Buildx
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build and push the Docker image
         env:
           DOCKER_FILE_BASE_DIR: "docker/images/pinot"
@@ -91,15 +91,15 @@ jobs:
     needs: [ generate-build-info, build-pinot-docker-image ]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v3
         name: Set up Docker Buildx
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Create Multi-Arch Manifest
         env:
           TAGS: "${{needs.generate-build-info.outputs.tags}}"

.github/workflows/build-pinot-base-docker-image.yml

@@ -32,15 +32,15 @@ jobs:
         openJdkDist: [ "amazoncorretto", "ms-openjdk" ]
     steps:
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v3
         name: Set up Docker Buildx
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build and push the Docker image
         env:
           OPEN_JDK_DIST: ${{ matrix.openJdkDist }}

.github/workflows/build-pinot-docker-image.yml

@@ -47,15 +47,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v2
+    - uses: docker/setup-buildx-action@v3
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot"

.github/workflows/build-presto-docker-image.yml

@@ -47,15 +47,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v2
+    - uses: docker/setup-buildx-action@v3
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot-presto"

.github/workflows/build-superset-docker-image.yml

@@ -43,15 +43,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v2
+    - uses: docker/setup-buildx-action@v3
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot-superset"

.github/workflows/pinot_compatibility_tests.yml

@@ -36,22 +36,22 @@ jobs:
         test_suite: [ "compatibility-verifier/sample-test-suite" ]
     name: Pinot Compatibility Regression Testing against ${{ github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on ${{ matrix.test_suite }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 11
-          distribution: 'adopt'
+          distribution: 'temurin'
           cache: 'maven'
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: v10.16.1
+          node-version: v16.15.0
           cache: 'npm'
           cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
       - name: Install npm
         run: |
-          npm install -g npm@6.10.0
+          npm install -g npm@8.5.5
           npm --version
       - name: Pinot Compatibility Regression Testing
         if : ${{github.event_name == 'workflow_dispatch'}}
@@ -74,7 +74,7 @@ jobs:
         if: always()
         run: |
           zip -1 -r artifacts.zip /tmp/compatibility-verifier/*
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Store compatibility verifier work directory
         if: always()
         with:

.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml

@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Multi-Stage Query Engine Compatibility Regression Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      oldCommit:
+        description: "Git hash (or tag) for old commit. (required)"
+        required: true
+      newCommit:
+        description: "Git hash (or tag) for new commit. (required)"
+        required: true
+
+jobs:
+  compatibility-verifier:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ]
+    name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on ${{ matrix.test_suite }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: v16.15.0
+          cache: 'npm'
+          cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+      - name: Install npm
+        run: |
+          npm install -g npm@8.5.5
+          npm --version
+      - name: Pinot Multi-Stage Query Engine Compatibility Regression Testing
+        if : ${{github.event_name == 'workflow_dispatch'}}
+        env:
+          OLD_COMMIT: ${{ github.event.inputs.oldCommit }}
+          NEW_COMMIT: ${{ github.event.inputs.newCommit }}
+          WORKING_DIR: /tmp/multi-stage-compatibility-verifier
+          TEST_SUITE: ${{ matrix.test_suite }}
+          MAVEN_OPTS: >
+            -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -XX:+IgnoreUnrecognizedVMOptions
+            --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+            --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
+      - name: Archive artifacts into zip
+        if: always()
+        run: |
+          zip -1 -r artifacts.zip /tmp/multi-stage-compatibility-verifier/*
+      - uses: actions/upload-artifact@v4
+        name: Store multi-stage compatibility verifier work directory
+        if: always()
+        with:
+          ## TODO: currently matrix.test_suite cannot be used as part of name due to invalid path character.
+          name: multi_stage_compatibility_verifier_work_dir
+          retention-days: 3
+          path: artifacts.zip