Merge branch 'apache:master' into fix-pid-occupied-check

apache · Apr 7, 2022 · ccd5f7d · ccd5f7d
2 parents 828109a + b0030ab
commit ccd5f7d
Show file tree

Hide file tree

Showing 703 changed files with 73,088 additions and 2,647 deletions.
diff --git a/.github/workflows/backward-compat-tests.yml b/.github/workflows/backward-compat-tests.yml
@@ -27,6 +27,7 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:

diff --git a/.github/workflows/bookie-tests.yml b/.github/workflows/bookie-tests.yml
@@ -27,10 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   test:
@@ -50,8 +51,17 @@ jobs:
           distribution: 'temurin'
           java-version: 11
 
-      - name: Run bookie test
-        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" ${GRADLE_ARGS}
+      - name: Maven build bookkeeper-server
+        run: mvn -B -nsu -am -pl bookkeeper-server clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+
+      - name: Run EntryLogTests
+        run: mvn -B -nsu -pl bookkeeper-server test -Dtest="org.apache.bookkeeper.bookie.TestEntryLog" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+
+      - name: Run InterleavedLedgerStorageTest
+        run: mvn -B -nsu -pl bookkeeper-server test -Dtest="org.apache.bookkeeper.bookie.TestInterleavedLederStorage" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+
+      - name: Run bookie tests
+        run: mvn -B -nsu -pl bookkeeper-server test -Dtest="org.apache.bookkeeper.bookie.*Test" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/client-tests.yml b/.github/workflows/client-tests.yml
@@ -27,10 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   test:
@@ -50,7 +51,7 @@ jobs:
           distribution: 'temurin'
           java-version: 11
       - name: Run client tests
-        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" ${GRADLE_ARGS}
+        run: mvn -B -am -nsu -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.client.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml
@@ -27,10 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   check:
@@ -49,9 +50,8 @@ jobs:
         with:
           distribution: 'temurin'
           java-version: 11
-      - name: Build with gradle
-        run: |
-          ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS}
+      - name: Build with Maven
+        run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
       - name: print JVM thread dumps when cancelled
         if: cancelled()
         run: ./dev/ci-tool print_thread_dumps
diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml
@@ -27,10 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   check:
@@ -48,9 +49,8 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
-      - name: Build with gradle
-        run: |
-          ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS}
+      - name: Build with Maven
+        run: mvn clean package spotbugs:check -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/gradle-build.yml b/.github/workflows/gradle-build.yml
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Build with gradle on JDK 11
+
+on:
+  push:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+    paths-ignore:
+      - 'site/**'
+      - 'site3/**'
+    workflow_dispatch:
+
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'temurin'
+          java-version: 11
+
+      - name: Build everything with gradle to validate the build works
+        run: ./gradlew build -x signDistTar -x test
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -27,10 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   test:
@@ -50,14 +51,14 @@ jobs:
           distribution: 'temurin'
           java-version: 11
 
-      - name: Build tar
-        run: ./gradlew stream:server:build -x test ${GRADLE_ARGS}
-      - name: run cluster integration test
-        run: ./gradlew :tests:integration:cluster:test ${GRADLE_ARGS} || (tail -n +1 tests/integration/cluster/build/reports/tests/test/classes/* && tail -n +1 tests/integration/cluster/build/container-logs/**/* && exit 1)
-      - name: run smoke test
-        run: ./gradlew tests:integration:smoke:test ${GRADLE_ARGS}
-      - name: run standalone test
-        run: ./gradlew tests:integration:standalone:test ${GRADLE_ARGS}
+      - name: Build with Maven
+        run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+
+      - name: Run metadata driver tests
+        run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+
+      - name: Run all integration tests
+        run: mvn -B -nsu -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/owasp-dep-check.yml b/.github/workflows/owasp-dep-check.yml
@@ -27,8 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
+env:
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   check:
@@ -42,18 +45,31 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
+      - name: Detect changed pom files
+        id: changes
+        uses: apache/pulsar-test-infra/paths-filter@master
+        with:
+          filters: |
+            poms:
+              - 'pom.xml'
+              - '**/pom.xml'
+
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
+        if: ${{ steps.changes.outputs.poms == 'true' }}
         with:
           java-version: 1.8
 
-      - name: run "clean build dependencyCheckAggregate" to trigger dependency check
-        run: ./gradlew clean build -x signDistTar -x test dependencyCheckAggregate
+      - name: run "clean install verify" to trigger dependency check
+        if: ${{ steps.changes.outputs.poms == 'true' }}
+        # excluding dlfs because it includes hadoop lib with
+        # CVEs that we cannot patch up anyways
+        run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs'
 
       - name: Upload report
         uses: actions/upload-artifact@v2
         if: ${{ cancelled() || failure() }}
         continue-on-error: true
         with:
           name: dependency report
-          path: build/reports/dependency-check-report.html
+          path: target/dependency-check-report.html
diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml
@@ -27,12 +27,13 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
   GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
 
-
 jobs:
   check:
 
@@ -51,10 +52,11 @@ jobs:
           distribution: 'temurin'
           java-version: 11
       - name: Validate pull request
-        run: ./gradlew build -x signDistTar -x test ${GRADLE_ARGS}
+        run: mvn clean -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: Check license files
         run: dev/check-all-licenses
 
+      # keeping on gradle, `mvn site javadoc:javadoc`, to figure out later
       - name: Generate Javadoc
         run: ./gradlew generateApiJavadoc
diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml
@@ -27,11 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
-
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   test:
@@ -51,7 +51,7 @@ jobs:
           distribution: 'temurin'
           java-version: 11
       - name: Run remaining tests
-        run: ./gradlew bookkeeper-server:test -PexcludeTests="**/org/apache/bookkeeper/bookie/*, **/org/apache/bookkeeper/client/*, **/org/apache/bookkeeper/replication/*, **/org/apache/bookkeeper/tls/*" ${GRADLE_ARGS}
+        run: mvn -B -nsu -am -pl bookkeeper-server clean install test -Dtest="!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/replication-tests.yml b/.github/workflows/replication-tests.yml
@@ -27,10 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   test:
@@ -50,7 +51,7 @@ jobs:
           distribution: 'temurin'
           java-version: 11
       - name: Run replication tests
-        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" ${GRADLE_ARGS}
+        run: mvn -B -nsu -am -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.replication.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml
@@ -26,10 +26,11 @@ on:
       - master
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   test:
@@ -48,22 +49,10 @@ jobs:
         with:
           distribution: 'temurin'
           java-version: 11
-      - name: Run stream:distributedlog:core tests
-        run: ./gradlew stream:distributedlog:core:test ${GRADLE_ARGS}
-      - name: Run stream:distributedlog:common tests
-        run: ./gradlew stream:distributedlog:common:test ${GRADLE_ARGS}
-      - name: Run stream:distributedlog:protocol tests
-        run: ./gradlew stream:distributedlog:protocol:test ${GRADLE_ARGS}
-      - name: Run stream:proto tests
-        run: ./gradlew stream:proto:test ${GRADLE_ARGS}
-      - name: Run stream:serve tests
-        run: ./gradlew stream:server:test ${GRADLE_ARGS}
-      - name: Run stream:statelib tests
-        run: ./gradlew stream:statelib:test ${GRADLE_ARGS}
-      - name: Run stream:storage:api:test tests
-        run: ./gradlew stream:storage:api:test ${GRADLE_ARGS}
-      - name: Run stream:storage:impl tests
-        run: ./gradlew stream:storage:impl:test ${GRADLE_ARGS}
+      - name: Build with Maven
+        run: mvn -B -nsu clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+      - name: Run StreamStorage tests
+        run: mvn -B -nsu -f stream/pom.xml verify -DstreamTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml
@@ -27,10 +27,11 @@ on:
       - branch-*
     paths-ignore:
       - 'site/**'
+      - 'site3/**'
     workflow_dispatch:
 
 env:
-  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   test:
@@ -50,7 +51,7 @@ jobs:
           distribution: 'temurin'
           java-version: 11
       - name: Run tls tests
-        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" ${GRADLE_ARGS}
+        run: mvn -B -am -nsu -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.tls.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()