Merge branch 'master' into upgrade-slf4j-2.0.12

.github/ISSUE_TEMPLATE/flaky-test.yml

@@ -18,7 +18,7 @@
 name: Flaky test
 title: "Flaky-test: test_class.test_method"
 description: Report a flaky test failure
-labels: [ "component/test", "flaky-tests" ]
+labels: [ "area/test", "type/flaky-tests" ]
 body:
   - type: markdown
     attributes:

.github/actions/tune-runner-vm/action.yml

@@ -77,12 +77,6 @@ runs:
             # stop Azure Linux agent to save RAM
             sudo systemctl stop walinuxagent.service || true
           
-            # enable docker experimental mode which is
-            # required for using "docker build --squash" / "-Ddocker.squash=true"
-            daemon_json="$(sudo cat /etc/docker/daemon.json  | jq '.experimental = true')"
-            echo "$daemon_json" | sudo tee /etc/docker/daemon.json
-            # restart docker daemon
-            sudo systemctl restart docker
             echo '::endgroup::'
 
             # show memory

.github/workflows/pulsar-ci.yaml

@@ -512,8 +512,8 @@ jobs:
       - name: Build java-test-image docker image
         run: |
           # build docker image
-          mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
-          -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \
+          DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
+          -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \
           -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
 
       - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
@@ -868,8 +868,8 @@ jobs:
         run: |
           # build docker image
           # include building of Connectors, Offloaders and server distros
-          mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
-          -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
+          DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
+          -Pmain,docker -Dmaven.test.skip=true \
           -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
 
       # check full build artifacts licenses

README.md

@@ -192,6 +192,10 @@ Check https://pulsar.apache.org for documentation and examples.
 
 ## Build custom docker images
 
+The commands used in the Apache Pulsar release process can be found in the [release process documentation](https://pulsar.apache.org/contribute/release-process/#stage-docker-images).
+
+Here are some general instructions for building custom docker images:
+
 * Docker images must be built with Java 8 for `branch-2.7` or previous branches because of [ISSUE-8445](https://github.com/apache/pulsar/issues/8445).
 * Java 11 is the recommended JDK version in `branch-2.8`, `branch-2.9` and `branch-2.10`.
 * Java 17 is the recommended JDK version in `master`.
@@ -200,6 +204,8 @@ The following command builds the docker images `apachepulsar/pulsar-all:latest`
 
 ```bash
 mvn clean install -DskipTests
+# setting DOCKER_CLI_EXPERIMENTAL=enabled is required in some environments with older docker versions
+export DOCKER_CLI_EXPERIMENTAL=enabled
 mvn package -Pdocker,-main -am -pl docker/pulsar-all -DskipTests
 ```
 

build/build_java_test_image.sh

@@ -20,12 +20,6 @@
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$SCRIPT_DIR/.."
-SQUASH_PARAM=""
-# check if docker experimental mode is enabled which is required for
-# using "docker build --squash" for squashing all intermediate layers of the build to a single layer
-if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]]; then
-  SQUASH_PARAM="-Ddocker.squash=true"
-fi
 mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \
-  -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \
+  -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true \
   "$@" install

build/run_unit_group.sh

@@ -139,13 +139,21 @@ function print_testng_failures() {
 function test_group_broker_flaky() {
   echo "::endgroup::"
   echo "::group::Running quarantined tests"
-  mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='' -DfailIfNoTests=false \
+  mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='flaky' -DfailIfNoTests=false \
     -DtestForkCount=2 ||
     print_testng_failures pulsar-broker/target/surefire-reports/testng-failed.xml "Quarantined test failure in" "Quarantined test failures"
   echo "::endgroup::"
   echo "::group::Running flaky tests"
-  mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='flaky' -DtestForkCount=2
+  mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='flaky' -DexcludedGroups='quarantine' -DtestForkCount=2
   echo "::endgroup::"
+  local modules_with_flaky_tests=$(git grep -l '@Test.*"flaky"' | grep '/src/test/java/' | \
+    awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker' | sort | uniq | \
+    perl -0777 -p -e 's/\n(\S)/,$1/g')
+  if [ -n "${modules_with_flaky_tests}" ]; then
+    echo "::group::Running flaky tests in modules '${modules_with_flaky_tests}'"
+    mvn_test --no-fail-fast -pl "${modules_with_flaky_tests}" -Dgroups='flaky' -DexcludedGroups='quarantine' -DfailIfNoTests=false
+    echo "::endgroup::"
+  fi
 }
 
 function test_group_proxy() {
@@ -179,7 +187,7 @@ function test_group_other() {
     perl -0777 -p -e 's/\n(\S)/,$1/g')
   if [ -n "${modules_with_quarantined_tests}" ]; then
     echo "::group::Running quarantined tests outside of pulsar-broker & pulsar-proxy (if any)"
-    mvn_test --no-fail-fast -pl "${modules_with_quarantined_tests}" test -Dgroups='quarantine' -DexcludedGroups='' \
+    mvn_test --no-fail-fast -pl "${modules_with_quarantined_tests}" test -Dgroups='quarantine' -DexcludedGroups='flaky' \
       -DfailIfNoTests=false || \
         echo "::warning::There were test failures in the 'quarantine' test group."
     echo "::endgroup::"

buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java

@@ -32,6 +32,10 @@ public class AnnotationListener implements IAnnotationTransformer {
     private static final long DEFAULT_TEST_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5);
     private static final String OTHER_GROUP = "other";
 
+    private static final String FLAKY_GROUP = "flaky";
+
+    private static final String QUARANTINE_GROUP = "quarantine";
+
     public AnnotationListener() {
         System.out.println("Created annotation listener");
     }
@@ -51,9 +55,27 @@ public void transform(ITestAnnotation annotation,
             annotation.setTimeOut(DEFAULT_TEST_TIMEOUT_MILLIS);
         }
 
+        replaceGroupsIfFlakyOrQuarantineGroupIsIncluded(annotation);
         addToOtherGroupIfNoGroupsSpecified(annotation);
     }
 
+    // A test method will inherit the test groups from the class level and this solution ensures that a test method
+    // added to the flaky or quarantine group will not be executed as part of other groups.
+    private void replaceGroupsIfFlakyOrQuarantineGroupIsIncluded(ITestAnnotation annotation) {
+        if (annotation.getGroups() != null && annotation.getGroups().length > 1) {
+            for (String group : annotation.getGroups()) {
+                if (group.equals(QUARANTINE_GROUP)) {
+                    annotation.setGroups(new String[]{QUARANTINE_GROUP});
+                    return;
+                }
+                if (group.equals(FLAKY_GROUP)) {
+                    annotation.setGroups(new String[]{FLAKY_GROUP});
+                    return;
+                }
+            }
+        }
+    }
+
     private void addToOtherGroupIfNoGroupsSpecified(ITestOrConfiguration annotation) {
         // Add test to "other" group if there's no specified group
         if (annotation.getGroups() == null || annotation.getGroups().length == 0) {

docker/pulsar/Dockerfile

@@ -118,5 +118,5 @@ ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin
 
 # The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value.
 ARG DEFAULT_USERNAME=pulsar
-RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D
+RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D -H -h /pulsar/data
 USER 10000

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java

@@ -355,15 +355,19 @@ private CompletableFuture<Void> computeCursorProperties(
             final Function<Map<String, String>, Map<String, String>> updateFunction) {
         CompletableFuture<Void> updateCursorPropertiesResult = new CompletableFuture<>();
 
-        final Stat lastCursorLedgerStat = ManagedCursorImpl.this.cursorLedgerStat;
-
         Map<String, String> newProperties = updateFunction.apply(ManagedCursorImpl.this.cursorProperties);
+        if (!isDurable()) {
+            this.cursorProperties = Collections.unmodifiableMap(newProperties);
+            updateCursorPropertiesResult.complete(null);
+            return updateCursorPropertiesResult;
+        }
+
         ManagedCursorInfo copy = ManagedCursorInfo
                 .newBuilder(ManagedCursorImpl.this.managedCursorInfo)
                 .clearCursorProperties()
                 .addAllCursorProperties(buildStringPropertiesMap(newProperties))
                 .build();
-
+        final Stat lastCursorLedgerStat = ManagedCursorImpl.this.cursorLedgerStat;
         ledger.getStore().asyncUpdateCursorInfo(ledger.getName(),
                 name, copy, lastCursorLedgerStat, new MetaStoreCallback<>() {
                     @Override

managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java

@@ -2446,7 +2446,7 @@ public void testRetentionSize() throws Exception {
         });
     }
 
-    @Test
+    @Test(groups = "flaky")
     public void testTimestampOnWorkingLedger() throws Exception {
         ManagedLedgerConfig conf = new ManagedLedgerConfig();
         conf.setMaxEntriesPerLedger(1);
@@ -3525,7 +3525,7 @@ public void testLedgerReachMaximumRolloverTime() throws Exception {
                 .until(() -> firstLedgerId != ml.addEntry("test".getBytes()).getLedgerId());
     }
 
-    @Test
+    @Test(groups = "flaky")
     public void testLedgerNotRolloverWithoutOpenState() throws Exception {
         ManagedLedgerConfig config = new ManagedLedgerConfig();
         config.setMaxEntriesPerLedger(2);

pom.xml

@@ -92,7 +92,7 @@ flexible messaging model and an intuitive client API.</description>
     <include>**/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java</include>
     <exclude/>
     <groups/>
-    <excludedGroups>quarantine</excludedGroups>
+    <excludedGroups>quarantine,flaky</excludedGroups>
 
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -227,7 +227,7 @@ flexible messaging model and an intuitive client API.</description>
     <hppc.version>0.9.1</hppc.version>
     <spark-streaming_2.10.version>2.1.0</spark-streaming_2.10.version>
     <assertj-core.version>3.24.2</assertj-core.version>
-    <lombok.version>1.18.30</lombok.version>
+    <lombok.version>1.18.32</lombok.version>
     <javax.annotation-api.version>1.3.2</javax.annotation-api.version>
     <jaxb-api>2.3.1</jaxb-api>
     <javax.activation.version>1.2.0</javax.activation.version>

pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java

@@ -304,7 +304,8 @@ private CompletableFuture<DecodedJWT> authenticateToken(String token) {
         return verifyIssuerAndGetJwk(jwt)
                 .thenCompose(jwk -> {
                     try {
-                        if (!jwt.getAlgorithm().equals(jwk.getAlgorithm())) {
+                        // verify the algorithm, if it is set ("alg" is optional in the JWK spec)
+                        if (jwk.getAlgorithm() != null && !jwt.getAlgorithm().equals(jwk.getAlgorithm())) {
                             incrementFailureMetric(AuthenticationExceptionCode.ALGORITHM_MISMATCH);
                             return CompletableFuture.failedFuture(
                                     new AuthenticationException("JWK's alg [" + jwk.getAlgorithm()

pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java

@@ -75,6 +75,7 @@ public class AuthenticationProviderOpenIDIntegrationTest {
     // These are the kid values for JWKs in the /keys endpoint
     String validJwk = "valid";
     String invalidJwk = "invalid";
+    String validJwkWithoutAlg = "valid_without_alg";
 
     // The valid issuer
     String issuer;
@@ -188,10 +189,16 @@ void beforeClass() throws IOException {
                                                 "kty":"RSA",
                                                 "n":"invalid-key",
                                                 "e":"AQAB"
+                                                },
+                                                {
+                                                "kid":"%s",
+                                                "kty":"RSA",
+                                                "n":"%s",
+                                                "e":"%s"
                                                 }
                                             ]
                                         }
-                                        """.formatted(validJwk, n, e, invalidJwk))));
+                                        """.formatted(validJwk, n, e, invalidJwk, validJwkWithoutAlg, n, e))));
 
         server.stubFor(
                 get(urlEqualTo("/missing-kid/.well-known/openid-configuration"))
@@ -275,6 +282,14 @@ public void testTokenWithValidJWK() throws Exception {
         assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get());
     }
 
+    @Test
+    public void testTokenWithValidJWKWithoutAlg() throws Exception {
+        String role = "superuser";
+        // test with a key in JWK that does not have an "alg" field. "alg" is optional in the JWK spec
+        String token = generateToken(validJwkWithoutAlg, issuer, role, "allowed-audience", 0L, 0L, 10000L);
+        assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get());
+    }
+
     @Test
     public void testTokenWithTrailingSlashAndValidJWK() throws Exception {
         String role = "superuser";

pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java

@@ -99,7 +99,7 @@ private static class StarterArguments {
         @Option(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker")
         private String fnWorkerConfigFile = "conf/functions_worker.yml";
 
-        @Option(names = {"-h", "--help"}, description = "Show this help message")
+        @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message")
         private boolean help = false;
 
         @Option(names = {"-g", "--generate-docs"}, description = "Generate docs")

pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java

@@ -161,7 +161,7 @@ private static class Arguments {
                 "--proxy-url"}, description = "Proxy-server URL to which to connect.", required = false)
         private String clusterProxyUrl;
 
-        @Option(names = {"-h", "--help"}, description = "Show this help message")
+        @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message")
         private boolean help = false;
 
         @Option(names = {"-g", "--generate-docs"}, description = "Generate docs")

pulsar-broker/src/main/java/org/apache/pulsar/PulsarVersionStarter.java

@@ -31,7 +31,7 @@ public class PulsarVersionStarter {
 
     @Command(name = "version", showDefaultValues = true, scope = ScopeType.INHERIT)
     private static class Arguments {
-        @Option(names = {"-h", "--help"}, description = "Show this help message")
+        @Option(names = {"-h", "--help"}, usageHelp = true, description = "Show this help message")
         private boolean help = false;
 
         @Option(names = {"-g", "--generate-docs"}, description = "Generate docs")